Skip to content
This repository has been archived by the owner on Nov 4, 2024. It is now read-only.

Stop going through artifact documents when querying CouchDB for several runs #263

Merged
merged 6 commits into from
Aug 15, 2024
Merged

Stop going through artifact documents when querying CouchDB for several runs #263

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
0026c13
Avoid loading artifact paths when getting several runs
eamansour Aug 14, 2024
73e7faa
Increase couchdb query results limit to 100 instead of 25
eamansour Aug 14, 2024
d65ed22
Move code to build runs queries into a separate function
eamansour Aug 14, 2024
212eada
Merge branch 'main' into iss1921-runs-get-performance
eamansour Aug 14, 2024
7440dd9
Empty commit to kick off build
eamansour Aug 14, 2024
90f4b13
Add more search criteria to unit test
eamansour Aug 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 67 additions & 58 deletions ...sa.ras.couchdb/src/main/java/dev/galasa/ras/couchdb/internal/CouchdbDirectoryService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,8 @@ public class CouchdbDirectoryService implements IResultArchiveStoreDirectoryServ

private final CouchdbRasStore store;

private static final int COUCHDB_RESULTS_LIMIT_PER_QUERY = 100;

public CouchdbDirectoryService(CouchdbRasStore store, LogFactory logFactory, HttpRequestFactory requestFactory) {
this.store = store;
this.logFactory = logFactory;
Expand All @@ -85,10 +87,13 @@ public boolean isLocal() {
return false;
}

private Path getRunArtifactPath(TestStructureCouchdb ts) throws CouchdbRasException {

private CouchdbRasFileSystemProvider createFileSystemProvider() {
ResultArchiveStoreFileStore fileStore = new ResultArchiveStoreFileStore();
CouchdbRasFileSystemProvider runProvider = new CouchdbRasFileSystemProvider(fileStore, store, logFactory);
return new CouchdbRasFileSystemProvider(fileStore, store, logFactory);
}

private Path getRunArtifactPath(TestStructureCouchdb ts) throws CouchdbRasException {
CouchdbRasFileSystemProvider runProvider = createFileSystemProvider();
if (ts.getArtifactRecordIds() == null || ts.getArtifactRecordIds().isEmpty()) {
return runProvider.getRoot();
}
Expand Down Expand Up @@ -327,58 +332,10 @@ private CouchdbRunResult fetchRun(String id) throws ParseException, IOException,

HttpPost httpPost = requestFactory.getHttpPostRequest(store.getCouchdbUri() + "/galasa_run/_find");

JsonObject selector = new JsonObject();
JsonArray and = new JsonArray();
selector.add("$and", and);

for(IRasSearchCriteria searchCriteria : searchCriterias) {
if (searchCriteria instanceof RasSearchCriteriaRequestor) {
RasSearchCriteriaRequestor sRequestor = (RasSearchCriteriaRequestor) searchCriteria;

inArray(and, "requestor", sRequestor.getRequestors());
} else if (searchCriteria instanceof RasSearchCriteriaRunName) {
RasSearchCriteriaRunName sRunName = (RasSearchCriteriaRunName) searchCriteria;

inArray(and, "runName", sRunName.getRunNames());
} else if (searchCriteria instanceof RasSearchCriteriaQueuedFrom) {
RasSearchCriteriaQueuedFrom sFrom = (RasSearchCriteriaQueuedFrom) searchCriteria;

JsonObject criteria = new JsonObject();
JsonObject jFrom = new JsonObject();
jFrom.addProperty("$gte", sFrom.getFrom().toString());
criteria.add("queued", jFrom);
and.add(criteria);
} else if (searchCriteria instanceof RasSearchCriteriaQueuedTo) {
RasSearchCriteriaQueuedTo sTo = (RasSearchCriteriaQueuedTo) searchCriteria;

JsonObject criteria = new JsonObject();
JsonObject jTo = new JsonObject();
jTo.addProperty("$lt", sTo.getTo().toString());
criteria.add("queued", jTo);
and.add(criteria);
} else if (searchCriteria instanceof RasSearchCriteriaTestName) {
RasSearchCriteriaTestName sTestName = (RasSearchCriteriaTestName) searchCriteria;

inArray(and, "testName", sTestName.getTestNames());
} else if (searchCriteria instanceof RasSearchCriteriaBundle) {
RasSearchCriteriaBundle sBundle = (RasSearchCriteriaBundle) searchCriteria;

inArray(and, "bundle", sBundle.getBundles());
} else if (searchCriteria instanceof RasSearchCriteriaResult) {
RasSearchCriteriaResult sResult = (RasSearchCriteriaResult) searchCriteria;

inArray(and, "result", sResult.getResults());
} else if(searchCriteria instanceof RasSearchCriteriaStatus){
RasSearchCriteriaStatus sStatus = (RasSearchCriteriaStatus) searchCriteria;
inArray(and, "status", sStatus.getStatusesAsStrings());
} else {
throw new ResultArchiveStoreException("Unrecognised search criteria class " + searchCriteria.getClass().getName());
}
}

Find find = new Find();
find.selector = selector;
find.selector = buildGetRunsQuery(searchCriterias);
find.execution_stats = true;
find.limit = COUCHDB_RESULTS_LIMIT_PER_QUERY;

while (true) {
String requestContent = store.getGson().toJson(find);
Expand Down Expand Up @@ -407,12 +364,12 @@ private CouchdbRunResult fetchRun(String id) throws ParseException, IOException,
}

for (TestStructureCouchdb ts : found.docs) {
Path runArtifactPath = getRunArtifactPath(ts);

// *** Add this run to the results
CouchdbRunResult cdbrr = new CouchdbRunResult(store, ts, runArtifactPath);
if (ts.isValid()) {
runs.add(cdbrr);
// Don't load the artifacts for the found runs, just set a root location for artifacts
CouchdbRasFileSystemProvider runProvider = createFileSystemProvider();

// Add this run to the results
runs.add(new CouchdbRunResult(store, ts, runProvider.getRoot()));
}
}

Expand Down Expand Up @@ -501,6 +458,58 @@ private String getRevision(String databaseName, String id) throws ResultArchiveS
}
}

private JsonObject buildGetRunsQuery(IRasSearchCriteria... searchCriterias) throws ResultArchiveStoreException {
JsonObject selector = new JsonObject();
JsonArray and = new JsonArray();
selector.add("$and", and);

for(IRasSearchCriteria searchCriteria : searchCriterias) {
if (searchCriteria instanceof RasSearchCriteriaRequestor) {
RasSearchCriteriaRequestor sRequestor = (RasSearchCriteriaRequestor) searchCriteria;

inArray(and, "requestor", sRequestor.getRequestors());
} else if (searchCriteria instanceof RasSearchCriteriaRunName) {
RasSearchCriteriaRunName sRunName = (RasSearchCriteriaRunName) searchCriteria;

inArray(and, "runName", sRunName.getRunNames());
} else if (searchCriteria instanceof RasSearchCriteriaQueuedFrom) {
RasSearchCriteriaQueuedFrom sFrom = (RasSearchCriteriaQueuedFrom) searchCriteria;

JsonObject criteria = new JsonObject();
JsonObject jFrom = new JsonObject();
jFrom.addProperty("$gte", sFrom.getFrom().toString());
criteria.add("queued", jFrom);
and.add(criteria);
} else if (searchCriteria instanceof RasSearchCriteriaQueuedTo) {
RasSearchCriteriaQueuedTo sTo = (RasSearchCriteriaQueuedTo) searchCriteria;

JsonObject criteria = new JsonObject();
JsonObject jTo = new JsonObject();
jTo.addProperty("$lt", sTo.getTo().toString());
criteria.add("queued", jTo);
and.add(criteria);
} else if (searchCriteria instanceof RasSearchCriteriaTestName) {
RasSearchCriteriaTestName sTestName = (RasSearchCriteriaTestName) searchCriteria;

inArray(and, "testName", sTestName.getTestNames());
} else if (searchCriteria instanceof RasSearchCriteriaBundle) {
RasSearchCriteriaBundle sBundle = (RasSearchCriteriaBundle) searchCriteria;

inArray(and, "bundle", sBundle.getBundles());
} else if (searchCriteria instanceof RasSearchCriteriaResult) {
RasSearchCriteriaResult sResult = (RasSearchCriteriaResult) searchCriteria;

inArray(and, "result", sResult.getResults());
} else if(searchCriteria instanceof RasSearchCriteriaStatus) {
RasSearchCriteriaStatus sStatus = (RasSearchCriteriaStatus) searchCriteria;
inArray(and, "status", sStatus.getStatusesAsStrings());
} else {
throw new ResultArchiveStoreException("Unrecognised search criteria class " + searchCriteria.getClass().getName());
}
}
return selector;
}

private void inArray(JsonArray and, String field, String[] inArray) {
if (inArray == null || inArray.length < 1) {
return;
Expand Down
Loading