perf(Accounts): optimize by having a pre-check on ElasticSearch

scripts/search.ts

@@ -252,7 +252,7 @@ program
     }
 
     const indexInputs = indexes.map(index => ({ index }));
-    const result = await elasticSearchGlobalSearch(indexInputs, query, { account, host, limit, user });
+    const result = await elasticSearchGlobalSearch(query, indexInputs, { account, host, limit, user });
     console.log('Result', JSON.stringify(result, null, 2));
   });
 

server/graphql/loaders/search.ts

@@ -17,7 +17,6 @@ type SearchParams = {
   indexParams: ElasticSearchIndexParams[ElasticSearchIndexName];
   forbidPrivate?: boolean;
   limit: number;
-  adminOfAccountIds: number[];
   account: Collective;
   host: Collective;
 };
@@ -86,7 +85,7 @@ export const generateSearchLoaders = req => {
       const firstRequest = groupedRequests[requestId][0];
       const { searchTerm, limit, account, host } = firstRequest;
       const indexes = getSearchIndexes(groupedRequests[requestId]);
-      const results = await elasticSearchGlobalSearch(indexes, searchTerm, {
+      const results = await elasticSearchGlobalSearch(searchTerm, indexes, {
         user: req.remoteUser,
         account,
         host,

server/graphql/v2/query/collection/AccountsCollectionQuery.ts

@@ -1,5 +1,10 @@
 import { GraphQLBoolean, GraphQLList, GraphQLNonNull, GraphQLString } from 'graphql';
+import { isNil, omitBy } from 'lodash';
 
+import { isElasticSearchConfigured } from '../../../../lib/elastic-search/client';
+import { formatIndexNameForElasticSearch } from '../../../../lib/elastic-search/common';
+import { ElasticSearchIndexName } from '../../../../lib/elastic-search/constants';
+import { elasticSearchGlobalSearch } from '../../../../lib/elastic-search/search';
 import { searchCollectivesInDB } from '../../../../lib/sql-search';
 import { GraphQLAccountCollection } from '../../collection/AccountCollection';
 import { AccountTypeToModelMapping, GraphQLAccountType, GraphQLCountryISO } from '../../enum';
@@ -102,7 +107,43 @@ const AccountsCollectionQuery = {
       ? await fetchAccountWithReference(args.includeVendorsForHost).then(({ id }) => id)
       : undefined;
 
+    let preFilteredCollectiveIds: number[];
+    if (
+      isElasticSearchConfigured() &&
+      args.searchTerm &&
+      args.tagSearchOperator === 'AND' &&
+      !args.includeArchived && // Archived collectives are not indexed in ElasticSearch
+      args.skipGuests && // Guests are not indexed in ElasticSearch
+      isNil(args.includeVendorsForHost)
+    ) {
+      // TODO hostCollectiveIds
+      // TODO rate limiting
+      const elasticSearchResult = await elasticSearchGlobalSearch(
+        cleanTerm,
+        {
+          index: ElasticSearchIndexName.COLLECTIVES,
+          indexParams: omitBy(
+            {
+              types: args.type?.length ? args.type.map(value => AccountTypeToModelMapping[value]) : null,
+              isHost: args.isHost,
+              tags: args.tag,
+              isActive: args.onlyActive ? true : null,
+            },
+            isNil,
+          ),
+        },
+        {
+          skipHighlight: true,
+        },
+      );
+
+      const bucketResult = elasticSearchResult.aggregations.by_index['buckets'][0];
+      const hits = bucketResult.top_hits_by_index.hits.hits;
+      preFilteredCollectiveIds = hits.map(hit => hit._source.id);
+    }
+
     const extraParameters = {
+      ids: preFilteredCollectiveIds,
       orderBy: args.orderBy || { field: 'RANK', direction: 'DESC' },
       types: args.type?.length ? args.type.map(value => AccountTypeToModelMapping[value]) : null,
       hostCollectiveIds,

server/lib/elastic-search/constants.ts

@@ -16,5 +16,6 @@ export interface ElasticSearchIndexParams extends Record<ElasticSearchIndexName,
     type?: CollectiveType;
     isHost?: boolean;
     tags?: string[];
+    isActive?: boolean;
   };
 }

server/lib/elastic-search/search.ts

@@ -51,6 +51,7 @@ const getIndexConditions = (
     case ElasticSearchIndexName.COLLECTIVES:
       params = params as ElasticSearchIndexParams[ElasticSearchIndexName.COLLECTIVES];
       return [
+        ...(typeof params.isActive === 'boolean' ? [{ term: { isActive: params.isActive } }] : []),
         ...(params.type ? [{ term: { type: params.type } }] : []),
         ...(!isNil(params.isHost) ? [{ term: { isHostAccount: params.isHost } }] : []),
         ...(!isNil(params.tags) && !isEmpty(params.tags) ? [{ terms: { tags: params.tags } }] : []),
@@ -184,24 +185,27 @@ const buildQuery = (
 };
 
 export const elasticSearchGlobalSearch = async (
-  requestedIndexes: ElasticSearchIndexRequest[],
   searchTerm: string,
+  requestedIndexes: ElasticSearchIndexRequest | Array<ElasticSearchIndexRequest>,
   {
     account,
     host,
     timeoutInSeconds = 30,
     limit = 50,
     user,
+    skipHighlight = false,
   }: {
     account?: Collective;
     host?: Collective;
     timeoutInSeconds?: number;
     limit?: number;
     user?: User;
+    skipHighlight?: boolean;
   } = {},
 ) => {
   const client = getElasticSearchClient({ throwIfUnavailable: true });
-  const { query, searchedFields, indexes } = buildQuery(searchTerm, requestedIndexes, user, account, host);
+  const indexesInputArray = Array.isArray(requestedIndexes) ? requestedIndexes : [requestedIndexes];
+  const { query, searchedFields, indexes } = buildQuery(searchTerm, indexesInputArray, user, account, host);
 
   // Due to permissions, we may end up searching on no index at all (e.g. trying to search for comments while unauthenticated)
   if (indexes.size === 0) {
@@ -232,16 +236,18 @@ export const elasticSearchGlobalSearch = async (
                     // We only need to retrieve the IDs, the rest will be fetched by the loaders
                     includes: ['id', 'uuid'],
                   },
-                  highlight: {
-                    pre_tags: ['<mark>'],
-                    post_tags: ['</mark>'],
-                    fragment_size: 40,
-                    number_of_fragments: 1,
-                    fields: Array.from(searchedFields).reduce((acc, field) => {
-                      acc[field] = {};
-                      return acc;
-                    }, {}),
-                  },
+                  highlight: skipHighlight
+                    ? undefined
+                    : {
+                        pre_tags: ['<mark>'],
+                        post_tags: ['</mark>'],
+                        fragment_size: 40,
+                        number_of_fragments: 1,
+                        fields: Array.from(searchedFields).reduce((acc, field) => {
+                          acc[field] = {};
+                          return acc;
+                        }, {}),
+                      },
                 },
               },
             },

server/lib/sql-search.ts

@@ -210,6 +210,7 @@ export const searchCollectivesInDB = async (
   offset = 0,
   limit = 100,
   {
+    ids,
     countries,
     hasCustomContributionsEnabled,
     hostCollectiveIds,
@@ -226,6 +227,7 @@ export const searchCollectivesInDB = async (
     types,
     ...args
   }: {
+    ids?: number[];
     countries?: string[];
     currency?: string;
     hasCustomContributionsEnabled?: boolean;
@@ -342,6 +344,7 @@ export const searchCollectivesInDB = async (
     AND c.name != 'incognito'
     AND c.name != 'anonymous'
     AND c."isIncognito" = FALSE ${dynamicConditions}
+    ${!isEmpty(ids) ? `AND c.id IN (:ids)` : ''}
     ${!isEmpty(args.consolidatedBalance) ? `AND ${CONSOLIDATED_BALANCE_SUBQUERY} ${getAmountRangeQuery(args.consolidatedBalance)}` : ''}
     ORDER BY __sort__ ${orderBy?.direction || 'DESC'}
     OFFSET :offset
@@ -351,6 +354,7 @@ export const searchCollectivesInDB = async (
       model: models.Collective,
       mapToModel: true,
       replacements: {
+        ids,
         types,
         term: term,
         slugifiedTerm: term ? slugify(term) : '',