From 16c285458b7510dd50f866af837f28344b8cc001 Mon Sep 17 00:00:00 2001
From: Benjamin Piouffle <benjamin@opencollective.com>
Date: Fri, 27 Dec 2024 17:31:00 +0100
Subject: [PATCH] perf(ElasticSearch): simplify IDs handling in SQL queries for
 sync performance

---
 server/lib/elastic-search/sync.ts | 42 +++++++++++++++++++++++++++++--
 1 file changed, 40 insertions(+), 2 deletions(-)

diff --git a/server/lib/elastic-search/sync.ts b/server/lib/elastic-search/sync.ts
index e7fe66c74bf..64e96ca661e 100644
--- a/server/lib/elastic-search/sync.ts
+++ b/server/lib/elastic-search/sync.ts
@@ -3,7 +3,7 @@
  */
 
 import config from 'config';
-import { chunk } from 'lodash';
+import { chunk, partition } from 'lodash';
 
 import { Op } from '../../models';
 import logger from '../logger';
@@ -53,6 +53,38 @@ async function removeDeletedEntries(indexName: ElasticSearchIndexName, fromDate:
   } while (deletedEntries.length === pageSize);
 }
 
+/**
+ * Takes a list like [1,2,3,4,0,165,7,8,9] and simplifies it to [[0,4],[7,9],165]
+ * to reduce the cost of running SQL queries with large IN clauses.
+ */
+const simplifyNumbersInRanges = (
+  ids: number[],
+): {
+  ranges: number[][];
+  lonelyNumbers: number[];
+} => {
+  // We need the list to be sorted
+  ids.sort((a, b) => a - b);
+  const results = ids.reduce(
+    (ranges, id) => {
+      const lastRange = ranges[ranges.length - 1];
+      if (Array.isArray(lastRange) && lastRange[1] + 1 === id) {
+        lastRange[1] = id; // Extend the range
+      } else if (typeof lastRange === 'number' && id === lastRange + 1) {
+        ranges[ranges.length - 1] = [lastRange, id]; // Convert the number to a range
+      } else {
+        ranges.push(id); // Add a lonely number
+      }
+
+      return ranges;
+    },
+    [] as (number | number[])[],
+  );
+
+  const [lonelyNumbers, ranges] = partition(results, x => typeof x === 'number');
+  return { ranges, lonelyNumbers };
+};
+
 export async function restoreUndeletedEntries(indexName: ElasticSearchIndexName, { log = false } = {}) {
   const client = getElasticSearchClient({ throwIfUnavailable: true });
   const adapter = ElasticSearchModelsAdapters[indexName];
@@ -85,10 +117,16 @@ export async function restoreUndeletedEntries(indexName: ElasticSearchIndexName,
   /* eslint-enable camelcase */
 
   // Search for entries that are not marked as deleted in the database
+  const { ranges, lonelyNumbers } = simplifyNumbersInRanges(allIds.map(Number));
   const undeletedEntries = (await adapter.getModel().findAll({
     attributes: ['id'],
-    where: { id: { [Op.not]: allIds } },
     raw: true,
+    where: {
+      [Op.and]: [
+        { id: { [Op.notIn]: lonelyNumbers } },
+        ...ranges.map(([start, end]) => ({ id: { [Op.notBetween]: [start, end] } })),
+      ],
+    },
   })) as unknown as Array<{ id: number }>;
 
   if (!undeletedEntries.length) {