Load all branches in gitlab data connector (#2325)

* Fix gitlab data connector for self-hosted instances (#2315) * Linting fix. * Load all branches in the GitLab data connector #2319 * #2319 lint fixes. * update fetch on fail --------- Co-authored-by: Błażej Owczarczyk <[email protected]>
Mintplex-Labs · Sep 19, 2024 · 4fa3d6d · 4fa3d6d
1 parent b7c7c0d
commit 4fa3d6d
Showing 1 changed file with 44 additions and 23 deletions.
diff --git a/collector/utils/extensions/RepoLoader/GitlabRepo/RepoLoader/index.js b/collector/utils/extensions/RepoLoader/GitlabRepo/RepoLoader/index.js
@@ -159,33 +159,54 @@ class GitLabRepoLoader {
   async getRepoBranches() {
     if (!this.#validGitlabUrl() || !this.projectId) return [];
     await this.#validateAccessToken();
+    this.branches = [];
+    let fetching = true;
+    let page = 1;
+    let perPage = 50;
 
-    try {
-      this.branches = await fetch(
-        `${this.apiBase}/api/v4/projects/${this.projectId}/repository/branches`,
-        {
-          method: "GET",
-          headers: {
-            Accepts: "application/json",
-            ...(this.accessToken ? { "PRIVATE-TOKEN": this.accessToken } : {}),
-          },
-        }
-      )
-        .then((res) => res.json())
-        .then((branches) => {
-          return branches.map((b) => b.name);
-        })
-        .catch((e) => {
-          console.error(e);
-          return [];
+    while (fetching) {
+      try {
+        const params = new URLSearchParams({
+          per_page: perPage,
+          page,
         });
+        const response = await fetch(
+          `${this.apiBase}/api/v4/projects/${
+            this.projectId
+          }/repository/branches?${params.toString()}`,
+          {
+            method: "GET",
+            headers: {
+              Accepts: "application/json",
+              ...(this.accessToken
+                ? { "PRIVATE-TOKEN": this.accessToken }
+                : {}),
+            },
+          }
+        )
+          .then((res) => res.json())
+          .then((branches) => {
+            if (!Array.isArray(branches) || branches.length === 0) {
+              fetching = false;
+              return [];
+            }
+            return branches.map((b) => b.name);
+          })
+          .catch((e) => {
+            console.error(e);
+            fetching = false;
+            return [];
+          });
 
-      return this.#branchPrefSort(this.branches);
-    } catch (err) {
-      console.log(`RepoLoader.getRepoBranches`, err);
-      this.branches = [];
-      return [];
+        this.branches.push(...response);
+        page++;
+      } catch (err) {
+        console.log(`RepoLoader.getRepoBranches`, err);
+        fetching = false;
+        return [];
+      }
     }
+    return this.#branchPrefSort(this.branches);
   }
 
   /**