From 4fa3d6d333154a5be84991fe497cb37da8db2411 Mon Sep 17 00:00:00 2001 From: Timothy Carambat Date: Thu, 19 Sep 2024 13:34:38 -0500 Subject: [PATCH] Load all branches in gitlab data connector (#2325) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix gitlab data connector for self-hosted instances (#2315) * Linting fix. * Load all branches in the GitLab data connector #2319 * #2319 lint fixes. * update fetch on fail --------- Co-authored-by: Błażej Owczarczyk --- .../RepoLoader/GitlabRepo/RepoLoader/index.js | 67 ++++++++++++------- 1 file changed, 44 insertions(+), 23 deletions(-) diff --git a/collector/utils/extensions/RepoLoader/GitlabRepo/RepoLoader/index.js b/collector/utils/extensions/RepoLoader/GitlabRepo/RepoLoader/index.js index 1a3737c383..7558322985 100644 --- a/collector/utils/extensions/RepoLoader/GitlabRepo/RepoLoader/index.js +++ b/collector/utils/extensions/RepoLoader/GitlabRepo/RepoLoader/index.js @@ -159,33 +159,54 @@ class GitLabRepoLoader { async getRepoBranches() { if (!this.#validGitlabUrl() || !this.projectId) return []; await this.#validateAccessToken(); + this.branches = []; + let fetching = true; + let page = 1; + let perPage = 50; - try { - this.branches = await fetch( - `${this.apiBase}/api/v4/projects/${this.projectId}/repository/branches`, - { - method: "GET", - headers: { - Accepts: "application/json", - ...(this.accessToken ? { "PRIVATE-TOKEN": this.accessToken } : {}), - }, - } - ) - .then((res) => res.json()) - .then((branches) => { - return branches.map((b) => b.name); - }) - .catch((e) => { - console.error(e); - return []; + while (fetching) { + try { + const params = new URLSearchParams({ + per_page: perPage, + page, }); + const response = await fetch( + `${this.apiBase}/api/v4/projects/${ + this.projectId + }/repository/branches?${params.toString()}`, + { + method: "GET", + headers: { + Accepts: "application/json", + ...(this.accessToken + ? { "PRIVATE-TOKEN": this.accessToken } + : {}), + }, + } + ) + .then((res) => res.json()) + .then((branches) => { + if (!Array.isArray(branches) || branches.length === 0) { + fetching = false; + return []; + } + return branches.map((b) => b.name); + }) + .catch((e) => { + console.error(e); + fetching = false; + return []; + }); - return this.#branchPrefSort(this.branches); - } catch (err) { - console.log(`RepoLoader.getRepoBranches`, err); - this.branches = []; - return []; + this.branches.push(...response); + page++; + } catch (err) { + console.log(`RepoLoader.getRepoBranches`, err); + fetching = false; + return []; + } } + return this.#branchPrefSort(this.branches); } /**