From 134add1898807a7fac9bc5fb86605c9cb580616e Mon Sep 17 00:00:00 2001 From: KillianG Date: Thu, 3 Oct 2024 16:09:33 +0200 Subject: [PATCH 01/22] Fix drctl status and failover Issue: ZENKO-4903 --- tests/ctst/steps/dr/drctl.ts | 3 ++- tests/ctst/steps/pra.ts | 15 ++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/ctst/steps/dr/drctl.ts b/tests/ctst/steps/dr/drctl.ts index 3bef123930..a29e1db425 100644 --- a/tests/ctst/steps/dr/drctl.ts +++ b/tests/ctst/steps/dr/drctl.ts @@ -109,7 +109,7 @@ type FailoverConfig = { sinkKubeconfigPath?: string; sinkKubeconfigData?: string; sinkZenkoInstance?: string; - sinkZenkoNamespace?: string; + sinkZenkoDrNamespace?: string; }; type FailbackConfig = { @@ -148,6 +148,7 @@ type StatusConfig = { sourceZenkoDrInstance?: string; sinkZenkoDrInstance?: string; output?: string; + outputFormat?: string; }; type ReplicationPauseConfig = { diff --git a/tests/ctst/steps/pra.ts b/tests/ctst/steps/pra.ts index 88f13b2a0c..b892818058 100644 --- a/tests/ctst/steps/pra.ts +++ b/tests/ctst/steps/pra.ts @@ -111,15 +111,14 @@ async function waitForPhase( sourceZenkoNamespace: 'default', sinkZenkoDrInstance: 'end2end-pra-sink', sourceZenkoDrInstance: 'end2end-source', - output: 'json', + outputFormat: 'json', }); if (!currentStatus) { - world.logger.debug('Failed to get DR status, retrying', { + world.logger.debug('Failed to get DR status', { currentStatus, }); - await Utils.sleep(1000); - continue; + throw new Error('Failed to get DR status'); } const lines = currentStatus.split('\n'); @@ -218,7 +217,8 @@ Then('the DR sink should be in phase {string}', { timeout: 360000 }, async funct throw new Error(`Unknown state ${state}`); } - await waitForPhase(this, 'sink', targetPhase); + const res = await waitForPhase(this, 'sink', targetPhase); + assert(res); }); Then('the DR source should be in phase {string}', { timeout: 360000 }, async function (this: Zenko, state: string) { @@ -246,7 +246,8 @@ Then('the DR source should be in phase {string}', { timeout: 360000 }, async fun throw new Error(`Unknown state ${state}`); } - await waitForPhase(this, 'source', targetPhase); + const res = await waitForPhase(this, 'source', targetPhase); + assert(res); }); Then('object {string} should {string} be {string} and have the storage class {string} on {string} site', @@ -323,7 +324,7 @@ Then('the kafka DR volume exists', { timeout: volumeTimeout + 2000 }, async func const failoverTimeout = 360000; When ('I request the failover state for the DR', { timeout: failoverTimeout + 2000 }, async function (this: Zenko) { await this.zenkoDrCtl?.failover({ - sinkZenkoNamespace: 'default', + sinkZenkoDrNamespace: 'default', wait: true, timeout: `${failoverTimeout.toString()}ms`, }); From 9fb37de6fdbd9b96a90c0b5207414ac818a39895 Mon Sep 17 00:00:00 2001 From: KillianG Date: Fri, 4 Oct 2024 08:04:02 +0200 Subject: [PATCH 02/22] Fix failover Issue: ZENKO-4903 --- tests/ctst/steps/dr/drctl.ts | 6 +++--- tests/ctst/steps/pra.ts | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/ctst/steps/dr/drctl.ts b/tests/ctst/steps/dr/drctl.ts index a29e1db425..7f03a61094 100644 --- a/tests/ctst/steps/dr/drctl.ts +++ b/tests/ctst/steps/dr/drctl.ts @@ -108,7 +108,7 @@ type FailoverConfig = { timeout?: string; sinkKubeconfigPath?: string; sinkKubeconfigData?: string; - sinkZenkoInstance?: string; + sinkZenkoDrInstance?: string; sinkZenkoDrNamespace?: string; }; @@ -117,8 +117,8 @@ type FailbackConfig = { timeout?: string; sinkKubeconfigPath?: string; sinkKubeconfigData?: string; - sinkZenkoInstance?: string; - sinkZenkoNamespace?: string; + sinkZenkoDrInstance?: string; + sinkZenkoDrNamespace?: string; }; type UninstallConfig = { diff --git a/tests/ctst/steps/pra.ts b/tests/ctst/steps/pra.ts index b892818058..e78f24ae5f 100644 --- a/tests/ctst/steps/pra.ts +++ b/tests/ctst/steps/pra.ts @@ -325,6 +325,7 @@ const failoverTimeout = 360000; When ('I request the failover state for the DR', { timeout: failoverTimeout + 2000 }, async function (this: Zenko) { await this.zenkoDrCtl?.failover({ sinkZenkoDrNamespace: 'default', + sinkZenkoDrInstance: 'end2end-pra-sink', wait: true, timeout: `${failoverTimeout.toString()}ms`, }); @@ -333,7 +334,8 @@ When ('I request the failover state for the DR', { timeout: failoverTimeout + 20 const failbackTimeout = 360000; When ('I resume operations for the DR', { timeout: failbackTimeout + 2000 }, async function (this: Zenko) { await this.zenkoDrCtl?.failback({ - sinkZenkoNamespace: 'default', + sinkZenkoDrNamespace: 'default', + sinkZenkoDrInstance: 'end2end-pra-sink', wait: true, timeout: `${failbackTimeout.toString()}ms`, }); From 03039e0292b66dc96af4fb65b32a9f11526753fe Mon Sep 17 00:00:00 2001 From: KillianG Date: Fri, 4 Oct 2024 08:48:01 +0200 Subject: [PATCH 03/22] Fix drctl replication pause Issue: ZENKO-4903 --- tests/ctst/steps/dr/drctl.ts | 6 ++---- tests/ctst/steps/pra.ts | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/ctst/steps/dr/drctl.ts b/tests/ctst/steps/dr/drctl.ts index 7f03a61094..b6f84a591d 100644 --- a/tests/ctst/steps/dr/drctl.ts +++ b/tests/ctst/steps/dr/drctl.ts @@ -156,10 +156,8 @@ type ReplicationPauseConfig = { sourceKubeconfigData?: string; sinkKubeconfigPath?: string; sinkKubeconfigData?: string; - sourceZenkoInstance?: string; - sourceZenkoNamespace?: string; - sinkZenkoInstance?: string; - sinkZenkoNamespace?: string; + sourceZenkoDrNamespace?: string; + sinkZenkoDrNamespace?: string; sourceZenkoDrInstance?: string; sinkZenkoDrInstance?: string; wait?: boolean; diff --git a/tests/ctst/steps/pra.ts b/tests/ctst/steps/pra.ts index e78f24ae5f..88f971f0bf 100644 --- a/tests/ctst/steps/pra.ts +++ b/tests/ctst/steps/pra.ts @@ -346,8 +346,8 @@ When('I pause the DR', { timeout: pauseTimeout + 2000 }, async function (this: Z await this.zenkoDrCtl?.replicationPause({ sourceZenkoDrInstance: 'end2end-source', sinkZenkoDrInstance: 'end2end-pra-sink', - sinkZenkoNamespace: 'default', - sourceZenkoNamespace: 'default', + sinkZenkoDrNamespace: 'default', + sourceZenkoDrNamespace: 'default', wait: true, timeout: `${pauseTimeout.toString()}ms`, }); From cbe6ff022bcb8b6f05319fc5bda48ea7d8a85c3f Mon Sep 17 00:00:00 2001 From: KillianG Date: Fri, 4 Oct 2024 10:28:04 +0200 Subject: [PATCH 04/22] Add timeout for not found as during a replication pause it is normal not to find the object Issue: ZENKO-4903 --- tests/ctst/steps/dr/drctl.ts | 16 ++++++---------- tests/ctst/steps/pra.ts | 10 +++++----- tests/ctst/steps/utils/utils.ts | 6 ++++++ 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/tests/ctst/steps/dr/drctl.ts b/tests/ctst/steps/dr/drctl.ts index b6f84a591d..c2f09a70cf 100644 --- a/tests/ctst/steps/dr/drctl.ts +++ b/tests/ctst/steps/dr/drctl.ts @@ -122,18 +122,16 @@ type FailbackConfig = { }; type UninstallConfig = { - sinkZenkoDrInstance?: string; - sourceZenkoDrInstance?: string; wait?: boolean; timeout?: string; sourceKubeconfigPath?: string; sourceKubeconfigData?: string; sinkKubeconfigPath?: string; sinkKubeconfigData?: string; - sinkZenkoInstance?: string; - sinkZenkoNamespace?: string; - sourceZenkoInstance?: string; - sourceZenkoNamespace?: string; + sinkZenkoDrInstance?: string; + sinkZenkoDrNamespace?: string; + sourceZenkoDrInstance?: string; + sourceZenkoDrNamespace?: string; }; type StatusConfig = { @@ -169,10 +167,8 @@ type ReplicationResumeConfig = { sourceKubeconfigData?: string; sinkKubeconfigPath?: string; sinkKubeconfigData?: string; - sourceZenkoInstance?: string; - sourceZenkoNamespace?: string; - sinkZenkoInstance?: string; - sinkZenkoNamespace?: string; + sourceZenkoDrNamespace?: string; + sinkZenkoDrNamespace?: string; sourceZenkoDrInstance?: string; sinkZenkoDrInstance?: string; wait?: boolean; diff --git a/tests/ctst/steps/pra.ts b/tests/ctst/steps/pra.ts index 88f971f0bf..e8482f4ff3 100644 --- a/tests/ctst/steps/pra.ts +++ b/tests/ctst/steps/pra.ts @@ -358,10 +358,10 @@ When('I resume the DR', { timeout: resumeTimeout + 2000 }, async function (this: await this.zenkoDrCtl?.replicationResume({ sourceZenkoDrInstance: 'end2end-source', sinkZenkoDrInstance: 'end2end-pra-sink', - sinkZenkoNamespace: 'default', - sourceZenkoNamespace: 'default', + sinkZenkoDrNamespace: 'default', + sourceZenkoDrNamespace: 'default', wait: true, - timeout: `${resumeTimeout.toString()}ms`, + timeout: `${pauseTimeout.toString()}ms`, }); }); @@ -370,8 +370,8 @@ When('I uninstall DR', { timeout: uninstallTimeout + 2000 }, async function (thi await this.zenkoDrCtl?.uninstall({ sourceZenkoDrInstance: 'end2end-source', sinkZenkoDrInstance: 'end2end-pra-sink', - sinkZenkoNamespace: 'default', - sourceZenkoNamespace: 'default', + sinkZenkoDrNamespace: 'default', + sourceZenkoDrNamespace: 'default', wait: true, timeout: `${uninstallTimeout.toString()}ms`, }); diff --git a/tests/ctst/steps/utils/utils.ts b/tests/ctst/steps/utils/utils.ts index 8f8ec9bc36..be573d5cc8 100644 --- a/tests/ctst/steps/utils/utils.ts +++ b/tests/ctst/steps/utils/utils.ts @@ -307,9 +307,15 @@ async function verifyObjectLocation(this: Zenko, objectName: string, this.addCommandParameter({ versionId }); } let conditionOk = false; + + const startTime = Date.now(); + while (!conditionOk) { const res = await S3.headObject(this.getCommandParameters()); if (res.err?.includes('NotFound')) { + if (Date.now() - startTime > 300000) { + throw new Error('Object not found after 300 seconds'); + } await Utils.sleep(1000); continue; } else if (res.err) { From 40f66f01327ef6f6bf38adab09c89362b28776ad Mon Sep 17 00:00:00 2001 From: williamlardier Date: Mon, 30 Sep 2024 13:45:46 +0200 Subject: [PATCH 05/22] Make sorbet endpoint dynamic with the current zenko instance name Issue: ZENKO-4909 --- .github/scripts/end2end/configs/zenko.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/scripts/end2end/configs/zenko.yaml b/.github/scripts/end2end/configs/zenko.yaml index 0f0c08da54..9465ee2013 100644 --- a/.github/scripts/end2end/configs/zenko.yaml +++ b/.github/scripts/end2end/configs/zenko.yaml @@ -101,6 +101,9 @@ spec: enable: true configurationOverrides: e2e-cold: + # for mock DMF, we need to override the endpoint to use in-cluster service: otherwise it + # relies on external hostname (ok thanks to coredns patch), but TLS cert is not valid + s3-endpoint: http://${ZENKO_NAME}-internal-s3api.default.svc.cluster.local debug: "true" command-timeout: "60s" pending-job-poll-after-age: "10s" From 11c0eba97e0982e21f764a1bdc6d9817d864c30c Mon Sep 17 00:00:00 2001 From: williamlardier Date: Mon, 30 Sep 2024 13:47:16 +0200 Subject: [PATCH 06/22] Sleep between ARWWI calls in PRA - Otherwise, we might be overloading the Vault service with multiple API calls at high frequency. Issue: ZENKO-4909 --- tests/ctst/steps/pra.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/ctst/steps/pra.ts b/tests/ctst/steps/pra.ts index e8482f4ff3..c0f61fc3c3 100644 --- a/tests/ctst/steps/pra.ts +++ b/tests/ctst/steps/pra.ts @@ -298,6 +298,9 @@ When('the DATA_ACCESSOR user tries to perform PutObject on {string} site', { tim } catch (err) { this.logger.error('Failed to setup entity', { err }); } + if (!conditionOk) { + await Utils.sleep(1000); + } } await putObject(this); From 3b60314521e5dd6ebba4c89f17647d855dd926ef Mon Sep 17 00:00:00 2001 From: KillianG Date: Fri, 30 Aug 2024 12:08:39 +0200 Subject: [PATCH 07/22] Use cache for pra install count Issue: ZENKO-4775 --- tests/ctst/steps/pra.ts | 12 ++++++++---- tests/ctst/world/Zenko.ts | 6 ++++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/ctst/steps/pra.ts b/tests/ctst/steps/pra.ts index c0f61fc3c3..f8cc5a1725 100644 --- a/tests/ctst/steps/pra.ts +++ b/tests/ctst/steps/pra.ts @@ -13,7 +13,7 @@ import { restoreObject, verifyObjectLocation, } from 'steps/utils/utils'; -import { Constants, Identity, IdentityEnum, SuperAdmin, Utils } from 'cli-testing'; +import { CacheHelper, Constants, Identity, IdentityEnum, SuperAdmin, Utils } from 'cli-testing'; import { safeJsonParse } from 'common/utils'; import assert from 'assert'; import { EntityType } from 'world/Zenko'; @@ -73,7 +73,7 @@ async function installPRA(world: Zenko, sinkS3Endpoint = 'http://s3.zenko.local' // prometheusHostname: 'prom.dr.zenko.local', // could be any name, cert will be auto-generated prometheusExternalIpsDiscovery: true, prometheusDisableTls: true, - forceRotateServiceCredentials: world.praInstallCount > 0, + forceRotateServiceCredentials: (CacheHelper.savedAcrossTests[Zenko.PRA_INSTALL_COUNT_KEY] as number) > 0, ...kafkaExternalIpOption, timeout, }); @@ -173,8 +173,12 @@ Given('a DR installed', { timeout: installTimeout + 2000 }, async function (this accessKey: Buffer.from(credentials.accessKeyId).toString('base64'), secretAccessKey: Buffer.from(credentials.secretAccessKey).toString('base64'), }); - await installPRA(this, undefined, `${installTimeout.toString()}ms`); - this.praInstallCount += 1; + + // Timeout is set to 1 second less than the cucumber + // timeout to see the command timeout instead of the step timeout + + await installPRA(this, undefined, `${(installTimeout - 1000).toString()}ms`); + (CacheHelper.savedAcrossTests[Zenko.PRA_INSTALL_COUNT_KEY] as number) += 1; return; }); diff --git a/tests/ctst/world/Zenko.ts b/tests/ctst/world/Zenko.ts index 7e097ea50b..bb3c44de3b 100644 --- a/tests/ctst/world/Zenko.ts +++ b/tests/ctst/world/Zenko.ts @@ -115,8 +115,7 @@ export default class Zenko extends World { static readonly PRIMARY_SITE_NAME = 'admin'; static readonly SECONDARY_SITE_NAME = 'dradmin'; - - public praInstallCount = 0; + static readonly PRA_INSTALL_COUNT_KEY = 'praInstallCount'; /** * @constructor @@ -144,6 +143,9 @@ export default class Zenko extends World { ...this.parameters, }); + CacheHelper.savedAcrossTests[Zenko.PRA_INSTALL_COUNT_KEY] = 0; + + if (this.parameters.AccountName && !Identity.hasIdentity(IdentityEnum.ACCOUNT, this.parameters.AccountName)) { Identity.addIdentity(IdentityEnum.ACCOUNT, this.parameters.AccountName, { accessKeyId: this.parameters.AccountAccessKey, From fa48e9cb69313fcb0547663cbdd319ba21a87525 Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Thu, 26 Sep 2024 12:11:49 +0200 Subject: [PATCH 08/22] Store zenko CRs in artifacts Issue: ZENKO-4893 --- .github/actions/archive-artifacts/action.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/actions/archive-artifacts/action.yaml b/.github/actions/archive-artifacts/action.yaml index 47aad4a4fc..b9ce2cb6b2 100644 --- a/.github/actions/archive-artifacts/action.yaml +++ b/.github/actions/archive-artifacts/action.yaml @@ -1,6 +1,6 @@ --- -name: "Archive artifact logs and data" -description: "Archive logs, pods and events" +name: Archive artifact logs and data +description: Archive logs, pods and events runs: using: composite @@ -12,6 +12,9 @@ runs: mkdir -p /tmp/artifacts/data/${STAGE}/kind-logs; kubectl get pods -A -o yaml > /tmp/artifacts/data/${STAGE}/kind-logs/all-pods.log; kubectl get events -A -o yaml > /tmp/artifacts/data/${STAGE}/kind-logs/all-events.log; + kubectl get zenko -A -o yaml > /tmp/artifacts/data/${STAGE}/kind-logs/all-zenkos.log; + kubectl get zenkodrsource -A -o yaml > /tmp/artifacts/data/${STAGE}/kind-logs/all-zenkodrsources.log; + kubectl get zenkodrsink -A -o yaml > /tmp/artifacts/data/${STAGE}/kind-logs/all-zenkodrsinks.log; kind export logs /tmp/artifacts/data/${STAGE}/kind-logs/kind-export; tar zcvf /tmp/artifacts/${{ github.sha }}-${STAGE}-logs-volumes.tgz /tmp/artifacts/data/${STAGE}/kind-logs; - name: Dump kafka From 89bcc9f6a53942fb93aa2e6310a612f6de21f143 Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Thu, 26 Sep 2024 12:16:53 +0200 Subject: [PATCH 09/22] Do not merge artifacts Issue: ZENKO-4893 --- .github/workflows/end2end.yaml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/end2end.yaml b/.github/workflows/end2end.yaml index d1053a659b..fd99ad5696 100644 --- a/.github/workflows/end2end.yaml +++ b/.github/workflows/end2end.yaml @@ -430,7 +430,7 @@ jobs: - name: Archive artifact logs and data uses: ./.github/actions/archive-artifacts env: - STAGE: end2end-http + STAGE: ${{ github.job }}.${{ github.run_attempt }} if: always() - name: Clean Up run: kind delete cluster @@ -495,7 +495,7 @@ jobs: timeout-minutes: 60 if: failure() && runner.debug == '1' - name: Upload results - if: "!cancelled() && env.TRUNK_TOKEN" + if: env.TRUNK_TOKEN && !cancelled() uses: trunk-io/analytics-uploader@main with: junit-paths: /artifacts/data/reports/*.xml @@ -507,7 +507,7 @@ jobs: - name: Archive artifact logs and data uses: ./.github/actions/archive-artifacts env: - STAGE: end2end-pra + STAGE: ${{ github.job }}.${{ github.run_attempt }} if: always() - name: Clean Up run: kind delete cluster @@ -554,8 +554,12 @@ jobs: - name: Run smoke tests run: bash run-e2e-test.sh "end2end" ${E2E_IMAGE_NAME}:${E2E_IMAGE_TAG} "smoke" "default" working-directory: ./.github/scripts/end2end + - name: Debug wait + uses: ./.github/actions/debug-wait + timeout-minutes: 60 + if: failure() && runner.debug == '1' - name: Upload results - if: "!cancelled() && env.TRUNK_TOKEN" + if: env.TRUNK_TOKEN && !cancelled() uses: trunk-io/analytics-uploader@main with: junit-paths: /artifacts/data/reports/*.xml @@ -569,14 +573,10 @@ jobs: # - name: Run vault e2e tests # run: bash vault-e2e-test.sh # working-directory: ./.github/scripts/end2end - - name: Debug wait - uses: ./.github/actions/debug-wait - timeout-minutes: 60 - if: failure() && runner.debug == '1' - name: Archive artifact logs and data uses: ./.github/actions/archive-artifacts env: - STAGE: end2end-https + STAGE: ${{ github.job }}.${{ github.run_attempt }} if: always() - name: Clean Up run: kind delete cluster @@ -617,7 +617,7 @@ jobs: run: bash run-e2e-test.sh "end2end" ${E2E_IMAGE_NAME}:${E2E_IMAGE_TAG} "backbeat" "default" working-directory: ./.github/scripts/end2end - name: Upload results - if: "!cancelled() && env.TRUNK_TOKEN" + if: env.TRUNK_TOKEN && !cancelled() uses: trunk-io/analytics-uploader@main with: junit-paths: /artifacts/data/reports/*.xml @@ -633,7 +633,7 @@ jobs: - name: Archive artifact logs and data uses: ./.github/actions/archive-artifacts env: - STAGE: end2end-sharded + STAGE: ${{ github.job }}.${{ github.run_attempt }} if: always() - name: Clean Up run: kind delete cluster @@ -681,7 +681,7 @@ jobs: run: bash run-e2e-ctst.sh "" "" "" "" --tags 'not @PRA' working-directory: ./.github/scripts/end2end - name: Upload results - if: "!cancelled() && env.TRUNK_TOKEN" + if: env.TRUNK_TOKEN && !cancelled() uses: trunk-io/analytics-uploader@main with: junit-paths: /artifacts/data/reports/*.xml @@ -697,7 +697,7 @@ jobs: - name: Archive artifact logs and data uses: ./.github/actions/archive-artifacts env: - STAGE: ctst-end2end-sharded + STAGE: ${{ github.job }}.${{ github.run_attempt }} if: always() - name: Clean Up run: kind delete cluster @@ -727,9 +727,9 @@ jobs: - ctst-end2end-sharded steps: - name: Upload final status - if: always() uses: scality/actions/upload_final_status@1.8.0 with: ARTIFACTS_USER: ${{ secrets.ARTIFACTS_USER }} ARTIFACTS_PASSWORD: ${{ secrets.ARTIFACTS_PASSWORD }} JOBS_RESULTS: ${{ join(needs.*.result) }} + if: always() From 058548f04d8016565f90ff32a0d26795db5a0431 Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Fri, 27 Sep 2024 10:00:45 +0200 Subject: [PATCH 10/22] Display junit results Issue: ZENKO-4899 --- .github/actions/archive-artifacts/action.yaml | 106 +++++++++++--- .github/workflows/end2end.yaml | 135 ++++-------------- 2 files changed, 115 insertions(+), 126 deletions(-) diff --git a/.github/actions/archive-artifacts/action.yaml b/.github/actions/archive-artifacts/action.yaml index b9ce2cb6b2..6d02f751da 100644 --- a/.github/actions/archive-artifacts/action.yaml +++ b/.github/actions/archive-artifacts/action.yaml @@ -2,29 +2,78 @@ name: Archive artifact logs and data description: Archive logs, pods and events +inputs: + user: + description: Artifacts user + required: true + password: + description: Artifacts password + required: true + junit-paths: + description: Path to junit reports + default: /artifacts/data/reports/*.xml + required: true + stage: + description: Stage name + default: ${{ github.job }}.${{ github.run_attempt }} + required: true + trunk_token: + description: Trunk token + required: false + zenko-name: + description: Zenko name + default: end2end + required: false + zenko-namespace: + description: Namespace + default: default + required: false + runs: using: composite steps: + - name: Publish test report + uses: mikepenz/action-junit-report@v4 + with: + annotate_only: true + check_name: ${{ inputs.stage}} + report_paths: ${{ inputs.junit-paths }} + job_summary: ${{ job.status != 'success' }} # Only show the summary if the job failed + detailed_summary: true + continue-on-error: true + + - name: Upload results + if: inputs.trunk_token && job.status != 'cancelled' + uses: trunk-io/analytics-uploader@v1.4.0 + with: + junit-paths: ${{ inputs.junit-paths }} + org-slug: ${{ github.repository_owner }} + token: ${{ inputs.trunk_token }} + continue-on-error: true + - name: Archive artifact logs and data shell: bash run: |- - set -exu; - mkdir -p /tmp/artifacts/data/${STAGE}/kind-logs; - kubectl get pods -A -o yaml > /tmp/artifacts/data/${STAGE}/kind-logs/all-pods.log; - kubectl get events -A -o yaml > /tmp/artifacts/data/${STAGE}/kind-logs/all-events.log; - kubectl get zenko -A -o yaml > /tmp/artifacts/data/${STAGE}/kind-logs/all-zenkos.log; - kubectl get zenkodrsource -A -o yaml > /tmp/artifacts/data/${STAGE}/kind-logs/all-zenkodrsources.log; - kubectl get zenkodrsink -A -o yaml > /tmp/artifacts/data/${STAGE}/kind-logs/all-zenkodrsinks.log; - kind export logs /tmp/artifacts/data/${STAGE}/kind-logs/kind-export; - tar zcvf /tmp/artifacts/${{ github.sha }}-${STAGE}-logs-volumes.tgz /tmp/artifacts/data/${STAGE}/kind-logs; + set -exu + + mkdir -p /tmp/artifacts/data/${STAGE}/kind-logs + kubectl get pods -A -o yaml > /tmp/artifacts/data/${STAGE}/kind-logs/all-pods.log + kubectl get events -A -o yaml > /tmp/artifacts/data/${STAGE}/kind-logs/all-events.log + kubectl get zenko -A -o yaml > /tmp/artifacts/data/${STAGE}/kind-logs/all-zenkos.log + kubectl get zenkodrsource -A -o yaml > /tmp/artifacts/data/${STAGE}/kind-logs/all-zenkodrsources.log + kubectl get zenkodrsink -A -o yaml > /tmp/artifacts/data/${STAGE}/kind-logs/all-zenkodrsinks.log + kind export logs /tmp/artifacts/data/${STAGE}/kind-logs/kind-export + tar zcvf /tmp/artifacts/${{ github.sha }}-${STAGE}-logs-volumes.tgz /tmp/artifacts/data/${STAGE}/kind-logs + env: + STAGE: ${{ inputs.stage }} + continue-on-error: true + - name: Dump kafka shell: bash - continue-on-error: true run: |- set -exu - NAMESPACE=${NAMESPACE:-default} - KAFKA=$(kubectl get pods -n ${NAMESPACE} -lkafka_cr=${ZENKO_NAME:-end2end}-base-queue -o jsonpath='{.items[0].metadata.name}') + KAFKA=$(kubectl get pods -n ${NAMESPACE} -lkafka_cr=${ZENKO_NAME}-base-queue -o jsonpath='{.items[0].metadata.name}') kubectl exec -in ${NAMESPACE} ${KAFKA} -c kafka -- \ env KAFKA_OPTS= kafka-topics.sh --bootstrap-server :9092 --list \ @@ -38,16 +87,21 @@ runs: env KAFKA_OPTS= kafka-consumer-groups.sh --bootstrap-server :9092 --describe --all-groups \ > /tmp/artifacts/data/${STAGE}/kafka-offsets.log - KAFKA_SERVICE=$(kubectl get services -n ${NAMESPACE} -lkafka_cr=${ZENKO_NAME:-end2end}-base-queue -o jsonpath='{.items[0].metadata.name}') + KAFKA_SERVICE=$(kubectl get services -n ${NAMESPACE} -lkafka_cr=${ZENKO_NAME}-base-queue -o jsonpath='{.items[0].metadata.name}') kubectl run -n ${NAMESPACE} kcat --image=edenhill/kcat:1.7.1 --restart=Never --command -- sleep 300 kubectl wait -n ${NAMESPACE} pod kcat --for=condition=ready cat /tmp/artifacts/data/${STAGE}/kafka-topics.log | grep -v '^__' | xargs -P 15 -I {} \ sh -c "kubectl exec -i -n ${NAMESPACE} kcat -- \ kcat -L -b ${KAFKA_SERVICE} -t {} -C -o beginning -e -q -J \ > /tmp/artifacts/data/${STAGE}/kafka-messages-{}.log" + env: + STAGE: ${{ inputs.stage }} + NAMESPACE: ${{ inputs.zenko-namespace }} + ZENKO_NAME: ${{ inputs.zenko-name }} + continue-on-error: true + - name: Dump MongoDB shell: bash - continue-on-error: true run: |- set -exu @@ -57,9 +111,29 @@ runs: NAMESPACE="${NAMESPACE:-default}" DUMP_DIR="/tmp/mongodb.dump" - kubectl exec -n ${NAMESPACE} data-db-mongodb-sharded-mongos-0 -- mongodump --db ${ZENKO_MONGODB_DATABASE} -u ${MONGODB_ROOT_USERNAME} -p ${MONGODB_ROOT_PASSWORD} --authenticationDatabase admin --out ${DUMP_DIR} + kubectl exec -n ${NAMESPACE} data-db-mongodb-sharded-mongos-0 -- mongodump \ + --db ${ZENKO_MONGODB_DATABASE} -u ${MONGODB_ROOT_USERNAME} -p ${MONGODB_ROOT_PASSWORD} \ + --authenticationDatabase admin --out ${DUMP_DIR} - kubectl exec -n ${NAMESPACE} data-db-mongodb-sharded-mongos-0 -- bash -c "for bson_file in ${DUMP_DIR}/${ZENKO_MONGODB_DATABASE}/*.bson; do json_file=\"${DUMP_DIR}/\$(basename \${bson_file} .bson).json\"; bsondump --outFile \${json_file} \${bson_file}; done" + kubectl exec -n ${NAMESPACE} data-db-mongodb-sharded-mongos-0 -- bash -c \ + "for bson_file in ${DUMP_DIR}/${ZENKO_MONGODB_DATABASE}/*.bson; do \ + json_file=\"${DUMP_DIR}/\$(basename \${bson_file} .bson).json\"; \ + bsondump --outFile \${json_file} \${bson_file}; \ + done" mkdir -p /tmp/artifacts/data/${STAGE}/mongodb-dump kubectl cp ${NAMESPACE}/data-db-mongodb-sharded-mongos-0:${DUMP_DIR} /tmp/artifacts/data/${STAGE}/mongodb-dump + env: + STAGE: ${{ inputs.stage }}.${{ github.run_attempt }} + NAMESPACE: ${{ inputs.zenko-namespace }} + ZENKO_NAME: ${{ inputs.zenko-name }} + continue-on-error: true + + - name: Upload artifacts # move into `archive-artifacts` action + uses: scality/action-artifacts@v4 + with: + method: upload + url: https://artifacts.scality.net + user: ${{ inputs.user }} + password: ${{ inputs.password }} + source: /tmp/artifacts diff --git a/.github/workflows/end2end.yaml b/.github/workflows/end2end.yaml index fd99ad5696..9a904eab63 100644 --- a/.github/workflows/end2end.yaml +++ b/.github/workflows/end2end.yaml @@ -417,32 +417,15 @@ jobs: uses: ./.github/actions/debug-wait timeout-minutes: 60 if: failure() && runner.debug == '1' - - name: Upload results - if: "!cancelled() && env.TRUNK_TOKEN" - uses: trunk-io/analytics-uploader@main - with: - junit-paths: /artifacts/data/reports/*.xml - org-slug: ${{ github.repository_owner }} - token: ${{ env.TRUNK_TOKEN }} - env: - TRUNK_TOKEN: ${{ secrets.TRUNK_TOKEN }} - continue-on-error: true - - name: Archive artifact logs and data + - name: Archive and publish artifacts uses: ./.github/actions/archive-artifacts - env: - STAGE: ${{ github.job }}.${{ github.run_attempt }} - if: always() - - name: Clean Up - run: kind delete cluster - - name: Upload artifacts - uses: scality/action-artifacts@v4 with: - method: upload - url: https://artifacts.scality.net user: ${{ secrets.ARTIFACTS_USER }} password: ${{ secrets.ARTIFACTS_PASSWORD }} - source: /tmp/artifacts + trunk_token: ${{ secrets.TRUNK_TOKEN }} if: always() + - name: Clean Up + run: kind delete cluster end2end-pra: needs: [build-kafka, check-dashboard-versions, lint-and-build-ctst] @@ -494,32 +477,15 @@ jobs: uses: ./.github/actions/debug-wait timeout-minutes: 60 if: failure() && runner.debug == '1' - - name: Upload results - if: env.TRUNK_TOKEN && !cancelled() - uses: trunk-io/analytics-uploader@main - with: - junit-paths: /artifacts/data/reports/*.xml - org-slug: ${{ github.repository_owner }} - token: ${{ env.TRUNK_TOKEN }} - env: - TRUNK_TOKEN: ${{ secrets.TRUNK_TOKEN }} - continue-on-error: true - - name: Archive artifact logs and data + - name: Archive and publish artifacts uses: ./.github/actions/archive-artifacts - env: - STAGE: ${{ github.job }}.${{ github.run_attempt }} - if: always() - - name: Clean Up - run: kind delete cluster - - name: Upload artifacts - uses: scality/action-artifacts@v4 with: - method: upload - url: https://artifacts.scality.net user: ${{ secrets.ARTIFACTS_USER }} password: ${{ secrets.ARTIFACTS_PASSWORD }} - source: /tmp/artifacts + trunk_token: ${{ secrets.TRUNK_TOKEN }} if: always() + - name: Clean Up + run: kind delete cluster end2end-https: needs: [build-kafka, build-test-image, check-dashboard-versions] @@ -554,41 +520,24 @@ jobs: - name: Run smoke tests run: bash run-e2e-test.sh "end2end" ${E2E_IMAGE_NAME}:${E2E_IMAGE_TAG} "smoke" "default" working-directory: ./.github/scripts/end2end - - name: Debug wait - uses: ./.github/actions/debug-wait - timeout-minutes: 60 - if: failure() && runner.debug == '1' - - name: Upload results - if: env.TRUNK_TOKEN && !cancelled() - uses: trunk-io/analytics-uploader@main - with: - junit-paths: /artifacts/data/reports/*.xml - org-slug: ${{ github.repository_owner }} - token: ${{ env.TRUNK_TOKEN }} - env: - TRUNK_TOKEN: ${{ secrets.TRUNK_TOKEN }} - continue-on-error: true # Temporarily disabled as CTST will test the same APIs more # extensively. # - name: Run vault e2e tests # run: bash vault-e2e-test.sh # working-directory: ./.github/scripts/end2end - - name: Archive artifact logs and data + - name: Debug wait + uses: ./.github/actions/debug-wait + timeout-minutes: 60 + if: failure() && runner.debug == '1' + - name: Archive and publish artifacts uses: ./.github/actions/archive-artifacts - env: - STAGE: ${{ github.job }}.${{ github.run_attempt }} - if: always() - - name: Clean Up - run: kind delete cluster - - name: Upload artifacts - uses: scality/action-artifacts@v4 with: - method: upload - url: https://artifacts.scality.net user: ${{ secrets.ARTIFACTS_USER }} password: ${{ secrets.ARTIFACTS_PASSWORD }} - source: /tmp/artifacts + trunk_token: ${{ secrets.TRUNK_TOKEN }} if: always() + - name: Clean Up + run: kind delete cluster end2end-sharded: needs: [build-kafka, build-test-image, check-dashboard-versions] @@ -616,36 +565,19 @@ jobs: - name: Run backbeat end to end tests run: bash run-e2e-test.sh "end2end" ${E2E_IMAGE_NAME}:${E2E_IMAGE_TAG} "backbeat" "default" working-directory: ./.github/scripts/end2end - - name: Upload results - if: env.TRUNK_TOKEN && !cancelled() - uses: trunk-io/analytics-uploader@main - with: - junit-paths: /artifacts/data/reports/*.xml - org-slug: ${{ github.repository_owner }} - token: ${{ env.TRUNK_TOKEN }} - env: - TRUNK_TOKEN: ${{ secrets.TRUNK_TOKEN }} - continue-on-error: true - name: Debug wait uses: ./.github/actions/debug-wait timeout-minutes: 60 if: failure() && runner.debug == '1' - - name: Archive artifact logs and data + - name: Archive and publish artifacts uses: ./.github/actions/archive-artifacts - env: - STAGE: ${{ github.job }}.${{ github.run_attempt }} - if: always() - - name: Clean Up - run: kind delete cluster - - name: Upload artifacts - uses: scality/action-artifacts@v4 with: - method: upload - url: https://artifacts.scality.net user: ${{ secrets.ARTIFACTS_USER }} password: ${{ secrets.ARTIFACTS_PASSWORD }} - source: /tmp/artifacts + trunk_token: ${{ secrets.TRUNK_TOKEN }} if: always() + - name: Clean Up + run: kind delete cluster ctst-end2end-sharded: needs: [build-kafka, lint-and-build-ctst, check-dashboard-versions] @@ -680,36 +612,19 @@ jobs: - name: Run CTST end to end tests run: bash run-e2e-ctst.sh "" "" "" "" --tags 'not @PRA' working-directory: ./.github/scripts/end2end - - name: Upload results - if: env.TRUNK_TOKEN && !cancelled() - uses: trunk-io/analytics-uploader@main - with: - junit-paths: /artifacts/data/reports/*.xml - org-slug: ${{ github.repository_owner }} - token: ${{ env.TRUNK_TOKEN }} - env: - TRUNK_TOKEN: ${{ secrets.TRUNK_TOKEN }} - continue-on-error: true - name: Debug wait uses: ./.github/actions/debug-wait timeout-minutes: 60 if: failure() && runner.debug == '1' - - name: Archive artifact logs and data + - name: Archive and publish artifacts uses: ./.github/actions/archive-artifacts - env: - STAGE: ${{ github.job }}.${{ github.run_attempt }} - if: always() - - name: Clean Up - run: kind delete cluster - - name: Upload artifacts - uses: scality/action-artifacts@v4 with: - method: upload - url: https://artifacts.scality.net user: ${{ secrets.ARTIFACTS_USER }} password: ${{ secrets.ARTIFACTS_PASSWORD }} - source: /tmp/artifacts + trunk_token: ${{ secrets.TRUNK_TOKEN }} if: always() + - name: Clean Up + run: kind delete cluster write-final-status: runs-on: ubuntu-latest @@ -727,7 +642,7 @@ jobs: - ctst-end2end-sharded steps: - name: Upload final status - uses: scality/actions/upload_final_status@1.8.0 + uses: scality/actions/upload_final_status@1.9.0 with: ARTIFACTS_USER: ${{ secrets.ARTIFACTS_USER }} ARTIFACTS_PASSWORD: ${{ secrets.ARTIFACTS_PASSWORD }} From 705ae3af4dd6c785411e02119a0a4b920fc82d0d Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Tue, 22 Oct 2024 16:29:50 +0200 Subject: [PATCH 11/22] Bump zenko-drctl 1.0.6 Issue: ZENKO-4899 --- solution/deps.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solution/deps.yaml b/solution/deps.yaml index 368dcf3832..effb3a104b 100644 --- a/solution/deps.yaml +++ b/solution/deps.yaml @@ -21,7 +21,7 @@ cloudserver: drctl: sourceRegistry: ghcr.io/scality image: zenko-drctl - tag: v1.0.4 + tag: v1.0.6 envsubst: DRCTL_TAG toolName: zenko-drctl fubectl: From 30237059eeaf9819d08edc0644bfa9ef8395d1c6 Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Tue, 22 Oct 2024 16:30:45 +0200 Subject: [PATCH 12/22] Bump zenko-operator 1.6.3 Issue: ZENKO-4899 --- solution/deps.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solution/deps.yaml b/solution/deps.yaml index effb3a104b..5427f2d1df 100644 --- a/solution/deps.yaml +++ b/solution/deps.yaml @@ -136,7 +136,7 @@ vault: zenko-operator: sourceRegistry: ghcr.io/scality image: zenko-operator - tag: 1.6.1 + tag: v1.6.3 envsubst: ZENKO_OPERATOR_TAG zenko-ui: sourceRegistry: ghcr.io/scality From 0405e61f03c6d8787e4e78a70c7e806527809ef5 Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Mon, 21 Oct 2024 16:20:51 +0200 Subject: [PATCH 13/22] Bump kind/k8s 1.24.17 Issue: ZenKO-4899 --- .github/scripts/end2end/bootstrap-kind.sh | 2 +- .github/scripts/end2end/install-kind-dependencies.sh | 4 ++-- .github/scripts/end2end/requirements.sh | 2 +- .github/workflows/end2end.yaml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/scripts/end2end/bootstrap-kind.sh b/.github/scripts/end2end/bootstrap-kind.sh index c075ef4e83..601404db27 100755 --- a/.github/scripts/end2end/bootstrap-kind.sh +++ b/.github/scripts/end2end/bootstrap-kind.sh @@ -2,7 +2,7 @@ set -exu -NODE_IMAGE=${1:-kindest/node:v1.23.4@sha256:0e34f0d0fd448aa2f2819cfd74e99fe5793a6e4938b328f657c8e3f81ee0dfb9} +NODE_IMAGE=${1:-kindest/node:v1.24.17@sha256:bad10f9b98d54586cba05a7eaa1b61c6b90bfc4ee174fdc43a7b75ca75c95e51} VOLUME_ROOT=${2:-/artifacts} WORKER_NODE_COUNT=${3:-0} CLUSTER_NAME=${CLUSTER_NAME:-kind} diff --git a/.github/scripts/end2end/install-kind-dependencies.sh b/.github/scripts/end2end/install-kind-dependencies.sh index 23470e45a2..461fef2352 100755 --- a/.github/scripts/end2end/install-kind-dependencies.sh +++ b/.github/scripts/end2end/install-kind-dependencies.sh @@ -11,9 +11,9 @@ VERSION_FILE="${REPOSITORY_DIR}/VERSION" source "${VERSION_FILE}" ZK_OPERATOR_VERSION=0.2.13 -CERT_MANAGER_VERSION=v1.6.1 +CERT_MANAGER_VERSION=v1.11.0 KAFKA_OPERATOR_VERSION=0.20.1 -INGRESS_NGINX_VERSION=controller-v1.1.0 +INGRESS_NGINX_VERSION=controller-v1.3.1 PROMETHEUS_VERSION=v0.52.1 KEYCLOAK_VERSION=18.4.4 diff --git a/.github/scripts/end2end/requirements.sh b/.github/scripts/end2end/requirements.sh index 71e693448c..ad66f95e39 100755 --- a/.github/scripts/end2end/requirements.sh +++ b/.github/scripts/end2end/requirements.sh @@ -1,7 +1,7 @@ #!/bin/bash KUBECTL_VERSION=1.21.1 -KIND_VERSION=v0.12.0 +KIND_VERSION=v0.24.0 HELM_VERSION=v3.5.3 KUSTOMIZE_VERSION=v4.4.1 YQ_VERSION=v4.27.5 diff --git a/.github/workflows/end2end.yaml b/.github/workflows/end2end.yaml index 9a904eab63..5dcb1ddcb6 100644 --- a/.github/workflows/end2end.yaml +++ b/.github/workflows/end2end.yaml @@ -16,7 +16,7 @@ env: WORKER_COUNT: '2' OPERATOR_REPO: git@github.com:scality/zenko-operator.git OPERATOR_IMAGE: "" - KIND_NODE_IMAGE: "kindest/node:v1.23.4@sha256:0e34f0d0fd448aa2f2819cfd74e99fe5793a6e4938b328f657c8e3f81ee0dfb9" + KIND_NODE_IMAGE: "kindest/node:v1.24.17@sha256:bad10f9b98d54586cba05a7eaa1b61c6b90bfc4ee174fdc43a7b75ca75c95e51" VOLUME_ROOT: /artifacts OIDC_REALM: "zenko" OIDC_CLIENT_ID: "zenko-ui" @@ -92,7 +92,7 @@ env: SKOPEO_VERSION: "v1.16.1" KUBECTL_VERSION: "1.31.0" TILT_VERSION: "0.33.19" - KIND_VERSION: "v0.12.0" + KIND_VERSION: "v0.24.0" ZENKO_ENABLE_SOSAPI: false TIME_PROGRESSION_FACTOR: 1 EXPIRE_ONE_DAY_EARLIER: true From 8caab9267890383ff43128a71dc63cd11fcb5134 Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Tue, 22 Oct 2024 12:30:34 +0200 Subject: [PATCH 14/22] Bump lag exporter 0.7.3 Issue: ZENKO-4899 --- solution/deps.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solution/deps.yaml b/solution/deps.yaml index 5427f2d1df..fdb165f202 100644 --- a/solution/deps.yaml +++ b/solution/deps.yaml @@ -72,7 +72,7 @@ kafka-cruise-control: kafka-lag-exporter: sourceRegistry: seglo image: kafka-lag-exporter - tag: 0.7.1 + tag: 0.7.3 envsubst: KAFKA_LAGEXPORTER_TAG mongodb-connector: tag: 1.13.0 From 63a34e76c3a73f10bc301c5d9c96d7055da1917f Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Tue, 22 Oct 2024 12:31:23 +0200 Subject: [PATCH 15/22] Bump s3utils 1.14.14 Issue: ZENKO-4899 --- solution/deps.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solution/deps.yaml b/solution/deps.yaml index fdb165f202..2a0afa01f3 100644 --- a/solution/deps.yaml +++ b/solution/deps.yaml @@ -100,7 +100,7 @@ s3utils: sourceRegistry: ghcr.io/scality dashboard: s3utils/s3utils-dashboards image: s3utils - tag: 1.14.13 + tag: 1.14.14 envsubst: S3UTILS_TAG scuba: sourceRegistry: ghcr.io/scality From 0ba60aa4a939d29b0f7006a305e596fc994b61bf Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Tue, 22 Oct 2024 12:31:42 +0200 Subject: [PATCH 16/22] Bump cloudserver 8.8.35 Issue: ZENKO-4899 --- solution/deps.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solution/deps.yaml b/solution/deps.yaml index 2a0afa01f3..1f3601a398 100644 --- a/solution/deps.yaml +++ b/solution/deps.yaml @@ -16,7 +16,7 @@ cloudserver: sourceRegistry: ghcr.io/scality dashboard: cloudserver/cloudserver-dashboards image: cloudserver - tag: 8.8.32 + tag: 8.8.35 envsubst: CLOUDSERVER_TAG drctl: sourceRegistry: ghcr.io/scality From 4d8229073f9de316a6317d698bcce113a4c46e0a Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Tue, 22 Oct 2024 12:32:30 +0200 Subject: [PATCH 17/22] Bump backbeat 8.6.49 Issue: ZENKO-4899 --- solution/deps.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solution/deps.yaml b/solution/deps.yaml index 1f3601a398..a61b586b65 100644 --- a/solution/deps.yaml +++ b/solution/deps.yaml @@ -6,7 +6,7 @@ backbeat: dashboard: backbeat/backbeat-dashboards image: backbeat policy: backbeat/backbeat-policies - tag: 8.6.48 + tag: 8.6.49 envsubst: BACKBEAT_TAG busybox: image: busybox From aec173806bffd3b5d6baa0f3c8de20a6285b91bb Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Tue, 22 Oct 2024 12:33:00 +0200 Subject: [PATCH 18/22] Bump pensieve-api 1.6.2 Issue: ZENKO-4899 --- solution/deps.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solution/deps.yaml b/solution/deps.yaml index a61b586b65..8605d3ef8b 100644 --- a/solution/deps.yaml +++ b/solution/deps.yaml @@ -80,7 +80,7 @@ mongodb-connector: pensieve-api: sourceRegistry: ghcr.io/scality image: pensieve-api - tag: 1.6.0 + tag: 1.6.2 envsubst: PENSIEVE_API_TAG rclone: sourceRegistry: rclone From 5cceac067dde8334a1419ebbb2e97bb72dafc4aa Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Tue, 22 Oct 2024 12:48:42 +0200 Subject: [PATCH 19/22] Bump scuba 1.0.7 Issue: ZENKO-4899 --- solution/deps.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solution/deps.yaml b/solution/deps.yaml index 8605d3ef8b..6be00d602a 100644 --- a/solution/deps.yaml +++ b/solution/deps.yaml @@ -106,7 +106,7 @@ scuba: sourceRegistry: ghcr.io/scality dashboard: scuba/scuba-dashboards image: scuba - tag: 1.0.2 + tag: 1.0.7 envsubst: SCUBA_TAG sorbet: sourceRegistry: ghcr.io/scality From 925e86dbb41f8e279052c10415b1c2013ade41ad Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Tue, 22 Oct 2024 16:32:25 +0200 Subject: [PATCH 20/22] Release zenko 2.6.68 Issue: ZENKO-4899 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 4d4e009ed5..170d498888 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ -VERSION="2.6.67" +VERSION="2.6.68" VERSION_SUFFIX= From 219329073f5a32d818a3f924603c2f37a6f08ce9 Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Wed, 23 Oct 2024 16:45:41 +0200 Subject: [PATCH 21/22] Bump kind/k8s 1.25.16 Issue: ZENKO-4899 --- .github/scripts/end2end/bootstrap-kind.sh | 2 +- .../end2end/install-kind-dependencies.sh | 22 +++++++++++++++++-- .github/workflows/end2end.yaml | 2 +- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/.github/scripts/end2end/bootstrap-kind.sh b/.github/scripts/end2end/bootstrap-kind.sh index 601404db27..7f4523add2 100755 --- a/.github/scripts/end2end/bootstrap-kind.sh +++ b/.github/scripts/end2end/bootstrap-kind.sh @@ -2,7 +2,7 @@ set -exu -NODE_IMAGE=${1:-kindest/node:v1.24.17@sha256:bad10f9b98d54586cba05a7eaa1b61c6b90bfc4ee174fdc43a7b75ca75c95e51} +NODE_IMAGE=${1:-kindest/node:v1.25.16@sha256:6110314339b3b44d10da7d27881849a87e092124afab5956f2e10ecdb463b025} VOLUME_ROOT=${2:-/artifacts} WORKER_NODE_COUNT=${3:-0} CLUSTER_NAME=${CLUSTER_NAME:-kind} diff --git a/.github/scripts/end2end/install-kind-dependencies.sh b/.github/scripts/end2end/install-kind-dependencies.sh index e6b0c2fc9a..a48d9068bb 100755 --- a/.github/scripts/end2end/install-kind-dependencies.sh +++ b/.github/scripts/end2end/install-kind-dependencies.sh @@ -10,9 +10,9 @@ VERSION_FILE="${REPOSITORY_DIR}/VERSION" source "${VERSION_FILE}" -ZK_OPERATOR_VERSION=0.2.13 +ZK_OPERATOR_VERSION=0.2.15 CERT_MANAGER_VERSION=v1.11.0 -KAFKA_OPERATOR_VERSION=0.23.0 +KAFKA_OPERATOR_VERSION=0.24.1 INGRESS_NGINX_VERSION=controller-v1.3.1 PROMETHEUS_VERSION=v0.52.1 KEYCLOAK_VERSION=18.4.4 @@ -128,6 +128,24 @@ build_solution_base_manifests() { sed -i "s/MONGODB_SHARDSERVER_RAM_LIMIT/${MONGODB_SHARDSERVER_RAM_LIMIT}/g" $DIR/_build/root/deploy/* sed -i "s/MONGODB_SHARDSERVER_RAM_REQUEST/${MONGODB_SHARDSERVER_RAM_REQUEST}/g" $DIR/_build/root/deploy/* sed -i "s/MONGODB_MONGOS_RAM_REQUEST/${MONGODB_MONGOS_RAM_REQUEST}/g" $DIR/_build/root/deploy/* + + # Limits and requests for MongoDB are computed based on the current system + # Detect total system RAM in GiB + TOTAL_RAM_GB=$(awk '/MemTotal/ {printf "%.0f", $2/1024/1024}' /proc/meminfo) + + # Compute MongoDB settings based on the total RAM + MONGODB_WIRETIGER_CACHE_SIZE_GB=$((TOTAL_RAM_GB * 335 / 1000)) + MONGODB_MONGOS_RAM_LIMIT=$((TOTAL_RAM_GB * 165 / 1000))Gi + MONGODB_SHARDSERVER_RAM_LIMIT=$((2 * MONGODB_WIRETIGER_CACHE_SIZE_GB))Gi + MONGODB_SHARDSERVER_RAM_REQUEST=${MONGODB_WIRETIGER_CACHE_SIZE_GB}Gi + MONGODB_MONGOS_RAM_REQUEST=$((TOTAL_RAM_GB * 33 / 1000))Gi + + # Replace values before deploying + sed -i "s/MONGODB_SHARDSERVER_EXTRA_FLAGS/--wiredTigerCacheSizeGB=${MONGODB_WIRETIGER_CACHE_SIZE_GB}/g" $DIR/_build/root/deploy/* + sed -i "s/MONGODB_MONGOS_RAM_LIMIT/${MONGODB_MONGOS_RAM_LIMIT}/g" $DIR/_build/root/deploy/* + sed -i "s/MONGODB_SHARDSERVER_RAM_LIMIT/${MONGODB_SHARDSERVER_RAM_LIMIT}/g" $DIR/_build/root/deploy/* + sed -i "s/MONGODB_SHARDSERVER_RAM_REQUEST/${MONGODB_SHARDSERVER_RAM_REQUEST}/g" $DIR/_build/root/deploy/* + sed -i "s/MONGODB_MONGOS_RAM_REQUEST/${MONGODB_MONGOS_RAM_REQUEST}/g" $DIR/_build/root/deploy/* } get_image_from_deps() { diff --git a/.github/workflows/end2end.yaml b/.github/workflows/end2end.yaml index f256e7362a..00919cedee 100644 --- a/.github/workflows/end2end.yaml +++ b/.github/workflows/end2end.yaml @@ -16,7 +16,7 @@ env: WORKER_COUNT: '2' OPERATOR_REPO: git@github.com:scality/zenko-operator.git OPERATOR_IMAGE: "" - KIND_NODE_IMAGE: "kindest/node:v1.24.17@sha256:bad10f9b98d54586cba05a7eaa1b61c6b90bfc4ee174fdc43a7b75ca75c95e51" + KIND_NODE_IMAGE: "kindest/node:v1.25.16@sha256:6110314339b3b44d10da7d27881849a87e092124afab5956f2e10ecdb463b025" VOLUME_ROOT: /artifacts OIDC_REALM: "zenko" OIDC_CLIENT_ID: "zenko-ui" From 7b3dc9de95844c5d63297b17eb39d6988f526d01 Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Wed, 23 Oct 2024 16:45:57 +0200 Subject: [PATCH 22/22] Release zenko 2.7.64 Issue: ZENKO-4899 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 177c41eb83..b9bf9c004d 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ -VERSION="2.7.63" +VERSION="2.7.64" VERSION_SUFFIX=