From 34a47dee2c536e67ecb34be5bef1f3836d4cd660 Mon Sep 17 00:00:00 2001 From: williamlardier Date: Wed, 4 Sep 2024 18:33:35 +0200 Subject: [PATCH 1/2] Switch to failover during restore in pra CTST test Issue: ZENKO-4887 --- tests/ctst/features/pra.feature | 8 +++++++ tests/ctst/steps/dr/drctl.ts | 13 +++++++++++ tests/ctst/steps/pra.ts | 38 ++++++++++++++++++++++++++------- 3 files changed, 51 insertions(+), 8 deletions(-) diff --git a/tests/ctst/features/pra.feature b/tests/ctst/features/pra.feature index 51d6bf6aaf..ee3b5de36b 100644 --- a/tests/ctst/features/pra.feature +++ b/tests/ctst/features/pra.feature @@ -41,11 +41,19 @@ Feature: PRA operations And the DATA_ACCESSOR user tries to perform PutObject on "DR" site Then it "should not" pass Vault authentication + # Switch to failover + When I request the failover state for the DR + Then the DR sink should be in phase "Failover" + # Restore on DR site When i restore object "obj2-1" for 2 days on "DR" site Then object "obj2-1" should "" be "restored" and have the storage class "e2e-cold" on "DR" site And object "obj2-1" should "" be "transitioned" and have the storage class "e2e-cold" on "Primary" site + # Switch to failback + When I resume operations for the DR + Then the DR sink should be in phase "Running" + # Pause / Resume DR When I pause the DR Then the DR source should be in phase "Paused" diff --git a/tests/ctst/steps/dr/drctl.ts b/tests/ctst/steps/dr/drctl.ts index 6ef99c6e79..3bef123930 100644 --- a/tests/ctst/steps/dr/drctl.ts +++ b/tests/ctst/steps/dr/drctl.ts @@ -112,6 +112,15 @@ type FailoverConfig = { sinkZenkoNamespace?: string; }; +type FailbackConfig = { + wait?: boolean; + timeout?: string; + sinkKubeconfigPath?: string; + sinkKubeconfigData?: string; + sinkZenkoInstance?: string; + sinkZenkoNamespace?: string; +}; + type UninstallConfig = { sinkZenkoDrInstance?: string; sourceZenkoDrInstance?: string; @@ -217,6 +226,10 @@ export default class ZenkoDrctl { return this.runCommand('failover', this.paramToCli(config)); } + async failback(config: FailbackConfig) { + return this.runCommand('failback', this.paramToCli(config)); + } + async status(config: StatusConfig) { return this.runCommand('status', this.paramToCli(config)); } diff --git a/tests/ctst/steps/pra.ts b/tests/ctst/steps/pra.ts index 60b43cd580..e052e4ef8c 100644 --- a/tests/ctst/steps/pra.ts +++ b/tests/ctst/steps/pra.ts @@ -303,13 +303,14 @@ When('the DATA_ACCESSOR user tries to perform PutObject on {string} site', { tim this.setResult(await S3.putObject(this.getCommandParameters())); }); -Then('the kafka DR volume exists', { timeout: 60000 }, async function (this: Zenko) { +const volumeTimeout = 60000; +Then('the kafka DR volume exists', { timeout: volumeTimeout + 2000 }, async function (this: Zenko) { const volumeClaim = await getPVCFromLabel(this, 'kafka_cr', 'end2end-pra-sink-base-queue'); this.logger.debug('kafka volume claim', { volumeClaim }); assert(volumeClaim); const volume = await this.zenkoDrCtl?.volumeGet({ volumeName: volumeClaim.spec?.volumeName, - timeout: '60s', + timeout: `${volumeTimeout.toString()}ms`, }); this.logger.debug('kafka volume from drctl', { volume }); assert(volume); @@ -320,36 +321,57 @@ Then('the kafka DR volume exists', { timeout: 60000 }, async function (this: Zen assert(volumeParsed.result!['volume phase'] === 'Bound'); }); -When('I pause the DR', { timeout: 360000 }, async function (this: Zenko) { +const failoverTimeout = 360000; +When ('I request the failover state for the DR', { timeout: failoverTimeout + 2000 }, async function (this: Zenko) { + await this.zenkoDrCtl?.failover({ + sinkZenkoNamespace: 'default', + wait: true, + timeout: `${failoverTimeout.toString()}ms`, + }); +}); + +const failbackTimeout = 360000; +When ('I resume operations for the DR', { timeout: failbackTimeout + 2000 }, async function (this: Zenko) { + await this.zenkoDrCtl?.failback({ + sinkZenkoNamespace: 'default', + wait: true, + timeout: `${failbackTimeout.toString()}ms`, + }); +}); + +const pauseTimeout = 360000; +When('I pause the DR', { timeout: pauseTimeout + 2000 }, async function (this: Zenko) { await this.zenkoDrCtl?.replicationPause({ sourceZenkoDrInstance: 'end2end-source', sinkZenkoDrInstance: 'end2end-pra-sink', sinkZenkoNamespace: 'default', sourceZenkoNamespace: 'default', wait: true, - timeout: '6m', + timeout: `${pauseTimeout.toString()}ms`, }); }); -When('I resume the DR', { timeout: 360000 }, async function (this: Zenko) { +const resumeTimeout = 360000; +When('I resume the DR', { timeout: resumeTimeout + 2000 }, async function (this: Zenko) { await this.zenkoDrCtl?.replicationResume({ sourceZenkoDrInstance: 'end2end-source', sinkZenkoDrInstance: 'end2end-pra-sink', sinkZenkoNamespace: 'default', sourceZenkoNamespace: 'default', wait: true, - timeout: '6m', + timeout: `${resumeTimeout.toString()}ms`, }); }); -When('I uninstall DR', { timeout: 360000 }, async function (this: Zenko) { +const uninstallTimeout = 360000; +When('I uninstall DR', { timeout: uninstallTimeout + 2000 }, async function (this: Zenko) { await this.zenkoDrCtl?.uninstall({ sourceZenkoDrInstance: 'end2end-source', sinkZenkoDrInstance: 'end2end-pra-sink', sinkZenkoNamespace: 'default', sourceZenkoNamespace: 'default', wait: true, - timeout: '6m', + timeout: `${uninstallTimeout.toString()}ms`, }); }); From 170d48d2c7862d2f4288d4d1b77022c17dbcc344 Mon Sep 17 00:00:00 2001 From: williamlardier Date: Fri, 13 Sep 2024 16:29:22 +0200 Subject: [PATCH 2/2] Test object expiration after failback Issue: ZENKO-4887 --- tests/ctst/features/pra.feature | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/ctst/features/pra.feature b/tests/ctst/features/pra.feature index ee3b5de36b..f0aede1e32 100644 --- a/tests/ctst/features/pra.feature +++ b/tests/ctst/features/pra.feature @@ -46,13 +46,14 @@ Feature: PRA operations Then the DR sink should be in phase "Failover" # Restore on DR site - When i restore object "obj2-1" for 2 days on "DR" site + When i restore object "obj2-1" for 200000 days on "DR" site Then object "obj2-1" should "" be "restored" and have the storage class "e2e-cold" on "DR" site And object "obj2-1" should "" be "transitioned" and have the storage class "e2e-cold" on "Primary" site # Switch to failback When I resume operations for the DR Then the DR sink should be in phase "Running" + And object "obj2-1" should "" be "transitioned" and have the storage class "e2e-cold" on "DR" site # Pause / Resume DR When I pause the DR