From 3b68b3d6fd1d2d9477658bf1e65f8e8b5a1a54ba Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Mon, 29 Jan 2024 16:29:03 +0100 Subject: [PATCH] Introduce MODE and TARGET for E2E tests --- Makefile | 10 +- docs/release/release-tasks.md | 2 +- hack/e2e.sh | 5 +- test/e2e/config/config-overrides.yaml | 36 +++ test/e2e/config/vsphere-dev.yaml | 221 ------------------ .../config/{vsphere-ci.yaml => vsphere.yaml} | 0 test/e2e/e2e_suite_test.go | 56 +++++ 7 files changed, 105 insertions(+), 225 deletions(-) create mode 100644 test/e2e/config/config-overrides.yaml delete mode 100644 test/e2e/config/vsphere-dev.yaml rename test/e2e/config/{vsphere-ci.yaml => vsphere.yaml} (100%) diff --git a/Makefile b/Makefile index 928f983c5e..e3394e9c1e 100644 --- a/Makefile +++ b/Makefile @@ -79,13 +79,16 @@ GINKGO_FOCUS ?= GINKGO_SKIP ?= GINKGO_NODES ?= 1 GINKGO_TIMEOUT ?= 3h -E2E_CONF_FILE ?= $(abspath test/e2e/config/vsphere-dev.yaml) +E2E_CONF_FILE ?= $(abspath test/e2e/config/vsphere.yaml) +E2E_CONF_OVERRIDE_FILE ?= $(abspath test/e2e/config/config-overrides.yaml) +E2E_CAPV_MODE ?= govmomi +E2E_TARGET_TYPE ?= vcenter +E2E_IPAM_KUBECONFIG ?= INTEGRATION_CONF_FILE ?= $(abspath test/integration/integration-dev.yaml) E2E_TEMPLATE_DIR := $(abspath test/e2e/data/infrastructure-vsphere/) SKIP_RESOURCE_CLEANUP ?= false USE_EXISTING_CLUSTER ?= false GINKGO_NOCOLOR ?= false -E2E_IPAM_KUBECONFIG ?= # to set multiple ginkgo skip flags, if any ifneq ($(strip $(GINKGO_SKIP)),) @@ -530,9 +533,12 @@ e2e: $(GINKGO) $(KUSTOMIZE) $(KIND) $(GOVC) ## Run e2e tests time $(GINKGO) -v --trace -focus="$(GINKGO_FOCUS)" $(_SKIP_ARGS) --nodes=$(GINKGO_NODES) -timeout=$(GINKGO_TIMEOUT) \ --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" ./test/e2e -- \ --e2e.config="$(E2E_CONF_FILE)" \ + --e2e.config-overrides="$(E2E_CONF_OVERRIDE_FILE)" \ --e2e.artifacts-folder="$(ARTIFACTS)" \ --e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \ --e2e.use-existing-cluster="$(USE_EXISTING_CLUSTER)" \ + --e2e.capv-mode="$(E2E_CAPV_MODE)" \ + --e2e.target-type="$(E2E_TARGET_TYPE)" \ --e2e.ipam-kubeconfig="$(E2E_IPAM_KUBECONFIG)" ## -------------------------------------- diff --git a/docs/release/release-tasks.md b/docs/release/release-tasks.md index 94e722738f..0c482f3f72 100644 --- a/docs/release/release-tasks.md +++ b/docs/release/release-tasks.md @@ -30,7 +30,7 @@ This comes down to changing occurrences of the old version to the new version, e 1. Goal is that our clusterctl upgrade tests are testing the right versions. For `v1.8` this means: - v1beta1: `v1.8` (will change with each new release) - v1beta1: `v1.7` (will change with each new release) - 2. Update providers in `vsphere-ci.yaml`, `vsphere-dev.yaml`, `integration-dev.yaml`: + 2. Update providers in `vsphere.yaml`, `integration-dev.yaml`: 1. Add a new `v1.7.0` entry. 2. Remove providers that are not used anymore in clusterctl upgrade tests. 3. Change `v1.7.99` to `v1.8.99`. diff --git a/hack/e2e.sh b/hack/e2e.sh index e1293eebba..953c756715 100755 --- a/hack/e2e.sh +++ b/hack/e2e.sh @@ -52,7 +52,10 @@ export VSPHERE_USERNAME="${GOVC_USERNAME}" export VSPHERE_PASSWORD="${GOVC_PASSWORD}" export VSPHERE_SSH_AUTHORIZED_KEY="${VM_SSH_PUB_KEY}" export VSPHERE_SSH_PRIVATE_KEY="/root/ssh/.private-key/private-key" -export E2E_CONF_FILE="${REPO_ROOT}/test/e2e/config/vsphere-ci.yaml" +export E2E_CONF_FILE="${REPO_ROOT}/test/e2e/config/vsphere.yaml" +export E2E_CONF_OVERRIDE_FILE="" +export E2E_CAPV_MODE="${CAPV_MODE:-govmomi}" +export E2E_TARGET_TYPE="${TARGET_TYPE:-vmc}" export ARTIFACTS="${ARTIFACTS:-${REPO_ROOT}/_artifacts}" export DOCKER_IMAGE_TAR="/tmp/images/image.tar" export GC_KIND="false" diff --git a/test/e2e/config/config-overrides.yaml b/test/e2e/config/config-overrides.yaml new file mode 100644 index 0000000000..353996000b --- /dev/null +++ b/test/e2e/config/config-overrides.yaml @@ -0,0 +1,36 @@ +--- +# This e2e config file contains an example of overrides for a tests targeting a user provided vCenter instance. +# Those overrides will be applied on top of the content of the config file used for CI (only variables and intervals are considered) + +variables: + # Change following values with the info about the user provided vCenter instance (current values are just examples) + VSPHERE_SERVER: "vcenter.vmware.com" + VSPHERE_TLS_THUMBPRINT: "AA:BB:CC:DD:11:22:33:44:EE:FF" + VSPHERE_DATACENTER: "SDDC-Datacenter" + VSPHERE_COMPUTE_CLUSTER: "cluster0" + VSPHERE_FOLDER: "FolderName" + VSPHERE_RESOURCE_POOL: "ResourcePool" + VSPHERE_DATASTORE: "WorkloadDatastore" + VSPHERE_STORAGE_POLICY: "Cluster API vSphere Storage Policy" + VSPHERE_NETWORK: "network-1" + VSPHERE_TEMPLATE: "ubuntu-2204-kube-v1.29.0" + FLATCAR_VSPHERE_TEMPLATE: "flatcar-stable-3602.2.3-kube-v1.29.0" + CONTROL_PLANE_ENDPOINT_IP: "10.0.0.10" + # Only for clusterctl upgrade tests + # WORKLOAD_CONTROL_PLANE_ENDPOINT_IP: + # Also following variables are required but it is recommended to use env variables to avoid disclosure of sensitive data + # VSPHERE_SSH_AUTHORIZED_KEY: + # VSPHERE_PASSWORD: + # VSPHERE_USERNAME: + + # Only for multivc_test info about a second user provided vCenter instance must be provided + VSPHERE2_SERVER: "vcenter2.vmware.com" + VSPHERE2_TLS_THUMBPRINT: "AA:BB:CC:DD:11:22:33:44:EE:FF" + VSPHERE2_RESOURCE_POOL: "ResourcePool" + VSPHERE2_TEMPLATE: "ubuntu-2204-kube-v1.29.0" + VSPHERE2_CONTROL_PLANE_ENDPOINT_IP: "10.0.0.11" + # Also following variables are required but it is recommended to use env variables to avoid disclosure of sensitive data + # VSPHERE2_USERNAME: + # VSPHERE2_PASSWORD: + +intervals: diff --git a/test/e2e/config/vsphere-dev.yaml b/test/e2e/config/vsphere-dev.yaml deleted file mode 100644 index 3a61e63279..0000000000 --- a/test/e2e/config/vsphere-dev.yaml +++ /dev/null @@ -1,221 +0,0 @@ ---- -# E2E test scenario using local dev images and manifests built from the source tree for following providers: -# - cluster-api -# - bootstrap kubeadm -# - control-plane kubeadm -# - vsphere - -# For creating local dev images built from the source tree; -# - from the CAPI repository root, `make docker-build REGISTRY=gcr.io/k8s-staging-cluster-api` to build the cluster-api, -# bootstrap kubeadm, control-plane kubeadm provider images. This step can be skipped to use upstream images. -# - from the CAPV repository root, `make e2e` to build the vsphere provider image and run e2e tests. - -images: - - name: registry.k8s.io/cluster-api/cluster-api-controller:v1.6.1 - loadBehavior: tryLoad - - name: registry.k8s.io/cluster-api/kubeadm-bootstrap-controller:v1.6.1 - loadBehavior: tryLoad - - name: registry.k8s.io/cluster-api/kubeadm-control-plane-controller:v1.6.1 - loadBehavior: tryLoad - - name: gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller:dev - loadBehavior: mustLoad - - name: quay.io/jetstack/cert-manager-cainjector:v1.12.2 - loadBehavior: tryLoad - - name: quay.io/jetstack/cert-manager-webhook:v1.12.2 - loadBehavior: tryLoad - - name: quay.io/jetstack/cert-manager-controller:v1.12.2 - loadBehavior: tryLoad - -providers: - - - name: cluster-api - type: CoreProvider - versions: - - name: v1.6.1 - # Use manifest from source files - value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.1/core-components.yaml" - type: "url" - contract: v1beta1 - files: - - sourcePath: "../data/shared/v1.9/v1beta1/metadata.yaml" - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - - name: v1.5.4 - # Use manifest from source files - value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.5.4/core-components.yaml" - type: "url" - contract: v1beta1 - files: - - sourcePath: "../data/shared/v1.8/v1beta1/metadata.yaml" - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - - - name: kubeadm - type: BootstrapProvider - versions: - - name: v1.6.1 - # Use manifest from source files - value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.1/bootstrap-components.yaml" - type: "url" - contract: v1beta1 - files: - - sourcePath: "../data/shared/v1.9/v1beta1/metadata.yaml" - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - - name: v1.5.4 - # Use manifest from source files - value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.5.4/bootstrap-components.yaml" - type: "url" - contract: v1beta1 - files: - - sourcePath: "../data/shared/v1.8/v1beta1/metadata.yaml" - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - - - name: kubeadm - type: ControlPlaneProvider - versions: - - name: v1.6.1 - # Use manifest from source files - value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.1/control-plane-components.yaml" - type: "url" - contract: v1beta1 - files: - - sourcePath: "../data/shared/v1.9/v1beta1/metadata.yaml" - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - - name: v1.5.4 - # Use manifest from source files - value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.5.4/control-plane-components.yaml" - type: "url" - contract: v1beta1 - files: - - sourcePath: "../data/shared/v1.8/v1beta1/metadata.yaml" - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - - - name: vsphere - type: InfrastructureProvider - versions: - - name: v1.10.99 - # Use manifest from source files - value: ../../../../cluster-api-provider-vsphere/config/default - contract: v1beta1 - replacements: - - old: gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller:main - new: gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller:dev - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - files: - # Add a cluster template - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-conformance.yaml" - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-install-on-bootstrap.yaml" - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-dhcp-overrides.yaml" - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-hw-upgrade.yaml" - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-ignition.yaml" - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-kcp-remediation.yaml" - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-md-remediation.yaml" - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-node-drain.yaml" - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-ownerrefs-finalizers.yaml" - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-pci.yaml" - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-remote-management.yaml" - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-storage-policy.yaml" - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-topology.yaml" - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template.yaml" - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/clusterclass-quick-start.yaml" - - sourcePath: "../data/shared/main/v1beta1_provider/metadata.yaml" - - name: v1.9.0 - # Use manifest from source files - value: "https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/v1.9.0/infrastructure-components.yaml" - type: "url" - contract: v1beta1 - files: - # Add a cluster template - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/v1.9/cluster-template-workload.yaml" - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/v1.9/clusterclass-quick-start.yaml" - - sourcePath: "../data/shared/v1.9/v1beta1_provider/metadata.yaml" - - name: v1.8.4 - # Use manifest from source files - value: "https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/v1.8.4/infrastructure-components.yaml" - type: "url" - contract: v1beta1 - files: - # Add a cluster template - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/v1.8/cluster-template-workload.yaml" - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/v1.8/clusterclass-quick-start.yaml" - - sourcePath: "../data/shared/v1.8/v1beta1_provider/metadata.yaml" - -variables: - # Ensure all Kubernetes versions used here are covered in patch-vsphere-template.yaml - KUBERNETES_VERSION: "v1.29.0" - KUBERNETES_VERSION_UPGRADE_FROM: "v1.28.0" - KUBERNETES_VERSION_UPGRADE_TO: "v1.29.0" - KUBERNETES_VERSION_LATEST_CI: "ci/latest-1.30" - CPI_IMAGE_K8S_VERSION: "v1.29.0" - CNI: "./data/cni/calico/calico.yaml" - EXP_CLUSTER_RESOURCE_SET: "true" - EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION: "true" - CONTROL_PLANE_MACHINE_COUNT: 1 - WORKER_MACHINE_COUNT: 1 - IP_FAMILY: "IPv4" - CLUSTER_CLASS_NAME: "quick-start" - # Following CAPV variables should be set before testing - VSPHERE_SERVER: "vcenter.vmware.com" - VSPHERE_TLS_THUMBPRINT: "AA:BB:CC:DD:11:22:33:44:EE:FF" - VSPHERE_DATACENTER: "SDDC-Datacenter" - VSPHERE_COMPUTE_CLUSTER: "cluster0" - VSPHERE_FOLDER: "FolderName" - VSPHERE_RESOURCE_POOL: "ResourcePool" - VSPHERE_DATASTORE: "WorkloadDatastore" - VSPHERE_STORAGE_POLICY: "Cluster API vSphere Storage Policy" - VSPHERE_NETWORK: "network-1" - VSPHERE_TEMPLATE: "ubuntu-2204-kube-v1.29.0" - FLATCAR_VSPHERE_TEMPLATE: "flatcar-stable-3602.2.3-kube-v1.29.0" - # WORKLOAD_CONTROL_PLANE_ENDPOINT_IP: - # Also following variables are required but it is recommended to use env variables to avoid disclosure of sensitive data - # VSPHERE_SSH_AUTHORIZED_KEY: - # VSPHERE_PASSWORD: - # VSPHERE_USERNAME: - # Dedicated IP to be used by kube-vip - # CONTROL_PLANE_ENDPOINT_IP: - # Sets the insecure-flag for vsphere-csi-controller config - VSPHERE_INSECURE_CSI: "true" - KUBETEST_CONFIGURATION: "./data/kubetest/conformance-fast.yaml" - NODE_DRAIN_TIMEOUT: "60s" - CLUSTER_TOPOLOGY: "true" - # These IDs correspond to Tesla T4s, they are the decimal representation of the hex values. - DEVICE_ID: 7864 - VENDOR_ID: 4318 - # CAPV feature flags - EXP_NODE_ANTI_AFFINITY: "true" - # Following CAPV variables is used for multivc_test.go. This is the second VSphere and should be set if multivc test is enabled. - VSPHERE2_SERVER: "vcenter2.vmware.com" - VSPHERE2_TLS_THUMBPRINT: "AA:BB:CC:DD:11:22:33:44:EE:FF" - VSPHERE2_RESOURCE_POOL: "ResourcePool" - VSPHERE2_TEMPLATE: "ubuntu-2204-kube-v1.29.0" - # Dedicated IP to be used by kube-vip - VSPHERE2_CONTROL_PLANE_ENDPOINT_IP: - # Following variables are also required and please use env variables to avoid disclosure of sensitive data - VSPHERE2_USERNAME: - VSPHERE2_PASSWORD: - CAPI_DIAGNOSTICS_ADDRESS: ":8080" - CAPI_INSECURE_DIAGNOSTICS: "true" - -intervals: - default/wait-controllers: ["5m", "10s"] - default/wait-cluster: ["5m", "10s"] - default/wait-control-plane: ["10m", "10s"] - default/wait-worker-nodes: ["10m", "10s"] - default/wait-delete-cluster: ["5m", "10s"] - default/wait-machine-upgrade: ["15m", "1m"] - default/wait-machine-remediation: ["15m", "10s"] - mhc-remediation/mhc-remediation: ["30m", "10s"] - node-drain/wait-deployment-available: ["3m", "10s"] - node-drain/wait-machine-deleted: ["2m", "10s"] - anti-affinity/wait-vm-redistribution: ["5m", "10s"] diff --git a/test/e2e/config/vsphere-ci.yaml b/test/e2e/config/vsphere.yaml similarity index 100% rename from test/e2e/config/vsphere-ci.yaml rename to test/e2e/config/vsphere.yaml diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 29cc162263..1e9cb8cf7d 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -36,6 +36,7 @@ import ( . "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" capiutil "sigs.k8s.io/cluster-api/util" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/yaml" infrav1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/v1beta1" vsphereframework "sigs.k8s.io/cluster-api-provider-vsphere/test/framework" @@ -47,11 +48,34 @@ const ( VsphereStoragePolicy = "VSPHERE_STORAGE_POLICY" ) +const ( + // GovmomiTestMode identify tests run with CAPV using govmomi to access vCenter. + GovmomiTestMode string = "govmomi" + + // SupervisorTestMode identify tests run with CAPV in supervisor mode (delegating to vm-operator all the interaction with vCenter). + SupervisorTestMode string = "supervisor" +) + +const ( + // VMCTestTarget identify tests targeting VMC infrastructure used for CAPV CI. + VMCTestTarget string = "vmc" + + // VCenterTestTarget identify tests targeting a user provided vCenter instance. + VCenterTestTarget string = "vcenter" + + // VCSimTestTarget identify tests targeting a vcsim instance (instead of a real vCenter). + VCSimTestTarget string = "vcsim" +) + // Test suite flags. var ( // configPath is the path to the e2e config file. configPath string + // configOverridesPath is the path to the e2e config file containing overrides to the content of configPath config file. + // Only variables and intervals are considered. + configOverridesPath string + // useExistingCluster instructs the test to use the current cluster instead // of creating a new one (default discovery rules apply). useExistingCluster bool @@ -65,6 +89,12 @@ var ( // skipCleanup prevents cleanup of test resources e.g. for debug purposes. skipCleanup bool + + // defines how CAPV should behave during this test. + testMode string + + // defines which type of infrastructure this test targets. + testTarget string ) // Test suite global vars. @@ -95,13 +125,21 @@ var ( ipAddressManager vsphereip.AddressManager ) +type configOverrides struct { + Variables map[string]string `json:"variables,omitempty"` + Intervals map[string][]string `json:"intervals,omitempty"` +} + func init() { flag.StringVar(&configPath, "e2e.config", "", "path to the e2e config file") + flag.StringVar(&configOverridesPath, "e2e.config-overrides", "", "path to the e2e config file containing overrides to the e2e config file") flag.StringVar(&artifactFolder, "e2e.artifacts-folder", "", "folder where e2e test artifact should be stored") flag.BoolVar(&alsoLogToFile, "e2e.also-log-to-file", true, "if true, ginkgo logs are additionally written to the `ginkgo-log.txt` file in the artifacts folder (including timestamps)") flag.BoolVar(&skipCleanup, "e2e.skip-resource-cleanup", false, "if true, the resource cleanup after tests will be skipped") flag.BoolVar(&useExistingCluster, "e2e.use-existing-cluster", false, "if true, the test uses the current cluster instead of creating a new one (default discovery rules apply)") flag.StringVar(&e2eIPAMKubeconfig, "e2e.ipam-kubeconfig", "", "path to the kubeconfig for the IPAM cluster") + flag.StringVar(&testMode, "e2e.capv-mode", GovmomiTestMode, "defines how CAPV should behave during this test, one of govmomi|supervisor") + flag.StringVar(&testTarget, "e2e.target-type", VMCTestTarget, "defines which type of infrastructure this test targets, one of vmc|vcenter|vcsim") } func TestE2E(t *testing.T) { @@ -143,6 +181,24 @@ var _ = SynchronizedBeforeSuite(func() []byte { var err error e2eConfig, err = vsphereframework.LoadE2EConfig(ctx, configPath) Expect(err).NotTo(HaveOccurred()) + if configOverridesPath != "" { + Expect(configOverridesPath).To(BeAnExistingFile(), "Invalid test suite argument. e2e.config-overrides should be an existing file.") + + Byf("Merging with e2e config overrides from %q", configOverridesPath) + configData, err := os.ReadFile(configOverridesPath) //nolint:gosec + Expect(err).ToNot(HaveOccurred(), "Failed to read e2e config overrides") + Expect(configData).ToNot(BeEmpty(), "The e2e config overrides should not be empty") + + configOverrides := &configOverrides{} + Expect(yaml.Unmarshal(configData, configOverrides)).To(Succeed(), "Failed to convert e2e config overrides to yaml") + + for k, v := range configOverrides.Variables { + e2eConfig.Variables[k] = v + } + for k, v := range configOverrides.Intervals { + e2eConfig.Intervals[k] = v + } + } Byf("Creating a clusterctl local repository into %q", artifactFolder) clusterctlConfigPath, err = vsphereframework.CreateClusterctlLocalRepository(ctx, e2eConfig, filepath.Join(artifactFolder, "repository"), true)