Add refactored e2e tests

Signed-off-by: Somtochi Onyekwere <[email protected]>

.github/workflows/e2e-azure.yaml

@@ -23,6 +23,9 @@ permissions:
 jobs:
   e2e-amd64-aks:
     runs-on: ubuntu-22.04
+    defaults:
+      run:
+        working-directory: ./tests/azure
     if: (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && github.actor != 'dependabot[bot]'
     steps:
       - name: Checkout
@@ -31,20 +34,18 @@ jobs:
         uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
         with:
           go-version: 1.20.x
-          cache-dependency-path: |
-            **/go.sum
-            **/go.mod
+          cache-dependency-path: tests/azure/go.sum
       - name: Setup Flux CLI
         run: |
           make build
           mkdir -p $HOME/.local/bin
           mv ./bin/flux $HOME/.local/bin
+        working-directory: ./
       - name: Setup SOPS
         run: |
-          wget https://github.com/mozilla/sops/releases/download/v3.7.1/sops-v3.7.1.linux
-          chmod +x sops-v3.7.1.linux
           mkdir -p $HOME/.local/bin
-          mv sops-v3.7.1.linux $HOME/.local/bin/sops
+          wget https://github.com/mozilla/sops/releases/download/v3.7.1/sops-v3.7.1.linux -O $HOME/.local/bin/sops
+          chmod +x $HOME/.local/bin/sops
       - name: Setup Terraform
         uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1  # v2
         with:
@@ -60,9 +61,62 @@ jobs:
           ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }}
           ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
         run: |
-          echo $HOME
-          echo $PATH
           ls $HOME/.local/bin
           az login --service-principal -u ${ARM_CLIENT_ID} -p ${ARM_CLIENT_SECRET} -t ${ARM_TENANT_ID}
-          cd ./tests/azure
           go test -v -coverprofile cover.out -timeout 60m .
+
+  refactored-e2e-amd64-aks:
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        working-directory: ./tests/integration
+    steps:
+      - name: CheckoutD
+        uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
+      - name: Setup Go
+        uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 # v4.0.0
+        with:
+          go-version: 1.20.x
+          cache-dependency-path: tests/integration/go.sum
+      - name: Setup Flux CLI
+        run: make build
+        working-directory: ./
+      - name: Setup SOPS
+        run: |
+          mkdir -p $HOME/.local/bin
+          wget -O $HOME/.local/bin/sops https://github.com/mozilla/sops/releases/download/v$SOPS_VER/sops-v$SOPS_VER.linux
+          chmod +x $HOME/.local/bin/sops
+        env:
+          SOPS_VER: 3.7.1
+      - name: Authenticate to Azure
+        uses: Azure/login@92a5484dfaf04ca78a94597f4f19fea633851fa2 # v1.4.6
+        with:
+          creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
+      - name: Set dynamic variables in .env
+        run: |
+          cat > .env <<EOF
+          export TF_VAR_tags='{ "environment"="github", "ci"="true", "repo"="image-reflector-controller", "createdat"="$(date -u +x%Y-%m-%d_%Hh%Mm%Ss)" }'
+          EOF
+      - name: Print .env for dynamic tag value reference
+        run: cat .env
+      - name: Run Azure e2e tests
+        env:
+          ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
+          ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }}
+          ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }}
+          ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
+          TF_VAR_azuredevops_org: ${{ secrets.AZUREDEVOPS_ORG }}
+          TF_VAR_azuredevops_pat: ${{ secrets.AZUREDEVOPS_PAT }}
+          TF_VAR_location: "southcentralus"
+          AZUREDEVOPS_SSH_CONTENTS: ${{ secrets.AZUREDEVOPS_ID_RSA }}
+          AZUREDEVOPS_SSH_PUB_CONTENTS: ${{ secrets.AZUREDEVOPS_ID_RSA_PUB }}
+        run: |
+          source .env
+          mkdir -p ./build/ssh
+          touch ./build/ssh/key
+          echo $AZUREDEVOPS_SSH_CONTENTS | base64 -d > build/ssh/key
+          export AZUREDEVOPS_SSH=build/ssh/key
+          touch ./build/ssh/key.pub
+          echo $AZUREDEVOPS_SSH_PUB_CONTENTS | base64 -d > ./build/ssh/key.pub
+          export AZUREDEVOPS_SSH_PUB=build/ssh/key.pub
+          make test-azure

Makefile

@@ -19,6 +19,7 @@ all: test build
 tidy:
 	go mod tidy -compat=1.20
 	cd tests/azure && go mod tidy -compat=1.20
+	cd tests/integration && go mod tidy -compat=1.20
 
 fmt:
 	go fmt ./...

tests/.gitignore

@@ -5,11 +5,14 @@
 ### Terraform ###
 # Local .terraform directories
 **/.terraform/*
+*.terraform.lock.hcl
+
+# test files
+build/
 
 # .tfstate files
 *.tfstate
 *.tfstate.*
-
 # Crash log files
 crash.log
 
@@ -37,4 +40,5 @@ override.tf.json
 .terraformrc
 terraform.rc
 
+.env
 # End of https://www.toptal.com/developers/gitignore/api/terraform

tests/integration/.env.sample

@@ -0,0 +1,17 @@
+## Azure
+export TF_VAR_azuredevops_org=
+export TF_VAR_azuredevops_pat=
+export TF_VAR_azure_location=
+## These are not terraform variables
+## but they are needed for the azure tests
+export AZUREDEVOPS_SSH=
+export AZUREDEVOPS_SSH_PUB=
+## Set the following only when authenticating using Service Principal (suited
+## for CI environment).
+# export ARM_CLIENT_ID=
+# export ARM_CLIENT_SECRET=
+# export ARM_SUBSCRIPTION_ID=
+# export ARM_TENANT_ID=
+
+## Common variables
+# export TF_VAR_tags='{"environment"="dev", "createdat"='"\"$(date -u +x%Y-%m-%d_%Hh%Mm%Ss)\""'}'

tests/integration/Makefile

@@ -0,0 +1,24 @@
+GO_TEST_ARGS ?=
+PROVIDER_ARG ?=
+TEST_TIMEOUT ?= 60m
+FLUX_BINARY ?= ../../bin/flux
+
+test: sops-check
+	mkdir -p build
+	cp $(FLUX_BINARY) build/flux
+	# These two versions of podinfo are pushed to the cloud registry and used in tests for ImageUpdateAutomation
+	docker pull ghcr.io/stefanprodan/podinfo:6.0.0
+	docker pull ghcr.io/stefanprodan/podinfo:6.0.1
+	go test -timeout $(TEST_TIMEOUT) -v ./ $(GO_TEST_ARGS) $(PROVIDER_ARG)
+
+test-azure:
+	$(MAKE) test PROVIDER_ARG="-provider azure" GO_TEST_ARGS="--tags azure $(GO_TEST_ARGS)"
+
+test-gcp:
+	$(MAKE) test PROVIDER_ARG="-provider gcp"
+
+
+sops-check:
+ifeq ($(shell which sops),)
+	$(error "no sops in PATH, consider installing")
+endif

tests/integration/README.md

@@ -0,0 +1,158 @@
+# E2E Tests
+
+The goal is to verify that Flux integration with cloud providers are actually working now and in the future.
+Currently, we only have tests for Azure.
+
+## General requirements
+
+These CLI tools need to be installed for each of the tests to run successfully.
+
+- Docker CLI for registry login.
+- [SOPS CLI](https://github.com/mozilla/sops) for encrypting files
+- kubectl for applying certain install manifests.
+
+## Azure
+
+### Architecture
+
+The [azure](./terraform/azure) Terraform creates the AKS cluster and related resources to run the tests. It creates:
+- An Azure Container Registry
+- An Azure Kubernetes Cluster
+- Two Azure DevOps repositories
+- Azure EventHub for sending notifications
+- An Azure Key Vault
+
+### Requirements
+
+- Azure account with an active subscription to be able to create AKS and ACR,
+  and permission to assign roles. Role assignment is required for allowing AKS workloads to access ACR.
+- Azure CLI, need to be logged in using `az login` as a User or as a Service Principal
+- An Azure DevOps organization, personal access token and ssh keys for accessing repositories
+  within the organization. The scope required for the personal access token is:
+  - `Project and Team` - read, write and manage access
+  - `Code` - Full
+  - Please take a look at the [terraform provider](https://registry.terraform.io/providers/microsoft/azuredevops/latest/docs/guides/authenticating_using_the_personal_access_token#create-a-personal-access-token)
+    for more explanation.
+  - Azure DevOps only supports RSA keys. Please see
+    [documentation](https://learn.microsoft.com/en-us/azure/devops/repos/git/use-ssh-keys-to-authenticate?view=azure-devops#set-up-ssh-key-authentication)
+    for how to set up SSH key authentication.
+
+**NOTE:** To use Service Principal (for example in CI environment), set the
+`ARM-*` variables in `.env`, source it and authenticate Azure CLI with:
+```console
+$ az login --service-principal -u $ARM_CLIENT_ID -p $ARM_CLIENT_SECRET --tenant $ARM_TENANT_ID
+```
+
+### Permissions
+
+Following permissions are needed for provisioning the infrastructure and running
+the tests:
+- `Microsoft.Kubernetes/*`
+- `Microsoft.Resources/*`
+- `Microsoft.Authorization/roleAssignments/{Read,Write,Delete}`
+- `Microsoft.ContainerRegistry/*`
+- `Microsoft.ContainerService/*`
+- `Microsoft.KeyVault/*`
+- `Microsoft.EventHub/*`
+
+
+## Tests
+
+Each test run is initiated by running `terraform apply` in the provider's terraform directory e.g terraform apply,
+it does this by using the [tftestenv package](https://github.com/fluxcd/test-infra/blob/main/tftestenv/testenv.go)
+within the `fluxcd/test-infra` repository. It then reads the output of the Terraform to get information needed
+for the tests like the kubernetes client ID, the azure DevOps repository urls, the key vault ID etc. This means that
+a lot of the communication with the Azure API is offset to Terraform instead of requiring it to be implemented in the test.
+
+The following tests are currently implemented:
+
+- Flux can be successfully installed on the cluster using the Flux CLI
+- source-controller can clone cloud provider repositories (Azure DevOps, Google Cloud Source Repositories) (https+ssh)
+- image-reflector-controller can list tags from provider container Registry image repositories
+- kustomize-controller can decrypt secrets using SOPS and provider key vault
+- image-automation-controller can create branches and push to cloud repositories (https+ssh)
+- source-controller can pull charts from cloud provider container registry Helm repositories
+
+The following tests are run only for Azure since it is supported in the notification-controller:
+
+- notification-controller can send commit status to Azure DevOps
+- notification-controller can forward events to Azure Event Hub
+
+### Running tests locally
+
+1. Ensure that you have the Flux CLI binary that is to be tested built and ready. You can build it by running
+   `make build` at the root of this repository. The binary is located at `./bin` directory at the root and by default
+   this is where the Makefile copies the binary for the tests from. If you have it in a different location, you can set it
+   with the `FLUX_BINARY` variable
+2. Copy `.env.sample` to `.env` and add the values for the different variables for the provider that you are running the tests for. 
+3. Run `make test-<provider>`, setting the location of the flux binary with `FLUX_BINARY` variable
+
+```console
+$ make test-azure
+make test PROVIDER_ARG="-provider azure"
+# These two versions of podinfo are pushed to the cloud registry and used in tests for ImageUpdateAutomation
+mkdir -p build
+cp ../../bin/flux build/flux
+docker pull ghcr.io/stefanprodan/podinfo:6.0.0
+6.0.0: Pulling from stefanprodan/podinfo
+Digest: sha256:e7eeab287181791d36c82c904206a845e30557c3a4a66a8143fa1a15655dae97
+Status: Image is up to date for ghcr.io/stefanprodan/podinfo:6.0.0
+ghcr.io/stefanprodan/podinfo:6.0.0
+docker pull ghcr.io/stefanprodan/podinfo:6.0.1
+6.0.1: Pulling from stefanprodan/podinfo
+Digest: sha256:1169f220a670cf640e45e1a7ac42dc381a441e9d4b7396432cadb75beb5b5d68
+Status: Image is up to date for ghcr.io/stefanprodan/podinfo:6.0.1
+ghcr.io/stefanprodan/podinfo:6.0.1
+go test -timeout 60m -v ./ -existing -provider azure --tags=integration
+2023/03/24 02:32:25 Setting up azure e2e test infrastructure
+2023/03/24 02:32:25 Terraform binary:  /usr/local/bin/terraform
+2023/03/24 02:32:25 Init Terraform
+....[some output has been cut out]
+2023/03/24 02:39:33 helm repository condition not ready
+--- PASS: TestACRHelmRelease (15.31s)
+=== RUN   TestKeyVaultSops
+--- PASS: TestKeyVaultSops (15.98s)
+PASS
+2023/03/24 02:40:12 Destroying environment...
+ok      github.com/fluxcd/flux2/tests/integration       947.341s
+```
+
+In the above, the test created a build directory build/ and the flux cli binary is copied build/flux. It would be used
+to bootstrap Flux on the cluster. You can configure the location of the Flux CLI binary by setting the FLUX_BINARY variable.
+We also pull two version of `ghcr.io/stefanprodan/podinfo` image. These images are pushed to the Azure Container Registry
+and used to test `ImageRepository` and `ImageUpdateAutomation`. The terraform resources get created and the tests are run.
+
+**IMPORTANT:** In case the terraform infrastructure results in a bad state, maybe due to a crash during the apply,
+the whole infrastructure can be destroyed by running terraform destroy in terraform/<provider> directory.
+
+### Debugging the tests
+
+For debugging environment provisioning, enable verbose output with `-verbose` test flag.
+
+```sh
+make test-azure GO_TEST_ARGS="-verbose"
+```
+
+The test environment is destroyed at the end by default. Run the tests with -retain flag to retain the created test infrastructure.
+
+```sh
+make test-azure GO_TEST_ARGS="-retain"
+```
+The tests require the infrastructure state to be clean. For re-running the tests with a retained infrastructure, set -existing flag.
+
+```sh
+make test-azure GO_TEST_ARGS="-retain -existing"
+```
+
+To delete an existing infrastructure created with -retain flag:
+
+```sh
+make test-azure GO_TEST_ARGS="-existing"
+```
+
+To debug issues on the cluster created by the test (provided you passed in the `-retain` flag):
+
+```sh
+export KUBECONFIG=./build/kubeconfig
+kubectl get pods
+```