make tests for notifications provider agnostic

Signed-off-by: Somtochi Onyekwere <[email protected]>

tests/.gitignore

@@ -13,7 +13,6 @@ build/
 # .tfstate files
 *.tfstate
 *.tfstate.*
-*.terraform.lock.hcl
 # Crash log files
 crash.log
 

tests/integration/.env.sample

@@ -1,11 +1,7 @@
 ## Azure
-export TF_VAR_azuredevops_org=
-export TF_VAR_azuredevops_pat=
-export TF_VAR_azure_location=
-## These are not terraform variables
-## but they are needed for the azure tests
-export AZUREDEVOPS_SSH=
-export AZUREDEVOPS_SSH_PUB=
+# export TF_VAR_azuredevops_org=
+# export TF_VAR_azuredevops_pat=
+# export TF_VAR_azure_location=
 ## Set the following only when authenticating using Service Principal (suited
 ## for CI environment).
 # export ARM_CLIENT_ID=
@@ -14,13 +10,20 @@ export AZUREDEVOPS_SSH_PUB=
 # export ARM_TENANT_ID=
 
 ## GCP
-export TF_VAR_gcp_project_id=
-export TF_VAR_gcp_zone=
-export TF_VAR_gcp_keyring=
-export TF_VAR_gcp_crypto_key=
+# export TF_VAR_gcp_project_id=
+# export TF_VAR_gcp_zone=
+# export TF_VAR_gcp_region=
+# export TF_VAR_gcp_keyring=
+# export TF_VAR_gcp_crypto_key=
+## Email address of a GCP user used for git repository cloning over ssh.
+# export TF_VAR_gcp_email=
 ## Set the following only when using service account.
 ## Provide absolute path to the service account JSON key file.
 # export GOOGLE_APPLICATION_CREDENTIALS=
 
 ## Common variables
 # export TF_VAR_tags='{"environment"="dev", "createdat"='"\"$(date -u +x%Y-%m-%d_%Hh%Mm%Ss)\""'}'
+## These are not terraform variables
+## but they are needed for the bootstrap tests
+# export GITREPO_SSH_PATH=
+# export GITREPO_SSH_PUB_PATH=

tests/integration/README.md

@@ -1,7 +1,7 @@
 # E2E Tests
 
 The goal is to verify that Flux integration with cloud providers are actually working now and in the future.
-Currently, we only have tests for Azure.
+Currently, we only have tests for Azure and GCP.
 
 ## General requirements
 
@@ -60,9 +60,10 @@ the tests:
 ### Architecture
 
 The [gcp](./terraform/gcp) terraform files create the GKE cluster and related resources to run the tests. It creates:
-- An Google Container Registry and Artifact Registry
-- An Google Kubernetes Cluster
+- A Google Container Registry and Artifact Registry
+- A Google Kubernetes Cluster
 - Two Google Cloud Source Repositories
+- A Google Pub/Sub Topic and a subscription to the service that would be used in the tests
 
 Note: It doesn't create Google KMS keyrings and crypto keys because these cannot be destroyed. Instead, you have
 to pass in the crypto key and keyring that would be used to test the sops encryption in Flux. Please see `.env.sample`
@@ -72,6 +73,8 @@ for the terraform variables
 
 - GCP account with an active project to be able to create GKE and GCR, and permission to assign roles.
 - Existing GCP KMS keyring and crypto key.
+  - [Create a Keyring](https://cloud.google.com/kms/docs/create-key-ring)
+  - [Create a Crypto Key](https://cloud.google.com/kms/docs/create-key)
 - gcloud CLI, need to be logged in using `gcloud auth login` as a User (not a
   Service Account), configure application default credentials with `gcloud auth
   application-default login` and docker credential helper with `gcloud auth configure-docker`.
@@ -86,7 +89,6 @@ for the terraform variables
   docker accordingly
   ```console
   $ gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://us-central1-docker.pkg.dev
-  $ gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://gcr.io
   ```
   In this case, the GCP client in terraform uses the Service Account to
   authenticate and the gcloud CLI is used only to authenticate with Google
@@ -100,15 +102,33 @@ for the terraform variables
   to prevent aggressive masking in the logs. Refer
   [aggressive replacement in logs](https://github.com/google-github-actions/auth/blob/v1.1.0/docs/TROUBLESHOOTING.md#aggressive--replacement-in-logs)
   for more details.
-- Register SSH Keys with Google Cloud. # add docs
+- Register [SSH Keys with Google Cloud](https://cloud.google.com/source-repositories/docs/authentication#ssh)
+  - Google Cloud supports these three SSH key types: RSA (only for keys with more than 2048 bits), ECDSA and ED25519
+  - **Note:** Google doesn't allow an SSH key to be associated with a service account email address. Therefore, there has to be an actual
+    user that the SSH keys are registered to, and the email of this user will be passed to terraform through the `TF_VAR_gcp_email`
+    variable.
+
+### Permissions
+
+Following roles are needed for provisioning the infrastructure and running the tests:
+
+- Compute Instance Admin (v1)
+- Kubernetes Engine Admin
+- Service Account User
+- Artifact Registry Administrator
+- Artifact Registry Repository Administrator
+- Cloud KMS Admin
+- Cloud KMS CryptoKey Encrypter
+- Source Repository Administrator
+- Pub/Sub Admin
 
 ## Tests
 
 Each test run is initiated by running `terraform apply` in the provider's terraform directory e.g terraform apply,
 it does this by using the [tftestenv package](https://github.com/fluxcd/test-infra/blob/main/tftestenv/testenv.go)
 within the `fluxcd/test-infra` repository. It then reads the output of the Terraform to get information needed
-for the tests like the kubernetes client ID, the azure DevOps repository urls, the key vault ID etc. This means that
-a lot of the communication with the Azure API is offset to Terraform instead of requiring it to be implemented in the test.
+for the tests like the kubernetes client ID, the cloud repository urls, the key vault ID etc. This means that
+a lot of the communication with the cloud provider API is offset to Terraform instead of requiring it to be implemented in the test.
 
 The following tests are currently implemented:
 
@@ -118,11 +138,11 @@ The following tests are currently implemented:
 - kustomize-controller can decrypt secrets using SOPS and provider key vault
 - image-automation-controller can create branches and push to cloud repositories (https+ssh)
 - source-controller can pull charts from cloud provider container registry Helm repositories
+- notification-controller can forward events to cloud Events Service(EventHub for Azure and Google Pub/Sub)
 
 The following tests are run only for Azure since it is supported in the notification-controller:
 
 - notification-controller can send commit status to Azure DevOps
-- notification-controller can forward events to Azure Event Hub
 
 ### Running tests locally
 
@@ -165,8 +185,9 @@ ok      github.com/fluxcd/flux2/tests/integration       947.341s
 
 In the above, the test created a build directory build/ and the flux cli binary is copied build/flux. It would be used
 to bootstrap Flux on the cluster. You can configure the location of the Flux CLI binary by setting the FLUX_BINARY variable.
-We also pull two version of `ghcr.io/stefanprodan/podinfo` image. These images are pushed to the Azure Container Registry
-and used to test `ImageRepository` and `ImageUpdateAutomation`. The terraform resources get created and the tests are run.
+We also pull two version of `ghcr.io/stefanprodan/podinfo` image. These images are pushed to the cloud provider's
+Container Registry and used to test `ImageRepository` and `ImageUpdateAutomation`. The terraform resources get created
+and the tests are run.
 
 **IMPORTANT:** In case the terraform infrastructure results in a bad state, maybe due to a crash during the apply,
 the whole infrastructure can be destroyed by running terraform destroy in terraform/<provider> directory.

tests/integration/azure_specific_test.go

@@ -21,201 +21,28 @@ package integration
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"io"
 	"log"
 	"strings"
 	"testing"
 	"time"
 
-	eventhub "github.com/Azure/azure-event-hubs-go/v3"
 	"github.com/microsoft/azure-devops-go-api/azuredevops"
 	"github.com/microsoft/azure-devops-go-api/azuredevops/git"
 	. "github.com/onsi/gomega"
 	giturls "github.com/whilp/git-urls"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/types"
 
 	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1"
 	notiv1 "github.com/fluxcd/notification-controller/api/v1"
 	notiv1beta2 "github.com/fluxcd/notification-controller/api/v1beta2"
-	events "github.com/fluxcd/pkg/apis/event/v1beta1"
 	"github.com/fluxcd/pkg/apis/meta"
 	sourcev1 "github.com/fluxcd/source-controller/api/v1"
 )
 
-func TestEventHubNotification(t *testing.T) {
-	g := NewWithT(t)
-
-	ctx := context.TODO()
-	branchName := "test-notification"
-	testID := branchName + "-" + randStringRunes(5)
-
-	// Start listening to eventhub with latest offset
-	// TODO(somtochiama): Make here provider agnostic
-	hub, err := eventhub.NewHubFromConnectionString(cfg.notificationURL)
-	g.Expect(err).ToNot(HaveOccurred())
-	c := make(chan string, 10)
-	handler := func(ctx context.Context, event *eventhub.Event) error {
-		c <- string(event.Data)
-		return nil
-	}
-	runtimeInfo, err := hub.GetRuntimeInformation(ctx)
-	g.Expect(err).ToNot(HaveOccurred())
-	g.Expect(len(runtimeInfo.PartitionIDs)).To(Equal(1))
-	listenerHandler, err := hub.Receive(ctx, runtimeInfo.PartitionIDs[0], handler, eventhub.ReceiveWithLatestOffset())
-	g.Expect(err).ToNot(HaveOccurred())
-
-	// Setup Flux resources
-	manifest := `apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: foobar`
-	repoUrl := getTransportURL(cfg.applicationRepository)
-	client, err := getRepository(ctx, t.TempDir(), repoUrl, defaultBranch, cfg.defaultAuthOpts)
-	g.Expect(err).ToNot(HaveOccurred())
-	files := make(map[string]io.Reader)
-	files["configmap.yaml"] = strings.NewReader(manifest)
-	err = commitAndPushAll(ctx, client, files, branchName)
-	g.Expect(err).ToNot(HaveOccurred())
-
-	namespace := corev1.Namespace{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: testID,
-		},
-	}
-	g.Expect(testEnv.Create(ctx, &namespace)).To(Succeed())
-	defer testEnv.Delete(ctx, &namespace)
-
-	secret := corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      testID,
-			Namespace: testID,
-		},
-		StringData: map[string]string{
-			"address": cfg.notificationURL,
-		},
-	}
-	g.Expect(testEnv.Create(ctx, &secret)).To(Succeed())
-	defer testEnv.Delete(ctx, &secret)
-
-	provider := notiv1beta2.Provider{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      testID,
-			Namespace: testID,
-		},
-		Spec: notiv1beta2.ProviderSpec{
-			Type:    "azureeventhub",
-			Address: repoUrl,
-			SecretRef: &meta.LocalObjectReference{
-				Name: testID,
-			},
-		},
-	}
-	g.Expect(testEnv.Create(ctx, &provider)).To(Succeed())
-	defer testEnv.Delete(ctx, &provider)
-
-	alert := notiv1beta2.Alert{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      testID,
-			Namespace: testID,
-		},
-		Spec: notiv1beta2.AlertSpec{
-			ProviderRef: meta.LocalObjectReference{
-				Name: provider.Name,
-			},
-			EventSources: []notiv1.CrossNamespaceObjectReference{
-				{
-					Kind:      "Kustomization",
-					Name:      testID,
-					Namespace: testID,
-				},
-			},
-		},
-	}
-	g.Expect(testEnv.Create(ctx, &alert)).ToNot(HaveOccurred())
-	defer testEnv.Delete(ctx, &alert)
-
-	g.Eventually(func() bool {
-		nn := types.NamespacedName{Name: alert.Name, Namespace: alert.Namespace}
-		alertObj := &notiv1beta2.Alert{}
-		err := testEnv.Get(ctx, nn, alertObj)
-		if err != nil {
-			return false
-		}
-		if err := checkReadyCondition(alertObj); err != nil {
-			t.Log(err)
-			return false
-		}
-
-		return true
-	}, testTimeout, testInterval).Should(BeTrue())
-
-	modifyKsSpec := func(spec *kustomizev1.KustomizationSpec) {
-		spec.Interval = metav1.Duration{Duration: 30 * time.Second}
-		spec.HealthChecks = []meta.NamespacedObjectKindReference{
-			{
-				APIVersion: "v1",
-				Kind:       "ConfigMap",
-				Name:       "foobar",
-				Namespace:  testID,
-			},
-		}
-	}
-	g.Expect(setUpFluxConfig(ctx, testID, nsConfig{
-		repoURL: repoUrl,
-		ref: &sourcev1.GitRepositoryRef{
-			Branch: branchName,
-		},
-		path:         "./",
-		modifyKsSpec: modifyKsSpec,
-	})).To(Succeed())
-	t.Cleanup(func() {
-		err := tearDownFluxConfig(ctx, testID)
-		if err != nil {
-			t.Logf("failed to delete resources in '%s' namespace: %s", testID, err)
-		}
-	})
-
-	g.Eventually(func() bool {
-		err := verifyGitAndKustomization(ctx, testEnv, testID, testID)
-		if err != nil {
-			t.Log(err)
-			return false
-		}
-		return true
-	}, testTimeout, testInterval).Should(BeTrue())
-
-	// Wait to read even from event hub
-	g.Eventually(func() bool {
-		select {
-		case eventJson := <-c:
-			event := &events.Event{}
-			err := json.Unmarshal([]byte(eventJson), event)
-			if err != nil {
-				t.Logf("the received event type does not match Flux format, error: %v", err)
-				return false
-			}
-
-			if event.InvolvedObject.Kind == kustomizev1.KustomizationKind &&
-				event.InvolvedObject.Name == testID && event.InvolvedObject.Namespace == testID {
-				return true
-			}
-
-			return false
-		default:
-			return false
-		}
-	}, testTimeout, 1*time.Second).Should(BeTrue())
-	err = listenerHandler.Close(ctx)
-	g.Expect(err).ToNot(HaveOccurred())
-	err = hub.Close(ctx)
-	g.Expect(err).ToNot(HaveOccurred())
-}
-
 func TestAzureDevOpsCommitStatus(t *testing.T) {
 	g := NewWithT(t)