tests: add helm upgrade tests

config/manager/manager.yaml

@@ -12,6 +12,7 @@ metadata:
   namespace: system
   labels:
     control-plane: controller-manager
+    app.kubernetes.io/name: gateway-operator
 spec:
   strategy:
     type: Recreate

go.mod

@@ -31,26 +31,56 @@ require (
 )
 
 require (
+	github.com/BurntSushi/toml v1.3.2 // indirect
 	github.com/avast/retry-go/v4 v4.5.1 // indirect
+	github.com/aws/aws-sdk-go v1.49.13 // indirect
 	github.com/bombsimon/logrusr/v3 v3.1.0 // indirect
+	github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
 	github.com/distribution/reference v0.5.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-sql-driver/mysql v1.7.1 // indirect
+	github.com/gonvenience/bunt v1.3.5 // indirect
+	github.com/gonvenience/neat v1.3.12 // indirect
+	github.com/gonvenience/term v1.0.2 // indirect
+	github.com/gonvenience/text v1.0.7 // indirect
+	github.com/gonvenience/wrap v1.1.2 // indirect
+	github.com/gonvenience/ytbx v1.4.4 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-github/v48 v48.2.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/gruntwork-io/go-commons v0.8.0 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/homeport/dyff v1.6.0 // indirect
+	github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 // indirect
 	github.com/kong/go-kong v0.54.0 // indirect
 	github.com/kr/text v0.2.0 // indirect
+	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
+	github.com/mattn/go-ciede2000 v0.0.0-20170301095244-782e8c62fec3 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/go-ps v1.0.0 // indirect
+	github.com/mitchellh/hashstructure v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
 	github.com/pelletier/go-toml v1.9.4 // indirect
+	github.com/pquerna/otp v1.2.0 // indirect
 	github.com/rogpeppe/go-internal v1.12.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/sergi/go-diff v1.3.1 // indirect
+	github.com/texttheater/golang-levenshtein v1.0.1 // indirect
 	github.com/tidwall/gjson v1.17.1 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
+	github.com/urfave/cli v1.22.14 // indirect
+	github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
 	go.opentelemetry.io/otel v1.24.0 // indirect
@@ -119,6 +149,7 @@ require (
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/go-cmp v0.6.0
 	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/gruntwork-io/terratest v0.46.13
 	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect

modules/cli/cli_test.go

@@ -66,7 +66,7 @@ func TestParse(t *testing.T) {
 				"--metrics-bind-address=:18080",
 			},
 			envVars: map[string]string{
-				"GATEWAY_OPERATOR_METRIC_BIND_ADDRESS":       ":28080",
+				"GATEWAY_OPERATOR_METRICS_BIND_ADDRESS":      ":28080",
 				"GATEWAY_OPERATOR_HEALTH_PROBE_BIND_ADDRESS": ":28081",
 			},
 			expectedCfg: func() manager.Config {

test/e2e/environment.go

@@ -92,6 +92,11 @@ type TestEnvOption func(opt *testEnvOptions)
 
 type testEnvOptions struct {
 	Image string
+	// InstallViaKustomize makes the test environment install the operator and all the
+	// dependencies via kustomize.
+	// NOTE: when this is false the caller is responsible for installing (and cleaning up)
+	// the operator in the test environment.
+	InstallViaKustomize bool
 }
 
 // WithOperatorImage allows configuring the operator image to use in the test environment.
@@ -101,11 +106,25 @@ func WithOperatorImage(image string) TestEnvOption {
 	}
 }
 
+// WithInstallViaKustomize makes the test environment install the operator and all the
+// dependencies via kustomize.
+func WithInstallViaKustomize() TestEnvOption {
+	return func(opts *testEnvOptions) {
+		opts.InstallViaKustomize = true
+	}
+}
+
 var loggerOnce sync.Once
 
 // CreateEnvironment creates a new independent testing environment for running isolated e2e test.
+// When running with Helm, the caller is responsible for cleaning up the environment.
 func CreateEnvironment(t *testing.T, ctx context.Context, opts ...TestEnvOption) TestEnvironment {
 	t.Helper()
+
+	const (
+		waitTime = 1 * time.Minute
+	)
+
 	var opt testEnvOptions
 	for _, o := range opts {
 		o(&opt)
@@ -154,12 +173,19 @@ func CreateEnvironment(t *testing.T, ctx context.Context, opts ...TestEnvOption)
 	if len(opt.Image) == 0 {
 		opt.Image = getOperatorImage(t)
 	}
-	kustomizeDir := PrepareKustomizeDir(t, opt.Image)
+
+	var kustomizeDir KustomizeDir
+	if opt.InstallViaKustomize {
+		kustomizeDir = PrepareKustomizeDir(t, opt.Image)
+	}
 
 	env, err := builder.Build(ctx)
 	require.NoError(t, err)
+
 	t.Cleanup(func() {
-		cleanupEnvironment(t, context.Background(), env, kustomizeDir.Tests())
+		if opt.InstallViaKustomize {
+			cleanupEnvironment(t, context.Background(), env, kustomizeDir.Tests())
+		}
 	})
 
 	t.Logf("waiting for cluster %s and all addons to become ready", env.Cluster().Name())
@@ -193,28 +219,32 @@ func CreateEnvironment(t *testing.T, ctx context.Context, opts ...TestEnvOption)
 	require.NoError(t, operatorv1alpha1.AddToScheme(clients.MgrClient.Scheme()))
 	require.NoError(t, operatorv1beta1.AddToScheme(clients.MgrClient.Scheme()))
 
-	t.Logf("deploying Gateway APIs CRDs from %s", testutils.GatewayExperimentalCRDsKustomizeURL)
-	require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), testutils.GatewayExperimentalCRDsKustomizeURL))
+	if opt.InstallViaKustomize {
+		t.Logf("deploying Gateway APIs CRDs from %s", testutils.GatewayExperimentalCRDsKustomizeURL)
+		require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), testutils.GatewayExperimentalCRDsKustomizeURL))
 
-	kicCRDsKustomizeURL := getCRDsKustomizeURLForKIC(t, versions.DefaultControlPlaneVersion)
-	t.Logf("deploying KIC CRDs from %s", kicCRDsKustomizeURL)
-	require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kicCRDsKustomizeURL))
+		kicCRDsKustomizeURL := getCRDsKustomizeURLForKIC(t, versions.DefaultControlPlaneVersion)
+		t.Logf("deploying KIC CRDs from %s", kicCRDsKustomizeURL)
+		require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kicCRDsKustomizeURL))
 
-	t.Log("creating system namespaces and serviceaccounts")
-	require.NoError(t, clusters.CreateNamespace(ctx, env.Cluster(), "kong-system"))
+		t.Log("creating system namespaces and serviceaccounts")
+		require.NoError(t, clusters.CreateNamespace(ctx, env.Cluster(), "kong-system"))
 
-	t.Log("deploying operator CRDs to test cluster via kustomize")
-	require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kustomizeDir.CRD(), "--server-side"))
+		t.Log("deploying operator CRDs to test cluster via kustomize")
+		require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kustomizeDir.CRD(), "--server-side"))
 
-	t.Log("deploying operator to test cluster via kustomize")
-	require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kustomizeDir.Tests(), "--server-side"))
+		t.Log("deploying operator to test cluster via kustomize")
+		require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kustomizeDir.Tests(), "--server-side"))
 
-	t.Log("waiting for operator deployment to complete")
-	require.NoError(t, waitForOperatorDeployment(ctx, clients.K8sClient))
+		t.Log("waiting for operator deployment to complete")
+		require.NoError(t, waitForOperatorDeployment(t, ctx, "kong-system", clients.K8sClient, waitTime))
 
-	t.Log("waiting for operator webhook service to be connective")
-	require.Eventually(t, waitForOperatorWebhookEventually(t, ctx, clients.K8sClient),
-		webhookReadinessTimeout, webhookReadinessTick)
+		t.Log("waiting for operator webhook service to be connective")
+		require.Eventually(t, waitForOperatorWebhookEventually(t, ctx, clients.K8sClient),
+			webhookReadinessTimeout, webhookReadinessTick)
+	} else {
+		t.Log("not deploying operator to test cluster via kustomize")
+	}
 
 	t.Log("environment is ready, starting tests")
 
@@ -257,43 +287,118 @@ func cleanupEnvironment(t *testing.T, ctx context.Context, env environments.Envi
 
 type deploymentAssertOptions func(*appsv1.Deployment) bool
 
-func deploymentAssertConditions(conds ...appsv1.DeploymentCondition) deploymentAssertOptions {
+func deploymentAssertConditions(t *testing.T, conds ...appsv1.DeploymentCondition) deploymentAssertOptions {
+	t.Helper()
+
 	return func(deployment *appsv1.Deployment) bool {
 		return lo.EveryBy(conds, func(cond appsv1.DeploymentCondition) bool {
-			return lo.ContainsBy(deployment.Status.Conditions, func(c appsv1.DeploymentCondition) bool {
+			if !lo.ContainsBy(deployment.Status.Conditions, func(c appsv1.DeploymentCondition) bool {
 				return c.Type == cond.Type &&
 					c.Status == cond.Status &&
 					c.Reason == cond.Reason
-			})
+			}) {
+				t.Logf("Deployment %s/%s does not have condition %#v", deployment.Namespace, deployment.Name, cond)
+				t.Logf("Deployment %s/%s current status: %#v", deployment.Namespace, deployment.Name, deployment.Status)
+				return false
+			}
+			return true
 		})
 	}
 }
 
-func waitForOperatorDeployment(ctx context.Context, k8sClient *kubernetes.Clientset, opts ...deploymentAssertOptions) error {
-outer:
+func waitForOperatorDeployment(
+	t *testing.T,
+	ctx context.Context,
+	ns string,
+	k8sClient *kubernetes.Clientset,
+	waitTime time.Duration,
+	opts ...deploymentAssertOptions,
+) error {
+	t.Helper()
+
+	timer := time.NewTimer(waitTime)
+	defer timer.Stop()
+	pollTimer := time.NewTicker(time.Second)
+	defer pollTimer.Stop()
+
 	for {
 		select {
+		case <-timer.C:
+			logOperatorPodLogs(t, ctx, k8sClient, ns)
+			return fmt.Errorf("timed out waiting for operator deployment in namespace %s", ns)
 		case <-ctx.Done():
+			logOperatorPodLogs(t, context.Background(), k8sClient, ns)
 			return ctx.Err()
-		default:
-			deployment, err := k8sClient.AppsV1().Deployments("kong-system").Get(ctx, "gateway-operator-controller-manager", metav1.GetOptions{})
+		case <-pollTimer.C:
+			listOpts := metav1.ListOptions{
+				// NOTE: This is a common label used by:
+				// - kustomize https://github.com/Kong/gateway-operator/blob/f98ef9358078ac100e143ab677a9ca836d0222a0/config/manager/manager.yaml#L15
+				// - helm https://github.com/Kong/charts/blob/4968b34ae7c252ab056b37cc137eaeb7a071e101/charts/gateway-operator/templates/deployment.yaml#L5-L6
+				//
+				// As long as kustomize is used for tests let's use this label selector.
+				LabelSelector: "app.kubernetes.io/name=gateway-operator",
+			}
+			deploymentList, err := k8sClient.AppsV1().Deployments(ns).List(ctx, listOpts)
 			if err != nil {
 				return err
 			}
+			if len(deploymentList.Items) == 0 {
+				t.Logf("No operator deployment found in namespace %s", ns)
+				continue
+			}
+
+			deployment := &deploymentList.Items[0]
+
 			if deployment.Status.AvailableReplicas <= 0 {
+				t.Logf("Deployment %s/%s has no AvailableReplicas", ns, deployment.Name)
 				continue
 			}
 
 			for _, opt := range opts {
 				if !opt(deployment) {
-					continue outer
+					continue
 				}
 			}
 			return nil
 		}
 	}
 }
 
+func logOperatorPodLogs(t *testing.T, ctx context.Context, k8sClient *kubernetes.Clientset, ns string) {
+	t.Helper()
+
+	pods, err := k8sClient.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{
+		LabelSelector: "app.kubernetes.io/name=gateway-operator",
+	})
+	if err != nil {
+		t.Logf("Failed to list operator pods in namespace %s: %v", ns, err)
+		return
+	}
+
+	if len(pods.Items) == 0 {
+		t.Logf("No operator pod found in namespace %s", ns)
+		return
+	}
+
+	result := k8sClient.CoreV1().Pods(ns).GetLogs(pods.Items[0].Name, &corev1.PodLogOptions{
+		Container:                    "manager",
+		InsecureSkipTLSVerifyBackend: true,
+	}).Do(ctx)
+
+	if result.Error() != nil {
+		t.Logf("Failed to get logs from operator pod %s/%s: %v", ns, pods.Items[0].Name, result.Error())
+		return
+	}
+
+	b, err := result.Raw()
+	if err != nil {
+		t.Logf("Failed to read logs from operator pod %s/%s: %v", ns, pods.Items[0].Name, err)
+		return
+	}
+
+	t.Logf("Operator pod logs:\n%s", string(b))
+}
+
 func waitForOperatorWebhookEventually(t *testing.T, ctx context.Context, k8sClient *kubernetes.Clientset) func() bool {
 	return func() bool {
 		if err := waitForOperatorWebhook(ctx, k8sClient); err != nil {