chore(tests): add logs in e2e tests

config/manager/manager.yaml

@@ -12,6 +12,7 @@ metadata:
   namespace: system
   labels:
     control-plane: controller-manager
+    app.kubernetes.io/name: gateway-operator
 spec:
   strategy:
     type: Recreate

test/e2e/environment.go

@@ -116,6 +116,11 @@ var loggerOnce sync.Once
 // When running with helm caller is responsible for cleaning up the environment.
 func CreateEnvironment(t *testing.T, ctx context.Context, opts ...TestEnvOption) TestEnvironment {
 	t.Helper()
+
+	const (
+		waitTime = 1 * time.Minute
+	)
+
 	var opt testEnvOptions
 	for _, o := range opts {
 		o(&opt)
@@ -230,7 +235,7 @@ func CreateEnvironment(t *testing.T, ctx context.Context, opts ...TestEnvOption)
 		require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kustomizeDir.Tests(), "--server-side"))
 
 		t.Log("waiting for operator deployment to complete")
-		require.NoError(t, waitForOperatorDeployment(ctx, "kong-system", clients.K8sClient))
+		require.NoError(t, waitForOperatorDeployment(t, ctx, "kong-system", clients.K8sClient, waitTime))
 
 		t.Log("waiting for operator webhook service to be connective")
 		require.Eventually(t, waitForOperatorWebhookEventually(t, ctx, clients.K8sClient),
@@ -280,52 +285,127 @@ func cleanupEnvironment(t *testing.T, ctx context.Context, env environments.Envi
 
 type deploymentAssertOptions func(*appsv1.Deployment) bool
 
-func deploymentAssertConditions(conds ...appsv1.DeploymentCondition) deploymentAssertOptions {
+type TestingT interface {
+	assert.TestingT
+	require.TestingT
+	Log(args ...interface{})
+	Logf(format string, args ...interface{})
+	Helper()
+}
+
+func deploymentAssertConditions(t TestingT, conds ...appsv1.DeploymentCondition) deploymentAssertOptions {
+	t.Helper()
+
 	return func(deployment *appsv1.Deployment) bool {
 		return lo.EveryBy(conds, func(cond appsv1.DeploymentCondition) bool {
-			return lo.ContainsBy(deployment.Status.Conditions, func(c appsv1.DeploymentCondition) bool {
+			if !lo.ContainsBy(deployment.Status.Conditions, func(c appsv1.DeploymentCondition) bool {
 				return c.Type == cond.Type &&
 					c.Status == cond.Status &&
 					c.Reason == cond.Reason
-			})
+			}) {
+				t.Logf("Deployment %s/%s does not have condition %#v", deployment.Namespace, deployment.Name, cond)
+				t.Logf("Deployment %s/%s current status: %#v", deployment.Namespace, deployment.Name, deployment.Status)
+				return false
+			}
+			return true
 		})
 	}
 }
 
-func waitForOperatorDeployment(ctx context.Context, ns string, k8sClient *kubernetes.Clientset, opts ...deploymentAssertOptions) error {
-outer:
+func waitForOperatorDeployment(
+	t TestingT,
+	ctx context.Context,
+	ns string,
+	k8sClient *kubernetes.Clientset,
+	waitTime time.Duration,
+	opts ...deploymentAssertOptions,
+) error {
+	t.Helper()
+
+	timer := time.NewTimer(waitTime)
+	defer timer.Stop()
+	pollTimer := time.NewTicker(time.Second)
+	defer pollTimer.Stop()
+
 	for {
 		select {
+		case <-timer.C:
+			logOperatorPodLogs(t, ctx, k8sClient, ns)
+			return fmt.Errorf("timed out waiting for operator deployment in namespace %s", ns)
 		case <-ctx.Done():
+			logOperatorPodLogs(t, context.Background(), k8sClient, ns)
 			return ctx.Err()
-		default:
+		case <-pollTimer.C:
 			listOpts := metav1.ListOptions{
+				// NOTE: This is a common label used by:
+				// - kustomize https://github.com/Kong/gateway-operator/blob/f98ef9358078ac100e143ab677a9ca836d0222a0/config/manager/manager.yaml#L15
+				// - helm https://github.com/Kong/charts/blob/4968b34ae7c252ab056b37cc137eaeb7a071e101/charts/gateway-operator/templates/deployment.yaml#L5-L6
+				//
+				// As long as kustomize is used for tests let's use this label selector.
 				LabelSelector: "app.kubernetes.io/name=gateway-operator",
 			}
 			deploymentList, err := k8sClient.AppsV1().Deployments(ns).List(ctx, listOpts)
 			if err != nil {
 				return err
 			}
 			if len(deploymentList.Items) == 0 {
+				t.Logf("No operator deployment found in namespace %s", ns)
 				continue
 			}
 
 			deployment := &deploymentList.Items[0]
 
 			if deployment.Status.AvailableReplicas <= 0 {
+				t.Logf("Deployment %s/%s has no AvailableReplicas", ns, deployment.Name)
+				t.Logf("Deployment %s/%s status: %#v", ns, deployment.Name, deployment.Status)
 				continue
 			}
 
 			for _, opt := range opts {
 				if !opt(deployment) {
-					continue outer
+					continue
 				}
 			}
 			return nil
 		}
 	}
 }
 
+func logOperatorPodLogs(t TestingT, ctx context.Context, k8sClient *kubernetes.Clientset, ns string) {
+	t.Helper()
+
+	pods, err := k8sClient.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{
+		LabelSelector: "app.kubernetes.io/name=gateway-operator",
+	})
+	if err != nil {
+		t.Logf("Failed to list operator pods in namespace %s: %v", ns, err)
+		return
+	}
+
+	if len(pods.Items) == 0 {
+		t.Logf("No operator pod found in namespace %s", ns)
+		return
+	}
+
+	result := k8sClient.CoreV1().Pods(ns).GetLogs(pods.Items[0].Name, &corev1.PodLogOptions{
+		Container:                    "manager",
+		InsecureSkipTLSVerifyBackend: true,
+	}).Do(ctx)
+
+	if result.Error() != nil {
+		t.Logf("Failed to get logs from operator pod %s/%s: %v", ns, pods.Items[0].Name, result.Error())
+		return
+	}
+
+	b, err := result.Raw()
+	if err != nil {
+		t.Logf("Failed to read logs from operator pod %s/%s: %v", ns, pods.Items[0].Name, err)
+		return
+	}
+
+	t.Logf("Operator pod logs:\n%s", string(b))
+}
+
 func waitForOperatorWebhookEventually(t *testing.T, ctx context.Context, k8sClient *kubernetes.Clientset) func() bool {
 	return func() bool {
 		if err := waitForOperatorWebhook(ctx, k8sClient); err != nil {

test/e2e/test_helm_install_upgrade.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/gruntwork-io/terratest/modules/helm"
 	"github.com/gruntwork-io/terratest/modules/k8s"
@@ -24,6 +25,8 @@ func TestUpgrade(t *testing.T) {
 		// Rel: https://github.com/Kong/charts/tree/main/charts/gateway-operator
 		chart = "kong/gateway-operator"
 		image = "docker.io/kong/gateway-operator-oss"
+
+		waitTime = 3 * time.Minute
 	)
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -59,24 +62,33 @@ func TestUpgrade(t *testing.T) {
 		},
 	}
 
-	var currentTag string
+	var (
+		currentRepository string
+		currentTag        string
+	)
 	if imageLoad != "" {
 		t.Logf("KONG_TEST_GATEWAY_OPERATOR_IMAGE_LOAD set to %q", imageLoad)
-		currentTag = vFromImage(t, imageLoad)
+		currentRepository, currentTag = splitRepoVersionFromImage(t, imageLoad)
 	} else if imageOverride != "" {
 		t.Logf("KONG_TEST_GATEWAY_OPERATOR_IMAGE_OVERRIDE set to %q", imageOverride)
-		currentTag = vFromImage(t, imageOverride)
+		currentRepository, currentTag = splitRepoVersionFromImage(t, imageOverride)
 	}
 
 	for _, tc := range testcases {
 		t.Run(tc.name, func(t *testing.T) {
-			var tag string
+			var (
+				tag              string
+				targetRepository = image
+			)
 			if tc.upgradeToCurrent {
 				if currentTag == "" {
-					t.Skipf("No KONG_TEST_GATEWAY_OPERATOR_IMAGE_OVERRIDE nor KONG_TEST_GATEWAY_OPERATOR_IMAGE_LOAD" +
-						" env specified. Please specify the image to upgrade to in order to run this test.")
+					t.Skip(
+						"No KONG_TEST_GATEWAY_OPERATOR_IMAGE_OVERRIDE nor KONG_TEST_GATEWAY_OPERATOR_IMAGE_LOAD env specified. " +
+							"Please specify the image to upgrade to in order to run this test.",
+					)
 				}
 				tag = currentTag
+				targetRepository = currentRepository
 			} else {
 				tag = tc.toVersion
 			}
@@ -108,16 +120,18 @@ func TestUpgrade(t *testing.T) {
 				}
 			})
 
-			require.NoError(t, waitForOperatorDeployment(ctx, e.Namespace.Name, e.Clients.K8sClient,
-				deploymentAssertConditions(deploymentReadyConditions()...),
+			require.NoError(t, waitForOperatorDeployment(t, ctx, e.Namespace.Name, e.Clients.K8sClient, waitTime,
+				deploymentAssertConditions(t, deploymentReadyConditions()...),
 			))
 
 			opts.SetValues["image.tag"] = tag
+			opts.SetValues["image.repository"] = targetRepository
 
 			require.NoError(t, helm.UpgradeE(t, opts, chart, releaseName))
-			require.NoError(t, waitForOperatorDeployment(ctx, e.Namespace.Name, e.Clients.K8sClient,
-				deploymentAssertConditions(deploymentReadyConditions()...),
-			))
+			require.NoError(t, waitForOperatorDeployment(t, ctx, e.Namespace.Name, e.Clients.K8sClient, waitTime,
+				deploymentAssertConditions(t, deploymentReadyConditions()...),
+			),
+			)
 		})
 	}
 }
@@ -137,10 +151,10 @@ func deploymentReadyConditions() []appsv1.DeploymentCondition {
 	}
 }
 
-func vFromImage(t *testing.T, image string) string {
+func splitRepoVersionFromImage(t *testing.T, image string) (string, string) {
 	splitImage := strings.Split(image, ":")
 	if len(splitImage) != 2 {
 		t.Fatalf("image %q does not contain a tag", image)
 	}
-	return splitImage[1]
+	return splitImage[0], splitImage[1]
 }

test/e2e/test_upgrade.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"testing"
+	"time"
 
 	"github.com/kong/kubernetes-testing-framework/pkg/clusters"
 	"github.com/stretchr/testify/require"
@@ -76,14 +77,18 @@ func testManifestsUpgrade(
 	ctx context.Context,
 	testParams upgradeTestParams,
 ) {
+	const (
+		waitTime = 1 * time.Minute
+	)
+
 	e := CreateEnvironment(t, ctx, WithOperatorImage(testParams.fromImage), WithInstallViaKustomize())
 
 	kustomizationDir := PrepareKustomizeDir(t, testParams.toImage)
 	t.Logf("deploying operator %q to test cluster %q via kustomize", testParams.toImage, e.Environment.Name())
 	require.NoError(t, clusters.KustomizeDeployForCluster(ctx, e.Environment.Cluster(), kustomizationDir.Tests(), "--server-side", "-v5"))
 	t.Log("waiting for operator deployment to complete")
-	require.NoError(t, waitForOperatorDeployment(ctx, "kong-system", e.Clients.K8sClient,
-		deploymentAssertConditions(
+	require.NoError(t, waitForOperatorDeployment(t, ctx, "kong-system", e.Clients.K8sClient, waitTime,
+		deploymentAssertConditions(t,
 			appsv1.DeploymentCondition{
 				Reason: "NewReplicaSetAvailable",
 				Status: "True",