diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 43ccfa9e0..6421eb12b 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -12,6 +12,7 @@ metadata: namespace: system labels: control-plane: controller-manager + app.kubernetes.io/name: gateway-operator spec: strategy: type: Recreate diff --git a/test/e2e/environment.go b/test/e2e/environment.go index 63e99422c..ea354917c 100644 --- a/test/e2e/environment.go +++ b/test/e2e/environment.go @@ -116,6 +116,11 @@ var loggerOnce sync.Once // When running with helm caller is responsible for cleaning up the environment. func CreateEnvironment(t *testing.T, ctx context.Context, opts ...TestEnvOption) TestEnvironment { t.Helper() + + const ( + waitTime = 1 * time.Minute + ) + var opt testEnvOptions for _, o := range opts { o(&opt) @@ -230,7 +235,7 @@ func CreateEnvironment(t *testing.T, ctx context.Context, opts ...TestEnvOption) require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kustomizeDir.Tests(), "--server-side")) t.Log("waiting for operator deployment to complete") - require.NoError(t, waitForOperatorDeployment(ctx, "kong-system", clients.K8sClient)) + require.NoError(t, waitForOperatorDeployment(t, ctx, "kong-system", clients.K8sClient, waitTime)) t.Log("waiting for operator webhook service to be connective") require.Eventually(t, waitForOperatorWebhookEventually(t, ctx, clients.K8sClient), @@ -280,26 +285,63 @@ func cleanupEnvironment(t *testing.T, ctx context.Context, env environments.Envi type deploymentAssertOptions func(*appsv1.Deployment) bool -func deploymentAssertConditions(conds ...appsv1.DeploymentCondition) deploymentAssertOptions { +type TestingT interface { + assert.TestingT + require.TestingT + Log(args ...interface{}) + Logf(format string, args ...interface{}) + Helper() +} + +func deploymentAssertConditions(t TestingT, conds ...appsv1.DeploymentCondition) deploymentAssertOptions { + t.Helper() + return func(deployment *appsv1.Deployment) bool { return lo.EveryBy(conds, func(cond appsv1.DeploymentCondition) bool { - return lo.ContainsBy(deployment.Status.Conditions, func(c appsv1.DeploymentCondition) bool { + if !lo.ContainsBy(deployment.Status.Conditions, func(c appsv1.DeploymentCondition) bool { return c.Type == cond.Type && c.Status == cond.Status && c.Reason == cond.Reason - }) + }) { + t.Logf("Deployment %s/%s does not have condition %#v", deployment.Namespace, deployment.Name, cond) + t.Logf("Deployment %s/%s current status: %#v", deployment.Namespace, deployment.Name, deployment.Status) + return false + } + return true }) } } -func waitForOperatorDeployment(ctx context.Context, ns string, k8sClient *kubernetes.Clientset, opts ...deploymentAssertOptions) error { -outer: +func waitForOperatorDeployment( + t TestingT, + ctx context.Context, + ns string, + k8sClient *kubernetes.Clientset, + waitTime time.Duration, + opts ...deploymentAssertOptions, +) error { + t.Helper() + + timer := time.NewTimer(waitTime) + defer timer.Stop() + pollTimer := time.NewTicker(time.Second) + defer pollTimer.Stop() + for { select { + case <-timer.C: + logOperatorPodLogs(t, ctx, k8sClient, ns) + return fmt.Errorf("timed out waiting for operator deployment in namespace %s", ns) case <-ctx.Done(): + logOperatorPodLogs(t, context.Background(), k8sClient, ns) return ctx.Err() - default: + case <-pollTimer.C: listOpts := metav1.ListOptions{ + // NOTE: This is a common label used by: + // - kustomize https://github.com/Kong/gateway-operator/blob/f98ef9358078ac100e143ab677a9ca836d0222a0/config/manager/manager.yaml#L15 + // - helm https://github.com/Kong/charts/blob/4968b34ae7c252ab056b37cc137eaeb7a071e101/charts/gateway-operator/templates/deployment.yaml#L5-L6 + // + // As long as kustomize is used for tests let's use this label selector. LabelSelector: "app.kubernetes.io/name=gateway-operator", } deploymentList, err := k8sClient.AppsV1().Deployments(ns).List(ctx, listOpts) @@ -307,18 +349,21 @@ outer: return err } if len(deploymentList.Items) == 0 { + t.Logf("No operator deployment found in namespace %s", ns) continue } deployment := &deploymentList.Items[0] if deployment.Status.AvailableReplicas <= 0 { + t.Logf("Deployment %s/%s has no AvailableReplicas", ns, deployment.Name) + t.Logf("Deployment %s/%s status: %#v", ns, deployment.Name, deployment.Status) continue } for _, opt := range opts { if !opt(deployment) { - continue outer + continue } } return nil @@ -326,6 +371,41 @@ outer: } } +func logOperatorPodLogs(t TestingT, ctx context.Context, k8sClient *kubernetes.Clientset, ns string) { + t.Helper() + + pods, err := k8sClient.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name=gateway-operator", + }) + if err != nil { + t.Logf("Failed to list operator pods in namespace %s: %v", ns, err) + return + } + + if len(pods.Items) == 0 { + t.Logf("No operator pod found in namespace %s", ns) + return + } + + result := k8sClient.CoreV1().Pods(ns).GetLogs(pods.Items[0].Name, &corev1.PodLogOptions{ + Container: "manager", + InsecureSkipTLSVerifyBackend: true, + }).Do(ctx) + + if result.Error() != nil { + t.Logf("Failed to get logs from operator pod %s/%s: %v", ns, pods.Items[0].Name, result.Error()) + return + } + + b, err := result.Raw() + if err != nil { + t.Logf("Failed to read logs from operator pod %s/%s: %v", ns, pods.Items[0].Name, err) + return + } + + t.Logf("Operator pod logs:\n%s", string(b)) +} + func waitForOperatorWebhookEventually(t *testing.T, ctx context.Context, k8sClient *kubernetes.Clientset) func() bool { return func() bool { if err := waitForOperatorWebhook(ctx, k8sClient); err != nil { diff --git a/test/e2e/test_helm_install_upgrade.go b/test/e2e/test_helm_install_upgrade.go index 8fa13b85e..95cd2c2c3 100644 --- a/test/e2e/test_helm_install_upgrade.go +++ b/test/e2e/test_helm_install_upgrade.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" "testing" + "time" "github.com/gruntwork-io/terratest/modules/helm" "github.com/gruntwork-io/terratest/modules/k8s" @@ -24,6 +25,8 @@ func TestUpgrade(t *testing.T) { // Rel: https://github.com/Kong/charts/tree/main/charts/gateway-operator chart = "kong/gateway-operator" image = "docker.io/kong/gateway-operator-oss" + + waitTime = 3 * time.Minute ) ctx, cancel := context.WithCancel(context.Background()) @@ -59,24 +62,33 @@ func TestUpgrade(t *testing.T) { }, } - var currentTag string + var ( + currentRepository string + currentTag string + ) if imageLoad != "" { t.Logf("KONG_TEST_GATEWAY_OPERATOR_IMAGE_LOAD set to %q", imageLoad) - currentTag = vFromImage(t, imageLoad) + currentRepository, currentTag = splitRepoVersionFromImage(t, imageLoad) } else if imageOverride != "" { t.Logf("KONG_TEST_GATEWAY_OPERATOR_IMAGE_OVERRIDE set to %q", imageOverride) - currentTag = vFromImage(t, imageOverride) + currentRepository, currentTag = splitRepoVersionFromImage(t, imageOverride) } for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { - var tag string + var ( + tag string + targetRepository = image + ) if tc.upgradeToCurrent { if currentTag == "" { - t.Skipf("No KONG_TEST_GATEWAY_OPERATOR_IMAGE_OVERRIDE nor KONG_TEST_GATEWAY_OPERATOR_IMAGE_LOAD" + - " env specified. Please specify the image to upgrade to in order to run this test.") + t.Skip( + "No KONG_TEST_GATEWAY_OPERATOR_IMAGE_OVERRIDE nor KONG_TEST_GATEWAY_OPERATOR_IMAGE_LOAD env specified. " + + "Please specify the image to upgrade to in order to run this test.", + ) } tag = currentTag + targetRepository = currentRepository } else { tag = tc.toVersion } @@ -108,16 +120,18 @@ func TestUpgrade(t *testing.T) { } }) - require.NoError(t, waitForOperatorDeployment(ctx, e.Namespace.Name, e.Clients.K8sClient, - deploymentAssertConditions(deploymentReadyConditions()...), + require.NoError(t, waitForOperatorDeployment(t, ctx, e.Namespace.Name, e.Clients.K8sClient, waitTime, + deploymentAssertConditions(t, deploymentReadyConditions()...), )) opts.SetValues["image.tag"] = tag + opts.SetValues["image.repository"] = targetRepository require.NoError(t, helm.UpgradeE(t, opts, chart, releaseName)) - require.NoError(t, waitForOperatorDeployment(ctx, e.Namespace.Name, e.Clients.K8sClient, - deploymentAssertConditions(deploymentReadyConditions()...), - )) + require.NoError(t, waitForOperatorDeployment(t, ctx, e.Namespace.Name, e.Clients.K8sClient, waitTime, + deploymentAssertConditions(t, deploymentReadyConditions()...), + ), + ) }) } } @@ -137,10 +151,10 @@ func deploymentReadyConditions() []appsv1.DeploymentCondition { } } -func vFromImage(t *testing.T, image string) string { +func splitRepoVersionFromImage(t *testing.T, image string) (string, string) { splitImage := strings.Split(image, ":") if len(splitImage) != 2 { t.Fatalf("image %q does not contain a tag", image) } - return splitImage[1] + return splitImage[0], splitImage[1] } diff --git a/test/e2e/test_upgrade.go b/test/e2e/test_upgrade.go index b6129871d..32576d52a 100644 --- a/test/e2e/test_upgrade.go +++ b/test/e2e/test_upgrade.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "testing" + "time" "github.com/kong/kubernetes-testing-framework/pkg/clusters" "github.com/stretchr/testify/require" @@ -76,14 +77,18 @@ func testManifestsUpgrade( ctx context.Context, testParams upgradeTestParams, ) { + const ( + waitTime = 1 * time.Minute + ) + e := CreateEnvironment(t, ctx, WithOperatorImage(testParams.fromImage), WithInstallViaKustomize()) kustomizationDir := PrepareKustomizeDir(t, testParams.toImage) t.Logf("deploying operator %q to test cluster %q via kustomize", testParams.toImage, e.Environment.Name()) require.NoError(t, clusters.KustomizeDeployForCluster(ctx, e.Environment.Cluster(), kustomizationDir.Tests(), "--server-side", "-v5")) t.Log("waiting for operator deployment to complete") - require.NoError(t, waitForOperatorDeployment(ctx, "kong-system", e.Clients.K8sClient, - deploymentAssertConditions( + require.NoError(t, waitForOperatorDeployment(t, ctx, "kong-system", e.Clients.K8sClient, waitTime, + deploymentAssertConditions(t, appsv1.DeploymentCondition{ Reason: "NewReplicaSetAvailable", Status: "True",