From e06832fce17b8521884461f5757ee9eec4cdd619 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Mon, 29 Apr 2024 13:39:13 +0200 Subject: [PATCH] chore(tests): add logs in e2e tests --- config/manager/manager.yaml | 1 + test/e2e/environment.go | 87 +++++++- test/e2e/test_helm_install_upgrade.go | 293 ++++++++++++++++++++++++-- test/e2e/test_upgrade.go | 9 +- 4 files changed, 360 insertions(+), 30 deletions(-) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 43ccfa9e0..6421eb12b 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -12,6 +12,7 @@ metadata: namespace: system labels: control-plane: controller-manager + app.kubernetes.io/name: gateway-operator spec: strategy: type: Recreate diff --git a/test/e2e/environment.go b/test/e2e/environment.go index 63e99422c..d25536192 100644 --- a/test/e2e/environment.go +++ b/test/e2e/environment.go @@ -116,6 +116,11 @@ var loggerOnce sync.Once // When running with helm caller is responsible for cleaning up the environment. func CreateEnvironment(t *testing.T, ctx context.Context, opts ...TestEnvOption) TestEnvironment { t.Helper() + + const ( + waitTime = 1 * time.Minute + ) + var opt testEnvOptions for _, o := range opts { o(&opt) @@ -230,7 +235,7 @@ func CreateEnvironment(t *testing.T, ctx context.Context, opts ...TestEnvOption) require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kustomizeDir.Tests(), "--server-side")) t.Log("waiting for operator deployment to complete") - require.NoError(t, waitForOperatorDeployment(ctx, "kong-system", clients.K8sClient)) + require.NoError(t, waitForOperatorDeployment(t, ctx, "kong-system", clients.K8sClient, waitTime)) t.Log("waiting for operator webhook service to be connective") require.Eventually(t, waitForOperatorWebhookEventually(t, ctx, clients.K8sClient), @@ -280,26 +285,55 @@ func cleanupEnvironment(t *testing.T, ctx context.Context, env environments.Envi type deploymentAssertOptions func(*appsv1.Deployment) bool -func deploymentAssertConditions(conds ...appsv1.DeploymentCondition) deploymentAssertOptions { +func deploymentAssertConditions(t *testing.T, conds ...appsv1.DeploymentCondition) deploymentAssertOptions { + t.Helper() + return func(deployment *appsv1.Deployment) bool { return lo.EveryBy(conds, func(cond appsv1.DeploymentCondition) bool { - return lo.ContainsBy(deployment.Status.Conditions, func(c appsv1.DeploymentCondition) bool { + if !lo.ContainsBy(deployment.Status.Conditions, func(c appsv1.DeploymentCondition) bool { return c.Type == cond.Type && c.Status == cond.Status && c.Reason == cond.Reason - }) + }) { + t.Logf("Deployment %s/%s does not have condition %#v", deployment.Namespace, deployment.Name, cond) + t.Logf("Deployment %s/%s current status: %#v", deployment.Namespace, deployment.Name, deployment.Status) + return false + } + return true }) } } -func waitForOperatorDeployment(ctx context.Context, ns string, k8sClient *kubernetes.Clientset, opts ...deploymentAssertOptions) error { -outer: +func waitForOperatorDeployment( + t *testing.T, + ctx context.Context, + ns string, + k8sClient *kubernetes.Clientset, + waitTime time.Duration, + opts ...deploymentAssertOptions, +) error { + t.Helper() + + timer := time.NewTimer(waitTime) + defer timer.Stop() + pollTimer := time.NewTicker(time.Second) + defer pollTimer.Stop() + for { select { + case <-timer.C: + logOperatorPodLogs(t, ctx, k8sClient, ns) + return fmt.Errorf("timed out waiting for operator deployment in namespace %s", ns) case <-ctx.Done(): + logOperatorPodLogs(t, context.Background(), k8sClient, ns) return ctx.Err() - default: + case <-pollTimer.C: listOpts := metav1.ListOptions{ + // NOTE: This is a common label used by: + // - kustomize https://github.com/Kong/gateway-operator/blob/f98ef9358078ac100e143ab677a9ca836d0222a0/config/manager/manager.yaml#L15 + // - helm https://github.com/Kong/charts/blob/4968b34ae7c252ab056b37cc137eaeb7a071e101/charts/gateway-operator/templates/deployment.yaml#L5-L6 + // + // As long as kustomize is used for tests let's use this label selector. LabelSelector: "app.kubernetes.io/name=gateway-operator", } deploymentList, err := k8sClient.AppsV1().Deployments(ns).List(ctx, listOpts) @@ -307,18 +341,20 @@ outer: return err } if len(deploymentList.Items) == 0 { + t.Logf("No operator deployment found in namespace %s", ns) continue } deployment := &deploymentList.Items[0] if deployment.Status.AvailableReplicas <= 0 { + t.Logf("Deployment %s/%s has no AvailableReplicas", ns, deployment.Name) continue } for _, opt := range opts { if !opt(deployment) { - continue outer + continue } } return nil @@ -326,6 +362,41 @@ outer: } } +func logOperatorPodLogs(t *testing.T, ctx context.Context, k8sClient *kubernetes.Clientset, ns string) { + t.Helper() + + pods, err := k8sClient.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name=gateway-operator", + }) + if err != nil { + t.Logf("Failed to list operator pods in namespace %s: %v", ns, err) + return + } + + if len(pods.Items) == 0 { + t.Logf("No operator pod found in namespace %s", ns) + return + } + + result := k8sClient.CoreV1().Pods(ns).GetLogs(pods.Items[0].Name, &corev1.PodLogOptions{ + Container: "manager", + InsecureSkipTLSVerifyBackend: true, + }).Do(ctx) + + if result.Error() != nil { + t.Logf("Failed to get logs from operator pod %s/%s: %v", ns, pods.Items[0].Name, result.Error()) + return + } + + b, err := result.Raw() + if err != nil { + t.Logf("Failed to read logs from operator pod %s/%s: %v", ns, pods.Items[0].Name, err) + return + } + + t.Logf("Operator pod logs:\n%s", string(b)) +} + func waitForOperatorWebhookEventually(t *testing.T, ctx context.Context, k8sClient *kubernetes.Clientset) func() bool { return func() bool { if err := waitForOperatorWebhook(ctx, k8sClient); err != nil { diff --git a/test/e2e/test_helm_install_upgrade.go b/test/e2e/test_helm_install_upgrade.go index 8fa13b85e..e17d0b3b3 100644 --- a/test/e2e/test_helm_install_upgrade.go +++ b/test/e2e/test_helm_install_upgrade.go @@ -5,14 +5,24 @@ import ( "fmt" "strings" "testing" + "time" "github.com/gruntwork-io/terratest/modules/helm" "github.com/gruntwork-io/terratest/modules/k8s" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" - "github.com/kong/gateway-operator/pkg/utils/test" + operatorv1beta1 "github.com/kong/gateway-operator/api/v1beta1" + "github.com/kong/gateway-operator/pkg/consts" + "github.com/kong/gateway-operator/pkg/utils/gateway" + testutils "github.com/kong/gateway-operator/pkg/utils/test" + "github.com/kong/gateway-operator/pkg/vars" + "github.com/kong/gateway-operator/test/helpers" ) func init() { @@ -24,6 +34,8 @@ func TestUpgrade(t *testing.T) { // Rel: https://github.com/Kong/charts/tree/main/charts/gateway-operator chart = "kong/gateway-operator" image = "docker.io/kong/gateway-operator-oss" + + waitTime = 3 * time.Minute ) ctx, cancel := context.WithCancel(context.Background()) @@ -33,12 +45,19 @@ func TestUpgrade(t *testing.T) { // and dumping diagnostics if the test fails. e := CreateEnvironment(t, ctx) + // assertion is run after the upgrade to assert the state of the resources in the cluster. + type assertion struct { + Name string + Func func(*assert.CollectT, *testutils.K8sClients) + } + testcases := []struct { name string fromVersion string toVersion string + objectsToDeploy []client.Object upgradeToCurrent bool - assertions func(*testing.T, *test.K8sClients) + assertions []assertion }{ // NOTE: We do not support versions earlier than 1.2 with the helm chart. // The initial version of the chart contained CRDs from KGO 1.2. which @@ -46,37 +65,207 @@ func TestUpgrade(t *testing.T) { // automatically (without manually deleting the CRDs). { name: "upgrade from 1.2.0 to 1.2.3", - fromVersion: "1.2.1", + fromVersion: "1.2.0", toVersion: "1.2.3", - assertions: func(t *testing.T, c *test.K8sClients) { - // TODO + objectsToDeploy: []client.Object{ + &operatorv1beta1.GatewayConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gwconf-upgrade-120-123", + }, + Spec: baseGatewayConfigurationSpec(), + }, + &gatewayv1.GatewayClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gwclass-upgrade-120-123", + }, + Spec: gatewayv1.GatewayClassSpec{ + ParametersRef: &gatewayv1.ParametersReference{ + Group: gatewayv1.Group(operatorv1beta1.SchemeGroupVersion.Group), + Kind: gatewayv1.Kind("GatewayConfiguration"), + Namespace: (*gatewayv1.Namespace)(&e.Namespace.Name), + Name: "gwconf-upgrade-120-123", + }, + ControllerName: gatewayv1.GatewayController(vars.ControllerName()), + }, + }, + &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "gw-upgrade-120-123-", + Labels: map[string]string{ + "gw-upgrade-120-123": "true", + }, + }, + Spec: gatewayv1.GatewaySpec{ + GatewayClassName: gatewayv1.ObjectName("gwclass-upgrade-120-123"), + Listeners: []gatewayv1.Listener{{ + Name: "http", + Protocol: gatewayv1.HTTPProtocolType, + Port: gatewayv1.PortNumber(80), + }}, + }, + }, + }, + assertions: []assertion{ + { + Name: "gateway is programmed", + Func: func(c *assert.CollectT, cl *testutils.K8sClients) { + gws, err := cl.GatewayClient.GatewayV1().Gateways(e.Namespace.Name).List(ctx, metav1.ListOptions{ + LabelSelector: "gw-upgrade-120-123=true", + }) + require.NoError(c, err) + require.Len(c, gws.Items, 1) + gw := &gws.Items[0] + assert.True(c, gateway.IsProgrammed(gw)) + assert.True(c, gateway.AreListenersProgrammed(gw)) + }, + }, }, }, { name: "upgrade from 1.2.3 to current", fromVersion: "1.2.3", upgradeToCurrent: true, + objectsToDeploy: []client.Object{ + &operatorv1beta1.GatewayConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gwconf-upgrade-123-current", + }, + Spec: baseGatewayConfigurationSpec(), + }, + &gatewayv1.GatewayClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gwclass-upgrade-123-current", + }, + Spec: gatewayv1.GatewayClassSpec{ + ParametersRef: &gatewayv1.ParametersReference{ + Group: gatewayv1.Group(operatorv1beta1.SchemeGroupVersion.Group), + Kind: gatewayv1.Kind("GatewayConfiguration"), + Namespace: (*gatewayv1.Namespace)(&e.Namespace.Name), + Name: "gwconf-upgrade-123-current", + }, + ControllerName: gatewayv1.GatewayController(vars.ControllerName()), + }, + }, + &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "gw-upgrade-123-current-", + Labels: map[string]string{ + "gw-upgrade-123-current": "true", + }, + }, + Spec: gatewayv1.GatewaySpec{ + GatewayClassName: gatewayv1.ObjectName("gwclass-upgrade-123-current"), + Listeners: []gatewayv1.Listener{{ + Name: "http", + Protocol: gatewayv1.HTTPProtocolType, + Port: gatewayv1.PortNumber(80), + }}, + }, + }, + }, + assertions: []assertion{ + { + Name: "gateway is programmed", + Func: func(c *assert.CollectT, cl *testutils.K8sClients) { + gws, err := cl.GatewayClient.GatewayV1().Gateways(e.Namespace.Name).List(ctx, metav1.ListOptions{ + LabelSelector: "gw-upgrade-123-current=true", + }) + require.NoError(c, err) + require.Len(c, gws.Items, 1) + gw := &gws.Items[0] + assert.True(c, gateway.IsProgrammed(gw)) + assert.True(c, gateway.AreListenersProgrammed(gw)) + }, + }, + }, + }, + { + name: "upgrade from nightly to current", + fromVersion: "nightly", + upgradeToCurrent: true, + objectsToDeploy: []client.Object{ + &operatorv1beta1.GatewayConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gwconf-upgrade-nightly-current", + }, + Spec: baseGatewayConfigurationSpec(), + }, + &gatewayv1.GatewayClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gwclass-upgrade-nightly-to-current", + }, + Spec: gatewayv1.GatewayClassSpec{ + ParametersRef: &gatewayv1.ParametersReference{ + Group: gatewayv1.Group(operatorv1beta1.SchemeGroupVersion.Group), + Kind: gatewayv1.Kind("GatewayConfiguration"), + Namespace: (*gatewayv1.Namespace)(&e.Namespace.Name), + Name: "gwconf-upgrade-nightly-to-current", + }, + ControllerName: gatewayv1.GatewayController(vars.ControllerName()), + }, + }, + &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "gw-upgrade-nightly-to-current-", + Labels: map[string]string{ + "gw-upgrade-nightly-to-current": "true", + }, + }, + Spec: gatewayv1.GatewaySpec{ + GatewayClassName: gatewayv1.ObjectName("gwclass-upgrade-nightly-to-current"), + Listeners: []gatewayv1.Listener{{ + Name: "http", + Protocol: gatewayv1.HTTPProtocolType, + Port: gatewayv1.PortNumber(80), + }}, + }, + }, + }, + assertions: []assertion{ + { + Name: "gateway is programmed", + Func: func(c *assert.CollectT, cl *testutils.K8sClients) { + gws, err := cl.GatewayClient.GatewayV1().Gateways(e.Namespace.Name).List(ctx, metav1.ListOptions{ + LabelSelector: "gw-upgrade-nightly-to-current=true", + }) + require.NoError(c, err) + require.Len(c, gws.Items, 1) + gw := &gws.Items[0] + assert.True(c, gateway.IsProgrammed(gw)) + assert.True(c, gateway.AreListenersProgrammed(gw)) + }, + }, + }, }, } - var currentTag string + var ( + currentRepository string + currentTag string + ) if imageLoad != "" { t.Logf("KONG_TEST_GATEWAY_OPERATOR_IMAGE_LOAD set to %q", imageLoad) - currentTag = vFromImage(t, imageLoad) + currentRepository, currentTag = splitRepoVersionFromImage(t, imageLoad) } else if imageOverride != "" { t.Logf("KONG_TEST_GATEWAY_OPERATOR_IMAGE_OVERRIDE set to %q", imageOverride) - currentTag = vFromImage(t, imageOverride) + currentRepository, currentTag = splitRepoVersionFromImage(t, imageOverride) } for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { - var tag string + var ( + tag string + targetRepository = image + ) if tc.upgradeToCurrent { if currentTag == "" { - t.Skipf("No KONG_TEST_GATEWAY_OPERATOR_IMAGE_OVERRIDE nor KONG_TEST_GATEWAY_OPERATOR_IMAGE_LOAD" + - " env specified. Please specify the image to upgrade to in order to run this test.") + t.Skip( + "No KONG_TEST_GATEWAY_OPERATOR_IMAGE_OVERRIDE nor KONG_TEST_GATEWAY_OPERATOR_IMAGE_LOAD env specified. " + + "Please specify the image to upgrade to in order to run this test.", + ) } tag = currentTag + targetRepository = currentRepository } else { tag = tc.toVersion } @@ -87,8 +276,10 @@ func TestUpgrade(t *testing.T) { } releaseName := strings.ReplaceAll(fmt.Sprintf("kgo-%s-to-%s", tc.fromVersion, tagInReleaseName), ".", "-") values := map[string]string{ - "image.tag": tc.fromVersion, - "image.repository": image, + "image.tag": tc.fromVersion, + "image.repository": image, + "readinessProbe.initialDelaySeconds": "1", + "readinessProbe.periodSeconds": "1", } opts := &helm.Options{ @@ -108,16 +299,35 @@ func TestUpgrade(t *testing.T) { } }) - require.NoError(t, waitForOperatorDeployment(ctx, e.Namespace.Name, e.Clients.K8sClient, - deploymentAssertConditions(deploymentReadyConditions()...), + // Deploy the objects that should be present before the upgrade. + cl := client.NewNamespacedClient(e.Clients.MgrClient, e.Namespace.Name) + for _, obj := range tc.objectsToDeploy { + require.NoError(t, cl.Create(ctx, obj)) + t.Cleanup(func() { + require.NoError(t, client.IgnoreNotFound(cl.Delete(ctx, obj))) + }) + } + + require.NoError(t, waitForOperatorDeployment(t, ctx, e.Namespace.Name, e.Clients.K8sClient, waitTime, + deploymentAssertConditions(t, deploymentReadyConditions()...), )) opts.SetValues["image.tag"] = tag + opts.SetValues["image.repository"] = targetRepository require.NoError(t, helm.UpgradeE(t, opts, chart, releaseName)) - require.NoError(t, waitForOperatorDeployment(ctx, e.Namespace.Name, e.Clients.K8sClient, - deploymentAssertConditions(deploymentReadyConditions()...), - )) + require.NoError(t, waitForOperatorDeployment(t, ctx, e.Namespace.Name, e.Clients.K8sClient, waitTime, + deploymentAssertConditions(t, deploymentReadyConditions()...), + ), + ) + + for _, assertion := range tc.assertions { + t.Run(assertion.Name, func(t *testing.T) { + require.EventuallyWithT(t, func(c *assert.CollectT) { + assertion.Func(c, e.Clients) + }, waitTime, time.Second) + }) + } }) } } @@ -137,10 +347,53 @@ func deploymentReadyConditions() []appsv1.DeploymentCondition { } } -func vFromImage(t *testing.T, image string) string { +func splitRepoVersionFromImage(t *testing.T, image string) (string, string) { splitImage := strings.Split(image, ":") if len(splitImage) != 2 { t.Fatalf("image %q does not contain a tag", image) } - return splitImage[1] + return splitImage[0], splitImage[1] +} + +func baseGatewayConfigurationSpec() operatorv1beta1.GatewayConfigurationSpec { + return operatorv1beta1.GatewayConfigurationSpec{ + DataPlaneOptions: &operatorv1beta1.GatewayConfigDataPlaneOptions{ + Deployment: operatorv1beta1.DataPlaneDeploymentOptions{ + DeploymentOptions: operatorv1beta1.DeploymentOptions{ + PodTemplateSpec: &corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: consts.DataPlaneProxyContainerName, + Image: helpers.GetDefaultDataPlaneImage(), + ReadinessProbe: &corev1.Probe{ + InitialDelaySeconds: 1, + PeriodSeconds: 1, + }, + }, + }, + }, + }, + }, + }, + }, + ControlPlaneOptions: &operatorv1beta1.ControlPlaneOptions{ + Deployment: operatorv1beta1.ControlPlaneDeploymentOptions{ + PodTemplateSpec: &corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: consts.ControlPlaneControllerContainerName, + Image: consts.DefaultControlPlaneImage, + ReadinessProbe: &corev1.Probe{ + InitialDelaySeconds: 1, + PeriodSeconds: 1, + }, + }, + }, + }, + }, + }, + }, + } } diff --git a/test/e2e/test_upgrade.go b/test/e2e/test_upgrade.go index b6129871d..f945eeb59 100644 --- a/test/e2e/test_upgrade.go +++ b/test/e2e/test_upgrade.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "testing" + "time" "github.com/kong/kubernetes-testing-framework/pkg/clusters" "github.com/stretchr/testify/require" @@ -76,14 +77,18 @@ func testManifestsUpgrade( ctx context.Context, testParams upgradeTestParams, ) { + const ( + waitTime = 3 * time.Minute + ) + e := CreateEnvironment(t, ctx, WithOperatorImage(testParams.fromImage), WithInstallViaKustomize()) kustomizationDir := PrepareKustomizeDir(t, testParams.toImage) t.Logf("deploying operator %q to test cluster %q via kustomize", testParams.toImage, e.Environment.Name()) require.NoError(t, clusters.KustomizeDeployForCluster(ctx, e.Environment.Cluster(), kustomizationDir.Tests(), "--server-side", "-v5")) t.Log("waiting for operator deployment to complete") - require.NoError(t, waitForOperatorDeployment(ctx, "kong-system", e.Clients.K8sClient, - deploymentAssertConditions( + require.NoError(t, waitForOperatorDeployment(t, ctx, "kong-system", e.Clients.K8sClient, waitTime, + deploymentAssertConditions(t, appsv1.DeploymentCondition{ Reason: "NewReplicaSetAvailable", Status: "True",