From a38eee41e163e3177ab85ee7d8c2469dfb4b9fb5 Mon Sep 17 00:00:00 2001 From: Jakub Warczarek Date: Mon, 25 Mar 2024 18:23:54 +0100 Subject: [PATCH] chore(tests): do not rely on Redis in TestTCPIngressTLSPassthrough (#5747) --- test/integration/tcpingress_test.go | 142 ++++++---------------------- 1 file changed, 30 insertions(+), 112 deletions(-) diff --git a/test/integration/tcpingress_test.go b/test/integration/tcpingress_test.go index d239f57550..59d2bc4a0f 100644 --- a/test/integration/tcpingress_test.go +++ b/test/integration/tcpingress_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + "github.com/google/uuid" ktfkong "github.com/kong/kubernetes-testing-framework/pkg/clusters/addons/kong" "github.com/kong/kubernetes-testing-framework/pkg/utils/kubernetes/generators" "github.com/stretchr/testify/assert" @@ -312,122 +313,51 @@ func TestTCPIngressTLSPassthrough(t *testing.T) { ctx := context.Background() ns, cleaner := helpers.Setup(ctx, t, env) - const ( - // Pinned because of - // https://github.com/Kong/kubernetes-ingress-controller/issues/2735#issuecomment-1194376496 breakage. - redisImage = "bitnami/redis:7.0.4-debian-11-r3" - ) - t.Log("setting up the TCPIngress TLS passthrough tests") - testName := "tlspass" - gatewayClient, err := clientset.NewForConfig(env.Cluster().Config()) - require.NoError(t, err) + + const tlsExampleHostname = "tlsroute.kong.example" t.Log("configuring secrets") - redisExampleTLSCert, redisExampleTLSKey := certificate.MustGenerateSelfSignedCertPEMFormat( - certificate.WithCommonName("secure-foo-bar"), certificate.WithDNSNames("redis.example"), + exampleTLSCert, exampleTLSKey := certificate.MustGenerateSelfSignedCertPEMFormat( + certificate.WithCommonName(tlsExampleHostname), certificate.WithDNSNames(tlsExampleHostname), ) - require.NoError(t, err) secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: "certs", + Name: tlsSecretName, Namespace: ns.Name, }, Data: map[string][]byte{ - "tls.crt": redisExampleTLSCert, - "tls.key": redisExampleTLSKey, + "tls.crt": exampleTLSCert, + "tls.key": exampleTLSKey, }, } + k8sClient := env.Cluster().Client() t.Log("deploying secrets") - secret, err = env.Cluster().Client().CoreV1().Secrets(ns.Name).Create(ctx, secret, metav1.CreateOptions{}) + _, err := k8sClient.CoreV1().Secrets(ns.Name).Create(ctx, secret, metav1.CreateOptions{}) assert.NoError(t, err) - t.Log("deploying Redis with certificate") - container := generators.NewContainer(testName, redisImage, 6379) - container.VolumeMounts = []corev1.VolumeMount{ - { - Name: "certificates", - MountPath: "/opt/certs", - }, - } - container.Env = []corev1.EnvVar{ - { - Name: "REDIS_TLS_ENABLED", - Value: "true", - }, - { - Name: "REDIS_TLS_PORT", - Value: "6379", - }, - { - Name: "REDIS_TLS_CA_FILE", - Value: "/opt/certs/tls.crt", - }, - { - Name: "REDIS_TLS_CERT_FILE", - Value: "/opt/certs/tls.crt", - }, - { - Name: "REDIS_TLS_KEY_FILE", - Value: "/opt/certs/tls.key", - }, - { - Name: "REDIS_PASSWORD", - Value: "garbage", - }, - } - deployment := generators.NewDeploymentForContainer(container) - deployment.Spec.Template.Spec.Volumes = []corev1.Volume{ - { - Name: "certificates", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: secret.Name, - }, + t.Log("creating a tcpecho deployment to test TLSRoute traffic routing") + testUUID := uuid.NewString() // go-echo sends a "Running on Pod ." immediately on connecting + deployment := generators.NewDeploymentForContainer(createTLSEchoContainer(tlsEchoPort, testUUID)) + deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: tlsSecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: tlsSecretName, }, }, - } - deployment, err = env.Cluster().Client().AppsV1().Deployments(ns.Name).Create(ctx, deployment, metav1.CreateOptions{}) + }) + deployment, err = k8sClient.AppsV1().Deployments(ns.Name).Create(ctx, deployment, metav1.CreateOptions{}) require.NoError(t, err) cleaner.Add(deployment) - t.Logf("exposing deployment %s via service", deployment.Name) + t.Logf("exposing deployment %s/%s via service", deployment.Namespace, deployment.Name) service := generators.NewServiceForDeployment(deployment, corev1.ServiceTypeLoadBalancer) - service, err = env.Cluster().Client().CoreV1().Services(ns.Name).Create(ctx, service, metav1.CreateOptions{}) + service, err = k8sClient.CoreV1().Services(ns.Name).Create(ctx, service, metav1.CreateOptions{}) require.NoError(t, err) cleaner.Add(service) - t.Log("waiting for deployment to be ready") - deploymentName := deployment.Name - require.Eventually(t, func() bool { - deployment, err := env.Cluster().Client().AppsV1().Deployments(ns.Name).Get(ctx, deploymentName, metav1.GetOptions{}) - if err != nil { - t.Logf("Failed to get deployment %s/%s , error %v", ns.Name, deploymentName, err) - return false - } - if deployment.Status.Replicas == deployment.Status.AvailableReplicas { - return true - } - t.Logf("deployment not ready: %d/%d pods available", deployment.Status.AvailableReplicas, deployment.Status.Replicas) - return false - }, ingressWait, waitTick, func() string { - // dump status of all pods. - podList, err := env.Cluster().Client().CoreV1().Pods(ns.Name).List( - ctx, metav1.ListOptions{ - LabelSelector: "app=" + testName, - }) - if err != nil { - return err.Error() - } - podStatusString := []string{} - for _, pod := range podList.Items { - podStatusString = append(podStatusString, fmt.Sprintf("pod %s/%s: phase %s", - pod.Namespace, pod.Name, pod.Status.Phase)) - } - return strings.Join(podStatusString, "\n") - }()) - t.Log("adding TCPIngress") tcp := &kongv1beta1.TCPIngress{ ObjectMeta: metav1.ObjectMeta{ @@ -441,37 +371,25 @@ func TestTCPIngressTLSPassthrough(t *testing.T) { Spec: kongv1beta1.TCPIngressSpec{ Rules: []kongv1beta1.IngressRule{ { - Host: "redis.example", + Host: tlsExampleHostname, Port: ktfkong.DefaultTLSServicePort, Backend: kongv1beta1.IngressBackend{ ServiceName: service.Name, - ServicePort: 6379, + ServicePort: tlsEchoPort, }, }, }, }, } + gatewayClient, err := clientset.NewForConfig(env.Cluster().Config()) + require.NoError(t, err) tcp, err = gatewayClient.ConfigurationV1beta1().TCPIngresses(ns.Name).Create(ctx, tcp, metav1.CreateOptions{}) require.NoError(t, err) cleaner.Add(tcp) - t.Log("verifying TCP Ingress for redis.example operational") - require.Eventually(t, func() bool { - conn, err := tls.Dial("tcp", proxyTLSURL, &tls.Config{ - InsecureSkipVerify: true, - ServerName: "redis.example", - }) - if err != nil { - t.Logf("failed to connect to %s, error %v, retrying...", proxyTLSURL, err) - return false - } - defer conn.Close() - err = conn.Handshake() - if err != nil { - t.Logf("failed to do tls handshake to %s, error %v, retrying...", proxyTLSURL, err) - return false - } - cert := conn.ConnectionState().PeerCertificates[0] - return cert.Subject.CommonName == "secure-foo-bar" + t.Log("verifying that the tcpecho is responding properly over TLS") + require.EventuallyWithT(t, func(c *assert.CollectT) { + err := tlsEchoResponds(proxyTLSURL, testUUID, tlsRouteHostname, tlsRouteHostname, true) + assert.NoError(c, err) }, ingressWait, waitTick) }