Fix cert rotation tests

This commit changes testing cert rotation from from checking
container restart to check consistently running containers.
This futher can be extended to query ironic endpoint with new
certificate.

Signed-off-by: Sunnatillo <[email protected]>

test/e2e/cert_rotation.go

@@ -3,7 +3,6 @@ package e2e
 import (
 	"context"
 	"errors"
-	"fmt"
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -34,6 +33,7 @@ func certRotation(ctx context.Context, inputGetter func() CertRotationInput) {
 	ironicNamespace := input.E2EConfig.GetVariable("NAMEPREFIX") + "-system"
 	ironicDeploymentName := input.E2EConfig.GetVariable("NAMEPREFIX") + ironicSuffix
 	ironicDeployment, err := getDeployment(ctx, clusterClient, ironicDeploymentName, ironicNamespace)
+	Expect(err).ToNot(HaveOccurred(), "Failed to get ironic Deployment")
 	Eventually(func() error {
 		ironicPod, err := getPodFromDeployment(ctx, clientSet, ironicDeployment, ironicNamespace)
 		if err != nil {
@@ -48,21 +48,6 @@ func certRotation(ctx context.Context, inputGetter func() CertRotationInput) {
 
 	time.Sleep(5 * time.Minute)
 
-	By("Get the current number of time containers were restarted")
-	containerNumRestart := make(map[string]int32)
-	containerNumRestart["ironic-httpd"] = 0
-	if mariadbEnabled {
-		containerNumRestart["mariadb"] = 0
-	}
-	Expect(err).ToNot(HaveOccurred())
-	ironicPod, err := getPodFromDeployment(ctx, clientSet, ironicDeployment, ironicNamespace)
-	Expect(err).ToNot(HaveOccurred())
-	for _, container := range ironicPod.Status.ContainerStatuses {
-		if _, exist := containerNumRestart[container.Name]; exist {
-			containerNumRestart[container.Name] = container.RestartCount
-		}
-	}
-
 	By("Force the cert-manager to regenerate the certificate by deleting the secrets")
 	secretList := []string{
 		"ironic-cert",
@@ -75,37 +60,30 @@ func certRotation(ctx context.Context, inputGetter func() CertRotationInput) {
 		Expect(err).ToNot(HaveOccurred(), "Cannot detele this secret: %s", secretName)
 	}
 
-	By("Wait for containers in the ironic pod to be restarted")
+	By("Wait until secrets are recreated")
 	Eventually(func() error {
-		ironicPod, err := getPodFromDeployment(ctx, clientSet, ironicDeployment, ironicNamespace)
-		if err != nil {
-			return err
-		}
-		// check for container in containerNumRestart list
-		for container := range containerNumRestart {
-			notFound := true
-			for _, ironicContainer := range ironicPod.Status.ContainerStatuses {
-				if ironicContainer.Name == container {
-					notFound = false
-					break
-				}
-			}
-			if notFound {
-				return fmt.Errorf("%s container does not exist in Ironic pod", container)
-			}
+		ironicSecretCreated := isSecretRecreated(ctx, clientSet, ironicNamespace, secretList[0])
+		mariaDBSecretCreated := true
+		if mariadbEnabled {
+			mariaDBSecretCreated = isSecretRecreated(ctx, clientSet, ironicNamespace, secretList[1])
 		}
-		if ironicPod.Status.Phase == corev1.PodRunning {
-			for _, container := range ironicPod.Status.ContainerStatuses {
-				if oldNumRestart, exist := containerNumRestart[container.Name]; exist {
-					if !(oldNumRestart < container.RestartCount) {
-						return fmt.Errorf("%s is not restarted", container.Name)
-					}
-				}
-			}
+		if ironicSecretCreated && mariaDBSecretCreated {
 			return nil
 		}
-		return errors.New("ironic pod is not in running state")
+		return errors.New("Secret is being recreated")
 	}, input.E2EConfig.GetIntervals(input.SpecName, "wait-pod-restart")...).Should(BeNil())
+
+	// TODO(Sunnatillo): Further extend the test with querying ironic endpoint with new certificates
+	By("Check if all containers are running in ironic pod")
+	Consistently(func() error {
+		ironicPod, err := getPodFromDeployment(ctx, clientSet, ironicDeployment, ironicNamespace)
+		Expect(err).ToNot(HaveOccurred(), "Cannot get ironic from Deployment")
+
+		if areAllContainersRunning(ironicPod) {
+			return nil
+		}
+		return errors.New("not all containers are running")
+	}, 200*time.Second, 20*time.Second).Should(BeNil(), "not all containers are in running state in ironic pod")
 	By("CERTIFICATE ROTATION TESTS PASSED!")
 }
 
@@ -135,3 +113,17 @@ func getPodFromDeployment(ctx context.Context, clientSet *kubernetes.Clientset,
 	Expect(podList.Items).To(HaveLen(1), "The number of ironic pod is not equal to 1, but %v\n", len(podList.Items))
 	return &podList.Items[0], nil
 }
+
+func isSecretRecreated(ctx context.Context, clientset *kubernetes.Clientset, namespace, secretName string) bool {
+	_, err := clientset.CoreV1().Secrets(namespace).Get(ctx, secretName, metav1.GetOptions{})
+	return err == nil
+}
+
+func areAllContainersRunning(pod *corev1.Pod) bool {
+	for _, containerStatus := range pod.Status.ContainerStatuses {
+		if containerStatus.State.Running == nil {
+			return false
+		}
+	}
+	return true
+}