diff --git a/tests/system-tests/internal/apiobjectshelper/apiobjectshelper.go b/tests/system-tests/internal/apiobjectshelper/apiobjectshelper.go index 1b1871493..0ee86a0bc 100644 --- a/tests/system-tests/internal/apiobjectshelper/apiobjectshelper.go +++ b/tests/system-tests/internal/apiobjectshelper/apiobjectshelper.go @@ -5,6 +5,14 @@ import ( "fmt" "time" + "github.com/openshift-kni/eco-goinfra/pkg/deployment" + "github.com/openshift-kni/eco-goinfra/pkg/pod" + "github.com/openshift-kni/eco-goinfra/pkg/rbac" + "github.com/openshift-kni/eco-goinfra/pkg/service" + "github.com/openshift-kni/eco-goinfra/pkg/serviceaccount" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/golang/glog" "github.com/openshift-kni/eco-goinfra/pkg/clients" "github.com/openshift-kni/eco-goinfra/pkg/namespace" @@ -81,3 +89,251 @@ func VerifyOperatorDeployment(apiClient *clients.Settings, return nil } + +// CreateServiceAccount creates the service account and verifies it was created. +func CreateServiceAccount(apiClient *clients.Settings, saName, nsName string) error { + glog.V(100).Infof(fmt.Sprintf("Creating ServiceAccount %q in %q namespace", + saName, nsName)) + glog.V(100).Infof("Creating SA %q in %q namespace", saName, nsName) + + deploySa := serviceaccount.NewBuilder(apiClient, saName, nsName) + + err := wait.PollUntilContextTimeout( + context.TODO(), + time.Second*15, + time.Minute, + true, + func(ctx context.Context) (bool, error) { + deploySa, err := deploySa.Create() + + if err != nil { + glog.V(100).Infof("Error creating SA %q in %q namespace: %v", saName, nsName, err) + + return false, nil + } + + glog.V(100).Infof("Created SA %q in %q namespace", + deploySa.Definition.Name, deploySa.Definition.Namespace) + + return true, nil + }) + + if err != nil { + return fmt.Errorf("failed to create ServiceAccount %q in %q namespace", saName, nsName) + } + + return nil +} + +// CreateClusterRBAC creates the RBAC and verifies it was created. +func CreateClusterRBAC( + apiClient *clients.Settings, + rbacName, clusterRole, saName, nsName string) error { + glog.V(100).Infof("Creating RBAC for SA %s", saName) + + glog.V(100).Infof("Creating ClusterRoleBinding %q", rbacName) + crbSa := rbac.NewClusterRoleBindingBuilder( + apiClient, + rbacName, + clusterRole, + rbacv1.Subject{ + Name: saName, + Kind: "ServiceAccount", + Namespace: nsName, + }) + + err := wait.PollUntilContextTimeout( + context.TODO(), + time.Second*15, + time.Minute, + true, + func(ctx context.Context) (bool, error) { + crbSa, err := crbSa.Create() + if err != nil { + glog.V(100).Infof( + "Error Creating ClusterRoleBinding %q : %v", crbSa.Definition.Name, err) + + return false, nil + } + + glog.V(100).Infof("ClusterRoleBinding %q created:\n\t%v", + crbSa.Definition.Name, crbSa) + + return true, nil + }) + + if err != nil { + return fmt.Errorf("failed to create ClusterRoleBinding '%s' during timeout %v; %w", + rbacName, time.Minute, err) + } + + return nil +} + +// DeleteService deletes the service and verifies it was removed. +func DeleteService(apiClient *clients.Settings, svcName, nsName string) error { + glog.V(100).Infof("Delete service %q from nsname %s", svcName, nsName) + + if svcObj, err := service.Pull( + apiClient, svcName, nsName); err == nil { + glog.V(100).Infof("Service %q found in %q nsname", svcName, nsName) + glog.V(100).Infof("Deleting service %q in %q nsname", svcName, nsName) + + err = wait.PollUntilContextTimeout( + context.TODO(), + time.Second*15, + time.Minute, + true, + func(ctx context.Context) (bool, error) { + err := svcObj.Delete() + + if err != nil { + glog.V(100).Infof("Error deleting service %q in %q nsname: %v", + svcName, nsName, err) + + return false, nil + } + + glog.V(100).Infof("Deleted service %q in %q nsname", svcName, nsName) + + return true, nil + }) + + if err != nil { + return fmt.Errorf("failed to delete service %q from %q ns", svcName, nsName) + } + } else { + glog.V(100).Infof("service %q not found in %q nsname", svcName, nsName) + } + + return nil +} + +// DeleteClusterRBAC deletes the RBAC and verifies it was removed. +func DeleteClusterRBAC(apiClient *clients.Settings, rbacName string) error { + glog.V(100).Infof("Deleting Cluster RBAC") + + glog.V(100).Infof("Assert ClusterRoleBinding %q exists", rbacName) + + if crbSa, err := rbac.PullClusterRoleBinding( + apiClient, + rbacName); err == nil { + glog.V(100).Infof("ClusterRoleBinding %q found. Deleting...", rbacName) + + err = wait.PollUntilContextTimeout( + context.TODO(), + time.Second*15, + time.Minute, + true, + func(ctx context.Context) (bool, error) { + err = crbSa.Delete() + + if err != nil { + glog.V(100).Infof("Error deleting ClusterRoleBinding %q : %v", rbacName, err) + + return false, nil + } + + glog.V(100).Infof("Deleted ClusterRoleBinding %q", rbacName) + + return true, nil + }) + + if err != nil { + return fmt.Errorf("failed to delete Cluster RBAC %q", rbacName) + } + } + + return nil +} + +// DeleteServiceAccount deletes the service account and verifies it was removed. +func DeleteServiceAccount(apiClient *clients.Settings, saName, nsName string) error { + glog.V(100).Infof("Removing Service Account") + glog.V(100).Infof("Assert SA %q exists in %q namespace", saName, nsName) + + if deploySa, err := serviceaccount.Pull( + apiClient, saName, nsName); err == nil { + glog.V(100).Infof("ServiceAccount %q found in %q namespace", saName, nsName) + glog.V(100).Infof("Deleting ServiceAccount %q in %q namespace", saName, nsName) + + err = wait.PollUntilContextTimeout( + context.TODO(), + time.Second*15, + time.Minute, + true, + func(ctx context.Context) (bool, error) { + err := deploySa.Delete() + + if err != nil { + glog.V(100).Infof("Error deleting ServiceAccount %q in %q namespace: %v", + saName, nsName, err) + + return false, nil + } + + glog.V(100).Infof("Deleted ServiceAccount %q in %q namespace", saName, nsName) + + return true, nil + }) + + if err != nil { + return fmt.Errorf("failed to delete ServiceAccount %q from %q ns", saName, nsName) + } + } else { + glog.V(100).Infof("ServiceAccount %q not found in %q namespace", saName, nsName) + } + + return nil +} + +// DeleteDeployment deletes the deployment and verifies it and all related pods were removed. +func DeleteDeployment( + apiClient *clients.Settings, + deploymentName, nsName string) error { + glog.V(100).Infof("Removing test deployment %q from %q ns", deploymentName, nsName) + + if deploymentObj, err := deployment.Pull(apiClient, deploymentName, nsName); err == nil { + glog.V(100).Infof("Deleting deployment %q from %q namespace", deploymentName, nsName) + + err = deploymentObj.DeleteAndWait(300 * time.Second) + + if err != nil { + glog.V(100).Infof("Error deleting deployment %q from %q namespace: %v", + deploymentName, nsName, err) + + return fmt.Errorf("failed to delete deployment %q from %q namespace: %w", + deploymentName, nsName, err) + } + } else { + glog.V(100).Infof("deployment %q not found in %q namespace", deploymentName, nsName) + } + + return nil +} + +// InsureAllPodsRemoved insure all deployment pods in namespace with the specific pod label were removed. +func InsureAllPodsRemoved( + apiClient *clients.Settings, + nsName, podLabel string) error { + glog.V(100).Infof("Ensuring pods in %q namespace with label %q are gone", nsName, podLabel) + + err := wait.PollUntilContextTimeout( + context.TODO(), + time.Second*3, + time.Minute*6, + true, + func(ctx context.Context) (bool, error) { + oldPods, _ := pod.List(apiClient, nsName, + metav1.ListOptions{LabelSelector: podLabel}) + + return len(oldPods) == 0, nil + }) + + if err != nil { + return fmt.Errorf("pods matching label(%q) still present in namespace %q", + podLabel, nsName) + } + + return nil +} diff --git a/tests/system-tests/rdscore/internal/rdscorecommon/egress-ip.go b/tests/system-tests/rdscore/internal/rdscorecommon/egress-ip.go index 0dbfebe5e..d395822bf 100644 --- a/tests/system-tests/rdscore/internal/rdscorecommon/egress-ip.go +++ b/tests/system-tests/rdscore/internal/rdscorecommon/egress-ip.go @@ -1,10 +1,14 @@ package rdscorecommon import ( + "context" "fmt" + "net" "strings" "time" + "k8s.io/apimachinery/pkg/util/wait" + "github.com/openshift-kni/eco-goinfra/pkg/pod" scc "github.com/openshift-kni/eco-gotests/tests/system-tests/internal/scc" @@ -16,7 +20,6 @@ import ( "github.com/openshift-kni/eco-goinfra/pkg/clients" "github.com/openshift-kni/eco-goinfra/pkg/deployment" "github.com/openshift-kni/eco-goinfra/pkg/egressip" - "github.com/openshift-kni/eco-gotests/tests/system-tests/internal/sniffer" . "github.com/openshift-kni/eco-gotests/tests/system-tests/rdscore/internal/rdscoreinittools" "github.com/openshift-kni/eco-gotests/tests/system-tests/rdscore/internal/rdscoreparams" corev1 "k8s.io/api/core/v1" @@ -24,78 +27,42 @@ import ( ) const ( - snifferNamespace = "rds-egress-ns" - proberDeployRBACName = "privileged-rdscore-prober" - proberDeploySAName = "rdscore-prober-sa" - proberRBACRole = "system:openshift:scc:privileged" - numberOfRequestsToSend = 10 - proberTargetProtocol = "http" + proberRBACRole = "system:openshift:scc:privileged" + centosIPPodLabel = "env=centos" ) var ( - egressIPNodesList = []string{RDSCoreConfig.EgressIPNodeOne, RDSCoreConfig.EgressIPNodeTwo} - proberTargetPort = RDSCoreConfig.EgressIPTcpPort - egressIPPodLabelsMap = map[string]string{ - strings.Split(RDSCoreConfig.EgressIPPodLabel, - "=")[0]: strings.Split(RDSCoreConfig.EgressIPPodLabel, "=")[1]} + strings.Split(rdscoreparams.EgressIPPodLabel, "=")[0]: strings.Split(rdscoreparams.EgressIPPodLabel, "=")[1]} egressIPPodSelector = metav1.ListOptions{ - LabelSelector: RDSCoreConfig.EgressIPPodLabel, + LabelSelector: rdscoreparams.EgressIPPodLabel, } ) -// createAgnhostDeployment creates the route, service and deployment that will be used as -// a source for EgressIP tests. Returns the route name that can be queried to run queries against the source pods. -// -//nolint:funlen -func createAgnhostDeployment( +// createAgnhostRBAC creates the SCC privileged, serviceAccount and RBAC. +func createAgnhostRBAC( apiClient *clients.Settings, - egressIPNamespace string, - scheduleOnHosts []string) error { - proberServiceName := fmt.Sprintf("%s-service", egressIPNamespace) - proberDeploymentName := fmt.Sprintf("%s-deployment", egressIPNamespace) + egressIPNamespace string) (string, error) { + proberDeploySAName := fmt.Sprintf("rdscore-prober-sa-%s", egressIPNamespace) + proberDeployRBACName := fmt.Sprintf("privileged-rdscore-rbac-%s", egressIPNamespace) var err error - glog.V(100).Infof("Checking prober deployment don't exist") - - err = apiobjectshelper.DeleteDeployment(apiClient, - proberDeploymentName, - egressIPNamespace, - RDSCoreConfig.EgressIPPodLabel) - - if err != nil { - return fmt.Errorf("failed to delete deployment %s from egressIPNamespace %s", - proberDeploymentName, egressIPNamespace) - } - - glog.V(100).Infof("Sleeping 10 seconds") - time.Sleep(10 * time.Second) - - glog.V(100).Infof("Adding SCC privileged to the prober namespace") + glog.V(100).Infof("Adding SCC privileged to the agnhost namespace") err = scc.AddPrivilegedSCCtoDefaultSA(egressIPNamespace) if err != nil { - return fmt.Errorf("failed to add SCC privileged to the prober namespace %s: %w", + return "", fmt.Errorf("failed to add SCC privileged to the agnhost namespace %s: %w", egressIPNamespace, err) } - glog.V(100).Infof("Removing Service") - - err = apiobjectshelper.DeleteService(apiClient, proberServiceName, egressIPNamespace) - - if err != nil { - return fmt.Errorf("failed to remove service %q from egressIPNamespace %q; %w", - proberServiceName, egressIPNamespace, err) - } - glog.V(100).Infof("Removing ServiceAccount") err = apiobjectshelper.DeleteServiceAccount(apiClient, proberDeploySAName, egressIPNamespace) if err != nil { - return fmt.Errorf("failed to remove serviceAccount %q from egressIPNamespace %q", + return "", fmt.Errorf("failed to remove serviceAccount %q from egressIPNamespace %q", proberDeploySAName, egressIPNamespace) } @@ -104,7 +71,7 @@ func createAgnhostDeployment( err = apiobjectshelper.CreateServiceAccount(apiClient, proberDeploySAName, egressIPNamespace) if err != nil { - return fmt.Errorf("failed to create serviceAccount %q in egressIPNamespace %q", + return "", fmt.Errorf("failed to create serviceAccount %q in egressIPNamespace %q", proberDeploySAName, egressIPNamespace) } @@ -113,7 +80,7 @@ func createAgnhostDeployment( err = apiobjectshelper.DeleteClusterRBAC(apiClient, proberDeployRBACName) if err != nil { - return fmt.Errorf("failed to delete prober RBAC %q", proberDeployRBACName) + return "", fmt.Errorf("failed to delete prober RBAC %q", proberDeployRBACName) } glog.V(100).Infof("Creating Cluster RBAC") @@ -122,17 +89,66 @@ func createAgnhostDeployment( proberDeploySAName, egressIPNamespace) if err != nil { - return fmt.Errorf("failed to create prober RBAC %q in egressIPNamespace %s", + return "", fmt.Errorf("failed to create prober RBAC %q in egressIPNamespace %s", proberDeployRBACName, egressIPNamespace) } + return proberDeploySAName, nil +} + +// cleanUpDeployment removes all qe deployment leftovers from the cluster according to the label. +func cleanUpDeployment( + apiClient *clients.Settings, + nsname, + scheduleOnHost string) error { + nameSuffix := strings.Split(scheduleOnHost, ".")[0] + agnhostDeploymentName := fmt.Sprintf("%s-deployment-%s", nsname, nameSuffix) + centosDeploymentName := fmt.Sprintf("%s-centos-deployment-%s", nsname, nameSuffix) + + glog.V(100).Infof("Insure egressIP deployment don't exist") + + err := apiobjectshelper.DeleteDeployment( + apiClient, + centosDeploymentName, + nsname) + + if err != nil { + return fmt.Errorf("failed to delete deployment %s from nsname %s", + agnhostDeploymentName, nsname) + } + + err = apiobjectshelper.DeleteDeployment( + apiClient, + agnhostDeploymentName, + nsname) + + if err != nil { + return fmt.Errorf("failed to delete deployment %s from nsname %s", + agnhostDeploymentName, nsname) + } + + return nil +} + +// createAgnhostDeployment creates the agnhost deployment that will be used as a source for EgressIP tests. +func createAgnhostDeployment( + apiClient *clients.Settings, + egressIPNamespace, + proberDeploySAName, + scheduleOnHost string) error { + replicaCnt := 1 + nameSuffix := strings.Split(scheduleOnHost, ".")[0] + agnhostDeploymentName := fmt.Sprintf("%s-deployment-%s", egressIPNamespace, nameSuffix) + + var err error + glog.V(100).Infof("Defining container configuration") containerCmd := []string{ "/agnhost", "netexec", "--http-port", - fmt.Sprintf("%d", RDSCoreConfig.EgressIPTcpPort), + RDSCoreConfig.EgressIPTcpPort, } deployContainer := defineProberContainer(RDSCoreConfig.EgressIPDeploymentImage, containerCmd) @@ -146,13 +162,14 @@ func createAgnhostDeployment( glog.V(100).Infof("Defining deployment configuration") - proberDeployment := defineProberDeployment( + proberDeployment := defineTestPodDeployment( apiClient, deployContainerCfg, - proberDeploymentName, + agnhostDeploymentName, egressIPNamespace, proberDeploySAName, - scheduleOnHosts, + scheduleOnHost, + replicaCnt, egressIPPodLabelsMap) glog.V(100).Infof("Creating deployment") @@ -160,22 +177,80 @@ func createAgnhostDeployment( proberDeployment, err = proberDeployment.CreateAndWaitUntilReady(5 * time.Minute) if err != nil { return fmt.Errorf("failed to create deployment %s in namespace %s: %w", - proberDeploymentName, egressIPNamespace, err) + agnhostDeploymentName, egressIPNamespace, err) } if proberDeployment == nil { return fmt.Errorf("failed to create deployment %s in namespace %s", - proberDeploymentName, egressIPNamespace) + agnhostDeploymentName, egressIPNamespace) } return nil } -func defineProberDeployment( +// createCentosDeployment creates the centos deployment that will be used in addition to the source for EgressIP tests. +func createCentosDeployment( + apiClient *clients.Settings, + nsName, + deploySAName, + scheduleOnHost string) error { + replicaCnt := 1 + nameSuffix := strings.Split(scheduleOnHost, ".")[0] + deploymentName := fmt.Sprintf("%s-centos-deployment-%s", nsName, nameSuffix) + centosIPPodLabelsMap := map[string]string{ + strings.Split(centosIPPodLabel, "=")[0]: strings.Split(centosIPPodLabel, "=")[1], + strings.Split(rdscoreparams.EgressIPPodLabel, "=")[0]: strings.Split(rdscoreparams.EgressIPPodLabel, "=")[1], + } + + var err error + + glog.V(100).Infof("Defining container configuration") + + containerCmd := []string{"/bin/sh", "-c", "sleep infinity"} + + deployContainer := defineProberContainer(RDSCoreConfig.NonEgressIPDeploymentImage, containerCmd) + + glog.V(100).Infof("Obtaining container definition") + + deployContainerCfg, err := deployContainer.GetContainerCfg() + if err != nil { + return fmt.Errorf("failed to obtain container definition: %w", err) + } + + glog.V(100).Infof("Defining deployment configuration") + + proberDeployment := defineTestPodDeployment( + apiClient, + deployContainerCfg, + deploymentName, + nsName, + deploySAName, + scheduleOnHost, + replicaCnt, + centosIPPodLabelsMap) + + glog.V(100).Infof("Creating deployment") + + proberDeployment, err = proberDeployment.CreateAndWaitUntilReady(5 * time.Minute) + if err != nil { + return fmt.Errorf("failed to create deployment %s in namespace %s: %w", + deploymentName, nsName, err) + } + + if proberDeployment == nil { + return fmt.Errorf("failed to create deployment %s in namespace %s", + deploymentName, nsName) + } + + return nil +} + +func defineTestPodDeployment( apiClient *clients.Settings, containerConfig *corev1.Container, deployName, deployNs, saName string, - scheduleOnHosts []string, + scheduleOnHost string, + replicaCnt int, deployLabels map[string]string) *deployment.Builder { glog.V(100).Infof("Defining deployment %q in %q ns", deployName, deployNs) @@ -188,7 +263,7 @@ func defineProberDeployment( { Key: "kubernetes.io/hostname", Operator: corev1.NodeSelectorOpIn, - Values: scheduleOnHosts, + Values: []string{scheduleOnHost}, }, }, }, @@ -197,21 +272,21 @@ func defineProberDeployment( }, } - proberDeployment := deployment.NewBuilder(apiClient, deployName, deployNs, deployLabels, *containerConfig) + podsDeployment := deployment.NewBuilder(apiClient, deployName, deployNs, deployLabels, *containerConfig) glog.V(100).Infof("Assigning ServiceAccount %q to the deployment", saName) - proberDeployment = proberDeployment.WithServiceAccountName(saName) + podsDeployment = podsDeployment.WithServiceAccountName(saName) glog.V(100).Infof("Setting Replicas count") - proberDeployment = proberDeployment.WithReplicas(int32(len(scheduleOnHosts))) + podsDeployment = podsDeployment.WithReplicas(int32(replicaCnt)) - proberDeployment = proberDeployment.WithHostNetwork(true) + podsDeployment = podsDeployment.WithHostNetwork(false) - proberDeployment = proberDeployment.WithAffinity(&nodeAffinity) + podsDeployment = podsDeployment.WithAffinity(&nodeAffinity) - return proberDeployment + return podsDeployment } func defineProberContainer(cImage string, cCmd []string) *pod.ContainerBuilder { @@ -253,210 +328,145 @@ func defineProberContainer(cImage string, cCmd []string) *pod.ContainerBuilder { return deployContainer } -// CreateEgressIPWithSingleNamespace create egressIP test setup to verify connectivity on the same namespace. -func CreateEgressIPWithSingleNamespace() { - By("Creating the packet sniffer deployment with number of pods equals number of EgressIP nodes") +func sendTrafficCheckIP(clientPods []*pod.Builder, cmdToRun, expectedIPs []string) error { + By("Validating pods source address") - _, err := - sniffer.CreatePacketSnifferDeployment( - APIClient, - proberTargetPort, - proberTargetProtocol, - RDSCoreConfig.EgressIPPacketSnifferInterface, - snifferNamespace, - egressIPNodesList) - Expect(err).ToNot(HaveOccurred(), - fmt.Sprintf("Failed to create sniffer deployment for %s and %s nodes in namespace %s: %v", - RDSCoreConfig.EgressIPNodeOne, RDSCoreConfig.EgressIPNodeTwo, snifferNamespace, err)) + for _, clientPod := range clientPods { + var parsedIP string - By("Creating the EgressIP test source deployment with number of pods equals number of EgressIP nodes") + err := wait.PollUntilContextTimeout( + context.TODO(), + time.Second*5, + time.Minute, + true, + func(ctx context.Context) (bool, error) { + result, err := clientPod.ExecCommand(cmdToRun, clientPod.Object.Spec.Containers[0].Name) - glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Create prober deployment based on the agnhost image") + if err != nil { + glog.V(100).Infof("Error running command from within a pod %q: %v", + clientPod.Object.Name, err) - err = createAgnhostDeployment( - APIClient, - RDSCoreConfig.EgressIPNamespaceOne, - egressIPNodesList) - Expect(err).ToNot(HaveOccurred(), - fmt.Sprintf("Failed to create prober deployment and ingress route for nodes %q: %v", - egressIPNodesList, err)) -} + return false, nil + } -// VerifyEgressIPConnectivityForSingleNamespace verifies egress traffic works with egressIP -// applied for the external target in the same namespace. -// -//nolint:funlen -func VerifyEgressIPConnectivityForSingleNamespace() { - By("Getting a map of source nodes and assigned Egress IPs for these nodes") + glog.V(100).Infof("Successfully executed command from within a pod %q: %v", + clientPod.Object.Name, err) + glog.V(100).Infof("Command's output:\n\t%v", result.String()) - egressIPObj, err := egressip.Pull(APIClient, RDSCoreConfig.EgressIPName) - Expect(err).ToNot(HaveOccurred(), - fmt.Sprintf("Failed to retrieve egressIP %s object: %v", RDSCoreConfig.EgressIPName, err)) + parsedIP, _, err = net.SplitHostPort(result.String()) - egressIPSet, err := egressIPObj.GetAssignedEgressIPMap() - Expect(err).ToNot(HaveOccurred(), - fmt.Sprintf("Failed to retrieve egressIP %s assigned egressIPs map: %v", RDSCoreConfig.EgressIPName, err)) - Expect(len(egressIPSet)).To(Equal(2), - fmt.Sprintf("EgressIPs assigned to the wrong number of nodes: %v", egressIPSet)) + if err != nil { + glog.V(100).Infof("Failed to parse %q for host/port pair", result.String()) - By("Spawning the prober pods on the EgressIP assignable hosts") + return false, nil + } - for clusterNode, proberTargetHost := range egressIPSet { - for i := 0; i < 2; i++ { - By(fmt.Sprintf("Sending requests from prober and making sure that %d requests with search string and "+ - "EgressIPs %v were seen", numberOfRequestsToSend, egressIPSet)) + glog.V(100).Infof("Comparing %q with expected %q", parsedIP, expectedIPs) - glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Get prober pod object") + for _, expectedIP := range expectedIPs { + if parsedIP == expectedIP { + return true, nil + } + } - proberPodObjects, err := pod.List(APIClient, RDSCoreConfig.EgressIPNamespaceOne, egressIPPodSelector) - Expect(err).ToNot(HaveOccurred(), - fmt.Sprintf("Failed to retrieve prober pods list from namespace %s with label %s: %v", - RDSCoreConfig.EgressIPNamespaceOne, RDSCoreConfig.EgressIPPodLabel, err)) + glog.V(100).Infof("Mismatched IP address. Expected %q got %q", expectedIPs, parsedIP) - var proberPodObj *pod.Builder + return false, nil + }) - for _, podObj := range proberPodObjects { - if podObj.Object.Spec.NodeName == clusterNode { - glog.V(rdscoreparams.RDSCoreLogLevel).Infof("pod: %s", podObj.Definition.Name) - proberPodObj = podObj - } - } - - Expect(proberPodObj).ToNot(Equal(nil), - fmt.Sprintf("prober pod not found running on the %s node", clusterNode)) - - err = sniffer.SendTrafficCheckLogs( - APIClient, - proberPodObj, - snifferNamespace, - RDSCoreConfig.EgressIPPodLabel, - proberTargetHost, - proberTargetProtocol, - proberTargetPort, - numberOfRequestsToSend, - numberOfRequestsToSend) - Expect(err).ToNot(HaveOccurred(), - fmt.Sprintf("Failed to find required number of requests %d: %v", numberOfRequestsToSend, err)) + if err != nil { + return fmt.Errorf("failed to run command from within pod %s: %w", clientPod.Object.Name, err) } } - By("Verify defined, but not assigned egressIP address is reachable") + return nil +} - glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Get prober pod object") +// CreateEgressIPBasicDeployment create egressIP test setup to verify connectivity with the external server. +func CreateEgressIPBasicDeployment() { + By("Creating the EgressIP test source deployment") - podSelector := metav1.ListOptions{ - LabelSelector: RDSCoreConfig.EgressIPPodLabel, - } + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Create prober deployment based on the agnhost image") - proberPodObjects, err := pod.List(APIClient, RDSCoreConfig.EgressIPNamespaceOne, podSelector) + err := cleanUpDeployment( + APIClient, + RDSCoreConfig.EgressIPNamespaceOne, + RDSCoreConfig.EgressIPNodeOne) Expect(err).ToNot(HaveOccurred(), - fmt.Sprintf("Failed to retrieve prober pods list from namespace %s with label %s: %v", - RDSCoreConfig.EgressIPNamespaceOne, RDSCoreConfig.EgressIPPodLabel, err)) + fmt.Sprintf("Failed to delete deployment in namespace %s from node %s: %v", + RDSCoreConfig.EgressIPNamespaceOne, RDSCoreConfig.EgressIPNodeOne, err)) - err = sniffer.SendTrafficCheckLogs( + err = cleanUpDeployment( APIClient, - proberPodObjects[0], - snifferNamespace, - RDSCoreConfig.EgressIPPodLabel, - RDSCoreConfig.EgressIPRemoteIPThree, - proberTargetProtocol, - proberTargetPort, - numberOfRequestsToSend, - numberOfRequestsToSend) + RDSCoreConfig.EgressIPNamespaceOne, + RDSCoreConfig.NonEgressIPNode) Expect(err).ToNot(HaveOccurred(), - fmt.Sprintf("Failed to find required number of requests %d: %v", numberOfRequestsToSend, err)) - - By("Verify undefined egressIP address is not reachable") + fmt.Sprintf("Failed to delete deployment in namespace %s from node %s: %v", + RDSCoreConfig.EgressIPNamespaceOne, RDSCoreConfig.NonEgressIPNode, err)) - err = sniffer.SendTrafficCheckLogs( + err = cleanUpDeployment( APIClient, - proberPodObjects[0], - snifferNamespace, - RDSCoreConfig.EgressIPPodLabel, - RDSCoreConfig.EgressIPRemoteIPFour, - proberTargetProtocol, - proberTargetPort, - numberOfRequestsToSend, - numberOfRequestsToSend) - Expect(err).To(HaveOccurred(), "No packets should be seen for the undefined egressIP address") -} - -// VerifyEgressIPWithSingleNamespace verifies egress traffic works with egressIP -// applied for the external target. -func VerifyEgressIPWithSingleNamespace() { - By("Creating egressIP test setup to verify connectivity on the same namespace") - - CreateEgressIPWithSingleNamespace() - - By("Verify egressIP connectivity for the same namespace") - - VerifyEgressIPConnectivityForSingleNamespace() -} - -// CreateEgressIPMixedNodesAndNamespaces create egressIP test setup to verify connectivity on the -// different namespaces and nodes. -func CreateEgressIPMixedNodesAndNamespaces() { - By("Creating the packet sniffer deployment with number of pods equals number of EgressIP nodes") - - _, err := - sniffer.CreatePacketSnifferDeployment( - APIClient, - proberTargetPort, - proberTargetProtocol, - RDSCoreConfig.EgressIPPacketSnifferInterface, - snifferNamespace, - egressIPNodesList) + RDSCoreConfig.EgressIPNamespaceTwo, + RDSCoreConfig.EgressIPNodeTwo) Expect(err).ToNot(HaveOccurred(), - fmt.Sprintf("Failed to create sniffer deployment for %s and %s nodes in namespace %s: %v", - RDSCoreConfig.EgressIPNodeOne, RDSCoreConfig.EgressIPNodeTwo, snifferNamespace, err)) - - By(fmt.Sprintf("Create prober pod deployment in namespace %s on the node %s", - RDSCoreConfig.EgressIPNamespaceOne, RDSCoreConfig.EgressIPNodeOne)) + fmt.Sprintf("Failed to delete deployment in namespace %s from node %s: %v", + RDSCoreConfig.EgressIPNamespaceTwo, RDSCoreConfig.EgressIPNodeTwo, err)) - err = createAgnhostDeployment( + err = apiobjectshelper.InsureAllPodsRemoved( APIClient, RDSCoreConfig.EgressIPNamespaceOne, - []string{RDSCoreConfig.EgressIPNodeOne}) + rdscoreparams.EgressIPPodLabel) Expect(err).ToNot(HaveOccurred(), - fmt.Sprintf("Failed to create prober deployment for node %s: %v", RDSCoreConfig.EgressIPNodeOne, err)) - - By(fmt.Sprintf("Create prober pod deployment in namespace %s on the node %s", - RDSCoreConfig.EgressIPNamespaceTwo, RDSCoreConfig.EgressIPNodeTwo)) + fmt.Sprintf("Failed to delete pods in namespace %s: %v", RDSCoreConfig.EgressIPNamespaceOne, err)) - err = createAgnhostDeployment( + err = apiobjectshelper.InsureAllPodsRemoved( APIClient, RDSCoreConfig.EgressIPNamespaceTwo, - []string{RDSCoreConfig.EgressIPNodeTwo}) + rdscoreparams.EgressIPPodLabel) Expect(err).ToNot(HaveOccurred(), - fmt.Sprintf("Failed to create prober deployment in namespace %s for node %s: %v", - RDSCoreConfig.EgressIPNamespaceTwo, RDSCoreConfig.EgressIPNodeTwo, err)) + fmt.Sprintf("Failed to delete pods in namespace %s: %v", RDSCoreConfig.EgressIPNamespaceTwo, err)) - By(fmt.Sprintf("Create prober deployment in namespace %s on the node %s", - RDSCoreConfig.EgressIPNamespaceTwo, RDSCoreConfig.NonEgressIPNodeOne)) + proberDeploySAName, err := createAgnhostRBAC(APIClient, RDSCoreConfig.EgressIPNamespaceOne) + Expect(err).ToNot(HaveOccurred(), + fmt.Sprintf("Failed to create prober RBAC in namespace %s: %v", RDSCoreConfig.EgressIPNamespaceOne, err)) err = createAgnhostDeployment( APIClient, - RDSCoreConfig.EgressIPNamespaceTwo, - []string{RDSCoreConfig.NonEgressIPNodeOne}) + RDSCoreConfig.EgressIPNamespaceOne, + proberDeploySAName, + RDSCoreConfig.EgressIPNodeOne) Expect(err).ToNot(HaveOccurred(), - fmt.Sprintf("Failed to create prober deployment for node %s: %v", RDSCoreConfig.NonEgressIPNodeOne, err)) + fmt.Sprintf("Failed to create prober deployment for node %s in namespace %s: %v", + RDSCoreConfig.EgressIPNodeOne, RDSCoreConfig.EgressIPNamespaceOne, err)) - By(fmt.Sprintf("Create prober deployment in namespace %s on the node %s", - RDSCoreConfig.EgressIPNamespaceThree, RDSCoreConfig.EgressIPNodeOne)) + for _, nodeToAssign := range []string{RDSCoreConfig.EgressIPNodeOne, RDSCoreConfig.NonEgressIPNode} { + err = createCentosDeployment( + APIClient, + RDSCoreConfig.EgressIPNamespaceOne, + proberDeploySAName, + nodeToAssign) + Expect(err).ToNot(HaveOccurred(), + fmt.Sprintf("Failed to create prober deployment for node %s in namespace %s: %v", + nodeToAssign, RDSCoreConfig.EgressIPNamespaceOne, err)) + } + + proberDeploySAName, err = createAgnhostRBAC(APIClient, RDSCoreConfig.EgressIPNamespaceTwo) + Expect(err).ToNot(HaveOccurred(), + fmt.Sprintf("Failed to create prober RBAC in namespace %s: %v", RDSCoreConfig.EgressIPNamespaceTwo, err)) err = createAgnhostDeployment( APIClient, - RDSCoreConfig.EgressIPNamespaceThree, - []string{RDSCoreConfig.EgressIPNodeOne}) + RDSCoreConfig.EgressIPNamespaceTwo, + proberDeploySAName, + RDSCoreConfig.EgressIPNodeTwo) Expect(err).ToNot(HaveOccurred(), - fmt.Sprintf("Failed to create prober deployment for node %s: %v", RDSCoreConfig.EgressIPNodeOne, err)) + fmt.Sprintf("Failed to create prober deployment for node %s in namespace %s: %v", + RDSCoreConfig.EgressIPNodeTwo, RDSCoreConfig.EgressIPNamespaceTwo, err)) } -// VerifyEgressIPConnectivityMixedNodesAndNamespaces verifies egress traffic works with egressIP -// applied for the external target on the different namespaces and nodes. -// -//nolint:funlen -func VerifyEgressIPConnectivityMixedNodesAndNamespaces() { +// VerifyEgressIPConnectivityWithExternalServer verifies egress traffic works with egressIP +// applied for the external target. +func VerifyEgressIPConnectivityWithExternalServer() { By("Getting a map of source nodes and assigned Egress IPs for these nodes") egressIPObj, err := egressip.Pull(APIClient, RDSCoreConfig.EgressIPName) @@ -469,102 +479,41 @@ func VerifyEgressIPConnectivityMixedNodesAndNamespaces() { Expect(len(egressIPSet)).To(Equal(2), fmt.Sprintf("EgressIPs assigned to the wrong number of nodes: %v", egressIPSet)) - By(fmt.Sprintf("Sending requests from prober pod in namespace %s "+ - "on the node %s and making sure that %d requests were seen", - RDSCoreConfig.EgressIPNamespaceOne, RDSCoreConfig.EgressIPNodeOne, numberOfRequestsToSend)) - - proberPodObjects, err := pod.List(APIClient, RDSCoreConfig.EgressIPNamespaceOne, egressIPPodSelector) - Expect(err).ToNot(HaveOccurred(), - fmt.Sprintf("Failed to retrieve prober pods list from namespace %s with label %s: %v", - RDSCoreConfig.EgressIPNamespaceOne, RDSCoreConfig.EgressIPPodLabel, err)) + By("Spawning the prober pods on the EgressIP assignable hosts") - err = sniffer.SendTrafficCheckLogs( - APIClient, - proberPodObjects[0], - snifferNamespace, - RDSCoreConfig.EgressIPPodLabel, - egressIPSet[RDSCoreConfig.EgressIPNodeOne], - proberTargetProtocol, - proberTargetPort, - numberOfRequestsToSend, - numberOfRequestsToSend) - Expect(err).ToNot(HaveOccurred(), - fmt.Sprintf("Failed to find required number of requests %d: %v", numberOfRequestsToSend, err)) + cmdToRun := []string{"/bin/bash", "-c", + fmt.Sprintf("curl --connect-timeout 5 -Ls http://%s:%s/clientip", + RDSCoreConfig.EgressIPRemoteIP, RDSCoreConfig.EgressIPTcpPort)} - By(fmt.Sprintf("Sending requests from prober pod in namespace %s "+ - "on the node %s and making sure that %d requests were seen", - RDSCoreConfig.EgressIPNamespaceTwo, RDSCoreConfig.EgressIPNodeTwo, numberOfRequestsToSend)) + expectedIPs := []string{egressIPSet[RDSCoreConfig.EgressIPNodeOne], egressIPSet[RDSCoreConfig.EgressIPNodeTwo]} - proberPodObjects, err = pod.List(APIClient, RDSCoreConfig.EgressIPNamespaceTwo, egressIPPodSelector) + proberPodObjects, err := pod.List(APIClient, RDSCoreConfig.EgressIPNamespaceOne, egressIPPodSelector) Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to retrieve prober pods list from namespace %s with label %s: %v", - RDSCoreConfig.EgressIPNamespaceTwo, RDSCoreConfig.EgressIPPodLabel, err)) + RDSCoreConfig.EgressIPNamespaceOne, rdscoreparams.EgressIPPodLabel, err)) - err = sniffer.SendTrafficCheckLogs( - APIClient, - proberPodObjects[0], - snifferNamespace, - RDSCoreConfig.EgressIPPodLabel, - egressIPSet[RDSCoreConfig.EgressIPNodeTwo], - proberTargetProtocol, - proberTargetPort, - numberOfRequestsToSend, - numberOfRequestsToSend) + err = sendTrafficCheckIP(proberPodObjects, cmdToRun, expectedIPs) Expect(err).ToNot(HaveOccurred(), - fmt.Sprintf("Failed to find required number of requests %d: %v", numberOfRequestsToSend, err)) - - By(fmt.Sprintf("Sending requests from prober pod in namespace %s "+ - "on the node %s and making sure that %d requests were seen", - RDSCoreConfig.EgressIPNamespaceTwo, RDSCoreConfig.NonEgressIPNodeOne, numberOfRequestsToSend)) + fmt.Sprintf("Server response was note received: %v", err)) proberPodObjects, err = pod.List(APIClient, RDSCoreConfig.EgressIPNamespaceTwo, egressIPPodSelector) Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to retrieve prober pods list from namespace %s with label %s: %v", - RDSCoreConfig.EgressIPNamespaceTwo, RDSCoreConfig.EgressIPPodLabel, err)) + RDSCoreConfig.EgressIPNamespaceTwo, rdscoreparams.EgressIPPodLabel, err)) - err = sniffer.SendTrafficCheckLogs( - APIClient, - proberPodObjects[0], - snifferNamespace, - RDSCoreConfig.EgressIPPodLabel, - egressIPSet[RDSCoreConfig.EgressIPNodeTwo], - proberTargetProtocol, - proberTargetPort, - numberOfRequestsToSend, - numberOfRequestsToSend) + err = sendTrafficCheckIP(proberPodObjects, cmdToRun, expectedIPs) Expect(err).ToNot(HaveOccurred(), - fmt.Sprintf("Failed to find required number of requests %d: %v", numberOfRequestsToSend, err)) - - By(fmt.Sprintf("Sending requests from prober pod in namespace %s "+ - "on the node %s and making sure that %d requests were seen", - RDSCoreConfig.EgressIPNamespaceThree, RDSCoreConfig.EgressIPNodeOne, numberOfRequestsToSend)) - - proberPodObjects, err = pod.List(APIClient, RDSCoreConfig.EgressIPNamespaceThree, egressIPPodSelector) - Expect(err).ToNot(HaveOccurred(), - fmt.Sprintf("Failed to retrieve prober pods list from namespace %s with label %s: %v", - RDSCoreConfig.EgressIPNamespaceThree, RDSCoreConfig.EgressIPPodLabel, err)) - - err = sniffer.SendTrafficCheckLogs( - APIClient, - proberPodObjects[0], - snifferNamespace, - RDSCoreConfig.EgressIPPodLabel, - egressIPSet[RDSCoreConfig.EgressIPNodeOne], - proberTargetProtocol, - proberTargetPort, - numberOfRequestsToSend, - numberOfRequestsToSend) - Expect(err).To(HaveOccurred(), "traffic receive for the pod from the non-egressIP assigned namespace") + fmt.Sprintf("Server response was not received: %v", err)) } -// VerifyEgressIPDifferentNodesAndNamespaces verifies egress traffic works with egressIP applied -// for the external target with two additional namespaces in use on a different nodes. -func VerifyEgressIPDifferentNodesAndNamespaces() { - By("Creating egressIP test setup to verify connectivity on the different namespaces and nodes") +// VerifyEgressIPWithTwoNamespacesIPv4 verifies egress traffic works with egressIP +// applied for the external target. +func VerifyEgressIPWithTwoNamespacesIPv4() { + By("Creating egressIP test setup to verify connectivity on the same namespace") - CreateEgressIPMixedNodesAndNamespaces() + CreateEgressIPBasicDeployment() By("Verify egressIP connectivity for the same namespace") - VerifyEgressIPConnectivityMixedNodesAndNamespaces() + VerifyEgressIPConnectivityWithExternalServer() } diff --git a/tests/system-tests/rdscore/internal/rdscoreconfig/config.go b/tests/system-tests/rdscore/internal/rdscoreconfig/config.go index c66755d62..7e04cfaab 100644 --- a/tests/system-tests/rdscore/internal/rdscoreconfig/config.go +++ b/tests/system-tests/rdscore/internal/rdscoreconfig/config.go @@ -376,6 +376,29 @@ type CoreConfig struct { EgressServiceDeploy2NodeSelector EnvMapString `yaml:"rdscore_egress_service_2_node_selector" envconfig:"ECO_RDSCORE_EGRESS_SVC_2_NODE_SELECTOR"` //nolint:lll,nolintlint EgressServiceDeploy2IPAddrPool string `yaml:"rdscore_egress_service_deploy_2_ipaddr_pool" envconfig:"ECO_RDSCORE_EGRESS_SVC_DEPLOY_2_IPADDR_POOL"` + EgressIPName string `yaml:"rdscore_egressip_name" envconfig:"ECO_RDSCORE_EGRESSIP_NAME"` + //nolint:lll,nolintlint + EgressIPDeploymentImage string `yaml:"rdscore_wlkd_egressip_image" envconfig:"ECO_RDSCORE_EGRESSIP_DEPLOY_IMG"` + //nolint:lll,nolintlint + NonEgressIPDeploymentImage string `yaml:"rdscore_wlkd_non_egressip_image" envconfig:"ECO_RDSCORE_NON_EGRESSIP_DEPLOY_IMG"` + //nolint:lll,nolintlint + EgressIPNodeOne string `yaml:"rdscore_wlkd_egressip_node_one" envconfig:"ECO_SYSTEM_RDSCORE_EGRESSIP_NODE_ONE"` + //nolint:lll,nolintlint + EgressIPNodeTwo string `yaml:"rdscore_wlkd_egressip_node_two" envconfig:"ECO_SYSTEM_RDSCORE_EGRESSIP_NODE_TWO"` + //nolint:lll,nolintlint + NonEgressIPNode string `yaml:"rdscore_wlkd_non_egressip_node" envconfig:"ECO_SYSTEM_RDSCORE_NON_EGRESSIP_NODE"` + //nolint:lll,nolintlint + EgressIPNamespaceLabel string `yaml:"rdscore_egressip_ns_label" envconfig:"ECO_RDSCORE_EGRESSIP_NS_LABEL"` + //nolint:lll,nolintlint + EgressIPNamespaceOne string `yaml:"rdscore_egressip_ns_one" envconfig:"ECO_RDSCORE_EGRESSIP_NS_ONE"` + //nolint:lll,nolintlint + EgressIPNamespaceTwo string `yaml:"rdscore_egressip_ns_two" envconfig:"ECO_RDSCORE_EGRESSIP_NS_TWO"` + //nolint:lll,nolintlint + EgressIPRemoteIP string `yaml:"rdscore_egressip_remote_ip" envconfig:"ECO_RDSCORE_EGRESSIP_REMOTE_IP"` + //nolint:lll,nolintlint + EgressIPRemoteIPv6 string `yaml:"rdscore_egressip_remote_ipv6" envconfig:"ECO_RDSCORE_EGRESSIP_REMOTE_IPV6"` + //nolint:lll,nolintlint + EgressIPTcpPort string `yaml:"rdscore_egressip_tcp_port_number" envconfig:"ECO_RDSCORE_EGRESSIP_TCP_PORT_NUMBER"` } // NewCoreConfig returns instance of CoreConfig config type. diff --git a/tests/system-tests/rdscore/internal/rdscoreconfig/default.yaml b/tests/system-tests/rdscore/internal/rdscoreconfig/default.yaml index 1b8abcf64..875889415 100644 --- a/tests/system-tests/rdscore/internal/rdscoreconfig/default.yaml +++ b/tests/system-tests/rdscore/internal/rdscoreconfig/default.yaml @@ -148,3 +148,9 @@ rdscore_egress_service_2_node_selector: node-role.kubernetes.io/worker: '' rdscore_egress_service_deploy_2_img: '' rdscore_egress_service_deploy_2_ipaddr_pool: 'rds-local-pool' +# EgressIP +rdscore_egressip_name: 'egressip-qe' +rdscore_wlkd_egressip_image: '' +rdscore_wlkd_egressip_node_one: '' +rdscore_wlkd_egressip_node_two: '' +rdscore_egressip_ns: rds-egressip diff --git a/tests/system-tests/rdscore/internal/rdscoreparams/const.go b/tests/system-tests/rdscore/internal/rdscoreparams/const.go index 7facb9271..0b113ce2d 100644 --- a/tests/system-tests/rdscore/internal/rdscoreparams/const.go +++ b/tests/system-tests/rdscore/internal/rdscoreparams/const.go @@ -39,4 +39,7 @@ const ( // MetalLBFRRContainerName name of the FRR container within a pod. MetalLBFRRContainerName = "frr" + + // EgressIPPodLabel label of the egressIP pods. + EgressIPPodLabel = "env=qe" ) diff --git a/tests/system-tests/rdscore/tests/00_validate_top_level.go b/tests/system-tests/rdscore/tests/00_validate_top_level.go index 067503148..8562c3036 100644 --- a/tests/system-tests/rdscore/tests/00_validate_top_level.go +++ b/tests/system-tests/rdscore/tests/00_validate_top_level.go @@ -33,6 +33,10 @@ var _ = Describe( Label("egress", "egress-etp-local"), reportxml.ID("76484"), rdscorecommon.VerifyEgressServiceWithLocalETP) + It("Verify Egress traffic works with egressIP applied for the external target", + Label("egressip", "egressip-single-ns"), reportxml.ID("75060"), + rdscorecommon.VerifyEgressIPWithTwoNamespacesIPv4) + It("Verifies workload reachable over BGP route", Label("frr"), reportxml.ID("76009"), rdscorecommon.ReachURLviaFRRroute)