Skip to content

Commit

Permalink
Merge pull request #10715 from zhanggbj/e2e_support_extra_labels
Browse files Browse the repository at this point in the history 
🌱 Enhance E2E Tests to Filter Out Extra Provider-Specific Metadata
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot authored Jul 4, 2024
2 parents a7f5043 + ef5323b commit 5a0c7a9
Showing 1 changed file with 48 additions and 25 deletions.
73 changes: 48 additions & 25 deletions test/e2e/clusterclass_rollout.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,12 @@ type ClusterClassRolloutSpecInput struct {
// Allows to inject a function to be run after test namespace is created.
// If not specified, this is a no-op.
PostNamespaceCreated func(managementClusterProxy framework.ClusterProxy, workloadClusterNamespace string)

// FilterMetadataBeforeValidation allows filtering out labels and annotations of Machines, InfraMachines,
// BootstrapConfigs and Nodes before we validate them.
// This can be e.g. used to filter out additional infrastructure provider specific labels that would
// otherwise lead to a failed test.
FilterMetadataBeforeValidation func(object client.Object) clusterv1.ObjectMeta
}

// ClusterClassRolloutSpec implements a test that verifies the ClusterClass rollout behavior.
Expand Down Expand Up @@ -111,6 +117,14 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion))
Expect(input.E2EConfig.Variables).To(HaveValidVersion(input.E2EConfig.GetVariable(KubernetesVersion)))

// Set a default function to ensure that FilterMetadataBeforeValidation has a default behavior for
// filtering metadata if it is not specified by infrastructure provider.
if input.FilterMetadataBeforeValidation == nil {
input.FilterMetadataBeforeValidation = func(obj client.Object) clusterv1.ObjectMeta {
return clusterv1.ObjectMeta{Labels: obj.GetLabels(), Annotations: obj.GetAnnotations()}
}
}

// Set up a Namespace where to host objects for this spec and create a watcher for the namespace events.
namespace, cancelWatches = framework.SetupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, input.PostNamespaceCreated)
clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult)
Expand Down Expand Up @@ -142,7 +156,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
WaitForMachinePools: input.E2EConfig.GetIntervals(specName, "wait-machine-pool-nodes"),
}, clusterResources)
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass)
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass, input.FilterMetadataBeforeValidation)

By("Rolling out changes to control plane, MachineDeployments, and MachinePools (in-place)")
machinesBeforeUpgrade := getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster)
Expand Down Expand Up @@ -218,7 +232,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
machinesAfterUpgrade := getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster)
g.Expect(machinesAfterUpgrade.Equal(machinesBeforeUpgrade)).To(BeTrue(), "Machines must not be replaced through in-place rollout")
}, 30*time.Second, 1*time.Second).Should(Succeed())
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass)
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass, input.FilterMetadataBeforeValidation)

By("Rolling out changes to control plane, MachineDeployments, and MachinePools (rollout)")
machinesBeforeUpgrade = getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster)
Expand Down Expand Up @@ -259,7 +273,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
machinesAfterUpgrade := getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster)
g.Expect(machinesAfterUpgrade.HasAny(machinesBeforeUpgrade.UnsortedList()...)).To(BeFalse(), "All Machines must be replaced through rollout")
}, input.E2EConfig.GetIntervals(specName, "wait-control-plane")...).Should(Succeed())
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass)
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass, input.FilterMetadataBeforeValidation)

By("Rolling out control plane and MachineDeployment (rolloutAfter)")
machinesBeforeUpgrade = getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster)
Expand Down Expand Up @@ -287,7 +301,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
machinesAfterUpgrade := getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster)
g.Expect(machinesAfterUpgrade.HasAny(machinesBeforeUpgrade.UnsortedList()...)).To(BeFalse(), "All Machines must be replaced through rollout with rolloutAfter")
}, input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade")...).Should(Succeed())
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass)
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass, input.FilterMetadataBeforeValidation)

By("PASSED!")
})
Expand All @@ -299,7 +313,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
}

// assertClusterObjects asserts cluster objects by checking that all objects have the right labels, annotations and selectors.
func assertClusterObjects(ctx context.Context, clusterProxy framework.ClusterProxy, cluster *clusterv1.Cluster, clusterClass *clusterv1.ClusterClass) {
func assertClusterObjects(ctx context.Context, clusterProxy framework.ClusterProxy, cluster *clusterv1.Cluster, clusterClass *clusterv1.ClusterClass, filterMetadataBeforeValidation func(object client.Object) clusterv1.ObjectMeta) {
By("Checking cluster objects have the right labels, annotations and selectors")

Eventually(func(g Gomega) {
Expand All @@ -312,12 +326,12 @@ func assertClusterObjects(ctx context.Context, clusterProxy framework.ClusterPro

// ControlPlane
assertControlPlane(g, clusterClassObjects, clusterObjects, cluster, clusterClass)
assertControlPlaneMachines(g, clusterObjects, cluster)
assertControlPlaneMachines(g, clusterObjects, cluster, filterMetadataBeforeValidation)

// MachineDeployments
assertMachineDeployments(g, clusterClassObjects, clusterObjects, cluster, clusterClass)
assertMachineSets(g, clusterObjects, cluster)
assertMachineSetsMachines(g, clusterObjects, cluster)
assertMachineSetsMachines(g, clusterObjects, cluster, filterMetadataBeforeValidation)

// MachinePools
assertMachinePools(g, clusterClassObjects, clusterObjects, cluster, clusterClass)
Expand Down Expand Up @@ -430,13 +444,14 @@ func assertControlPlane(g Gomega, clusterClassObjects clusterClassObjects, clust
))
}

func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster *clusterv1.Cluster) {
func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster *clusterv1.Cluster, filterMetadataBeforeValidation func(object client.Object) clusterv1.ObjectMeta) {
controlPlaneMachineTemplateMetadata := mustMetadata(contract.ControlPlane().MachineTemplate().Metadata().Get(clusterObjects.ControlPlane))
controlPlaneInfrastructureMachineTemplateTemplateMetadata := mustMetadata(contract.InfrastructureMachineTemplate().Template().Metadata().Get(clusterObjects.ControlPlaneInfrastructureMachineTemplate))

for _, machine := range clusterObjects.ControlPlaneMachines {
// ControlPlane Machine.metadata
g.Expect(machine.Labels).To(BeEquivalentTo(
machineMetadata := filterMetadataBeforeValidation(machine)
g.Expect(machineMetadata.Labels).To(BeEquivalentTo(
union(
map[string]string{
clusterv1.ClusterNameLabel: cluster.Name,
Expand All @@ -449,17 +464,18 @@ func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster
))
g.Expect(
union(
machine.Annotations,
machineMetadata.Annotations,
).without(g, controlplanev1.KubeadmClusterConfigurationAnnotation),
).To(BeEquivalentTo(
controlPlaneMachineTemplateMetadata.Annotations,
))

// ControlPlane Machine InfrastructureMachine.metadata
infrastructureMachine := clusterObjects.InfrastructureMachineByMachine[machine.Name]
infrastructureMachineMetadata := filterMetadataBeforeValidation(infrastructureMachine)
controlPlaneMachineTemplateInfrastructureRef, err := contract.ControlPlane().MachineTemplate().InfrastructureRef().Get(clusterObjects.ControlPlane)
g.Expect(err).ToNot(HaveOccurred())
g.Expect(infrastructureMachine.GetLabels()).To(BeEquivalentTo(
g.Expect(infrastructureMachineMetadata.Labels).To(BeEquivalentTo(
union(
map[string]string{
clusterv1.ClusterNameLabel: cluster.Name,
Expand All @@ -471,7 +487,7 @@ func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster
controlPlaneInfrastructureMachineTemplateTemplateMetadata.Labels,
),
))
g.Expect(infrastructureMachine.GetAnnotations()).To(BeEquivalentTo(
g.Expect(infrastructureMachineMetadata.Annotations).To(BeEquivalentTo(
union(
map[string]string{
clusterv1.TemplateClonedFromGroupKindAnnotation: groupKind(controlPlaneMachineTemplateInfrastructureRef),
Expand All @@ -484,7 +500,8 @@ func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster

// ControlPlane Machine BootstrapConfig.metadata
bootstrapConfig := clusterObjects.BootstrapConfigByMachine[machine.Name]
g.Expect(bootstrapConfig.GetLabels()).To(BeEquivalentTo(
bootstrapConfigMetadata := filterMetadataBeforeValidation(bootstrapConfig)
g.Expect(bootstrapConfigMetadata.Labels).To(BeEquivalentTo(
union(
map[string]string{
clusterv1.ClusterNameLabel: cluster.Name,
Expand All @@ -497,16 +514,18 @@ func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster
))
g.Expect(
union(
bootstrapConfig.GetAnnotations(),
bootstrapConfigMetadata.Annotations,
).without(g, clusterv1.MachineCertificatesExpiryDateAnnotation),
).To(BeEquivalentTo(
controlPlaneMachineTemplateMetadata.Annotations,
))

// ControlPlane Machine Node.metadata
node := clusterObjects.NodesByMachine[machine.Name]
for k, v := range getManagedLabels(machine.Labels) {
g.Expect(node.GetLabels()).To(HaveKeyWithValue(k, v))
nodeMetadata := filterMetadataBeforeValidation(node)

for k, v := range getManagedLabels(machineMetadata.Labels) {
g.Expect(nodeMetadata.Labels).To(HaveKeyWithValue(k, v))
}
}
}
Expand Down Expand Up @@ -790,7 +809,7 @@ func assertMachineSets(g Gomega, clusterObjects clusterObjects, cluster *cluster
}
}

func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster *clusterv1.Cluster) {
func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster *clusterv1.Cluster, filterMetadataBeforeValidation func(object client.Object) clusterv1.ObjectMeta) {
for _, machineDeployment := range clusterObjects.MachineDeployments {
mdTopology := getMDTopology(cluster, machineDeployment)
infrastructureMachineTemplate := clusterObjects.InfrastructureMachineTemplateByMachineDeployment[machineDeployment.Name]
Expand All @@ -802,8 +821,9 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster
machineTemplateHash := machineSet.Labels[clusterv1.MachineDeploymentUniqueLabel]

for _, machine := range clusterObjects.MachinesByMachineSet[machineSet.Name] {
machineMetadata := filterMetadataBeforeValidation(machine)
// MachineDeployment MachineSet Machine.metadata
g.Expect(machine.Labels).To(BeEquivalentTo(
g.Expect(machineMetadata.Labels).To(BeEquivalentTo(
union(
map[string]string{
clusterv1.ClusterNameLabel: cluster.Name,
Expand All @@ -816,13 +836,14 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster
machineSet.Spec.Template.Labels,
),
))
g.Expect(machine.Annotations).To(BeEquivalentTo(
g.Expect(machineMetadata.Annotations).To(BeEquivalentTo(
machineSet.Spec.Template.Annotations,
))

// MachineDeployment MachineSet Machine InfrastructureMachine.metadata
infrastructureMachine := clusterObjects.InfrastructureMachineByMachine[machine.Name]
g.Expect(infrastructureMachine.GetLabels()).To(BeEquivalentTo(
infrastructureMachineMetadata := filterMetadataBeforeValidation(infrastructureMachine)
g.Expect(infrastructureMachineMetadata.Labels).To(BeEquivalentTo(
union(
map[string]string{
clusterv1.ClusterNameLabel: cluster.Name,
Expand All @@ -836,7 +857,7 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster
infrastructureMachineTemplateTemplateMetadata.Labels,
),
))
g.Expect(infrastructureMachine.GetAnnotations()).To(BeEquivalentTo(
g.Expect(infrastructureMachineMetadata.Annotations).To(BeEquivalentTo(
union(
map[string]string{
clusterv1.TemplateClonedFromGroupKindAnnotation: groupKind(&machineSet.Spec.Template.Spec.InfrastructureRef),
Expand All @@ -849,7 +870,8 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster

// MachineDeployment MachineSet Machine BootstrapConfig.metadata
bootstrapConfig := clusterObjects.BootstrapConfigByMachine[machine.Name]
g.Expect(bootstrapConfig.GetLabels()).To(BeEquivalentTo(
bootstrapConfigMetadata := filterMetadataBeforeValidation(bootstrapConfig)
g.Expect(bootstrapConfigMetadata.Labels).To(BeEquivalentTo(
union(
map[string]string{
clusterv1.ClusterNameLabel: cluster.Name,
Expand All @@ -863,7 +885,7 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster
bootstrapConfigTemplateTemplateMetadata.Labels,
),
))
g.Expect(bootstrapConfig.GetAnnotations()).To(BeEquivalentTo(
g.Expect(bootstrapConfigMetadata.Annotations).To(BeEquivalentTo(
union(
map[string]string{
clusterv1.TemplateClonedFromGroupKindAnnotation: groupKind(machineSet.Spec.Template.Spec.Bootstrap.ConfigRef),
Expand All @@ -876,8 +898,9 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster

// MachineDeployment MachineSet Machine Node.metadata
node := clusterObjects.NodesByMachine[machine.Name]
for k, v := range getManagedLabels(machine.Labels) {
g.Expect(node.GetLabels()).To(HaveKeyWithValue(k, v))
nodeMetadata := filterMetadataBeforeValidation(node)
for k, v := range getManagedLabels(machineMetadata.Labels) {
g.Expect(nodeMetadata.Labels).To(HaveKeyWithValue(k, v))
}
}
}
Expand Down

0 comments on commit 5a0c7a9

Please sign in to comment.