From 89dc2ed423941a42aefaa613c88701ebdfc4c72d Mon Sep 17 00:00:00 2001 From: Christian Schlotter Date: Mon, 15 Jan 2024 12:16:46 +0100 Subject: [PATCH 1/2] test: stream logs for CPI and CSI to artifacts --- .../cloudprovider/csi/vsphere-csi-driver.yaml | 4 ++ ...luster-template-external-loadbalancer.yaml | 4 ++ templates/cluster-template-ignition.yaml | 4 ++ templates/cluster-template-node-ipam.yaml | 4 ++ templates/cluster-template-topology.yaml | 4 ++ templates/cluster-template.yaml | 4 ++ test/e2e/log_collector.go | 39 +++++++++++++++++++ test/e2e/quick_start_test.go | 26 +++++++------ test/go.mod | 2 +- test/go.sum | 4 +- 10 files changed, 80 insertions(+), 15 deletions(-) diff --git a/packaging/flavorgen/cloudprovider/csi/vsphere-csi-driver.yaml b/packaging/flavorgen/cloudprovider/csi/vsphere-csi-driver.yaml index 3f6277e758..7a13a95a63 100644 --- a/packaging/flavorgen/cloudprovider/csi/vsphere-csi-driver.yaml +++ b/packaging/flavorgen/cloudprovider/csi/vsphere-csi-driver.yaml @@ -193,6 +193,8 @@ apiVersion: apps/v1 metadata: name: vsphere-csi-controller namespace: vmware-system-csi + labels: + component: "vsphere-csi" spec: replicas: 3 strategy: @@ -579,6 +581,8 @@ apiVersion: apps/v1 metadata: name: vsphere-csi-node-windows namespace: vmware-system-csi + labels: + component: "vsphere-csi" spec: selector: matchLabels: diff --git a/templates/cluster-template-external-loadbalancer.yaml b/templates/cluster-template-external-loadbalancer.yaml index 2bba00492b..9bb0e58745 100644 --- a/templates/cluster-template-external-loadbalancer.yaml +++ b/templates/cluster-template-external-loadbalancer.yaml @@ -588,6 +588,8 @@ data: apiVersion: apps/v1 kind: Deployment metadata: + labels: + component: vsphere-csi name: vsphere-csi-controller namespace: vmware-system-csi spec: @@ -957,6 +959,8 @@ data: apiVersion: apps/v1 kind: DaemonSet metadata: + labels: + component: vsphere-csi name: vsphere-csi-node-windows namespace: vmware-system-csi spec: diff --git a/templates/cluster-template-ignition.yaml b/templates/cluster-template-ignition.yaml index f3076ca552..804aa07347 100644 --- a/templates/cluster-template-ignition.yaml +++ b/templates/cluster-template-ignition.yaml @@ -792,6 +792,8 @@ data: apiVersion: apps/v1 kind: Deployment metadata: + labels: + component: vsphere-csi name: vsphere-csi-controller namespace: vmware-system-csi spec: @@ -1161,6 +1163,8 @@ data: apiVersion: apps/v1 kind: DaemonSet metadata: + labels: + component: vsphere-csi name: vsphere-csi-node-windows namespace: vmware-system-csi spec: diff --git a/templates/cluster-template-node-ipam.yaml b/templates/cluster-template-node-ipam.yaml index d411852ea8..482d7f1ed7 100644 --- a/templates/cluster-template-node-ipam.yaml +++ b/templates/cluster-template-node-ipam.yaml @@ -713,6 +713,8 @@ data: apiVersion: apps/v1 kind: Deployment metadata: + labels: + component: vsphere-csi name: vsphere-csi-controller namespace: vmware-system-csi spec: @@ -1082,6 +1084,8 @@ data: apiVersion: apps/v1 kind: DaemonSet metadata: + labels: + component: vsphere-csi name: vsphere-csi-node-windows namespace: vmware-system-csi spec: diff --git a/templates/cluster-template-topology.yaml b/templates/cluster-template-topology.yaml index 54c1fc98be..cc9eeebfb8 100644 --- a/templates/cluster-template-topology.yaml +++ b/templates/cluster-template-topology.yaml @@ -487,6 +487,8 @@ data: apiVersion: apps/v1 kind: Deployment metadata: + labels: + component: vsphere-csi name: vsphere-csi-controller namespace: vmware-system-csi spec: @@ -856,6 +858,8 @@ data: apiVersion: apps/v1 kind: DaemonSet metadata: + labels: + component: vsphere-csi name: vsphere-csi-node-windows namespace: vmware-system-csi spec: diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index 6113fc7dd6..d6f60cce31 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -703,6 +703,8 @@ data: apiVersion: apps/v1 kind: Deployment metadata: + labels: + component: vsphere-csi name: vsphere-csi-controller namespace: vmware-system-csi spec: @@ -1072,6 +1074,8 @@ data: apiVersion: apps/v1 kind: DaemonSet metadata: + labels: + component: vsphere-csi name: vsphere-csi-node-windows namespace: vmware-system-csi spec: diff --git a/test/e2e/log_collector.go b/test/e2e/log_collector.go index 5068a31318..10b5c4bbfe 100644 --- a/test/e2e/log_collector.go +++ b/test/e2e/log_collector.go @@ -29,6 +29,7 @@ import ( "golang.org/x/crypto/ssh" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" + "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/controller-runtime/pkg/client" kinderrors "sigs.k8s.io/kind/pkg/errors" ) @@ -154,3 +155,41 @@ func readPrivateKey() ([]byte, error) { return os.ReadFile(filepath.Clean(privateKeyFilePath)) } + +func watchVSphereComponentLogsFunc(ctx context.Context, artifactFolder string) func(managementClusterProxy framework.ClusterProxy, workloadClusterNamespace, workloadClusterName string) { + return func(managementClusterProxy framework.ClusterProxy, workloadClusterNamespace, workloadClusterName string) { + workloadClusterProxy := managementClusterProxy.GetWorkloadCluster(ctx, workloadClusterNamespace, workloadClusterName) + // CPI + framework.WatchDaemonSetLogsByLabelSelector(ctx, framework.WatchDaemonSetLogsByLabelSelectorInput{ + GetLister: workloadClusterProxy.GetClient(), + Cache: workloadClusterProxy.GetCache(ctx), + ClientSet: workloadClusterProxy.GetClientSet(), + Labels: map[string]string{ + "component": "cloud-controller-manager", + }, + LogPath: filepath.Join(artifactFolder, "clusters", workloadClusterName, "logs"), + }) + + // CSI Deployment + framework.WatchDeploymentLogsByLabelSelector(ctx, framework.WatchDeploymentLogsByLabelSelectorInput{ + GetLister: workloadClusterProxy.GetClient(), + Cache: workloadClusterProxy.GetCache(ctx), + ClientSet: workloadClusterProxy.GetClientSet(), + Labels: map[string]string{ + "component": "vsphere-csi", + }, + LogPath: filepath.Join(artifactFolder, "clusters", workloadClusterName, "logs"), + }) + + // CSI Daemonset + framework.WatchDaemonSetLogsByLabelSelector(ctx, framework.WatchDaemonSetLogsByLabelSelectorInput{ + GetLister: workloadClusterProxy.GetClient(), + Cache: workloadClusterProxy.GetCache(ctx), + ClientSet: workloadClusterProxy.GetClientSet(), + Labels: map[string]string{ + "component": "vsphere-csi", + }, + LogPath: filepath.Join(artifactFolder, "clusters", workloadClusterName, "logs"), + }) + } +} diff --git a/test/e2e/quick_start_test.go b/test/e2e/quick_start_test.go index cf49913240..de530f86b8 100644 --- a/test/e2e/quick_start_test.go +++ b/test/e2e/quick_start_test.go @@ -37,12 +37,13 @@ var _ = Describe("Cluster Creation using Cluster API quick-start test", func() { var _ = Describe("ClusterClass Creation using Cluster API quick-start test [PR-Blocking] [ClusterClass]", func() { capi_e2e.QuickStartSpec(ctx, func() capi_e2e.QuickStartSpecInput { return capi_e2e.QuickStartSpecInput{ - E2EConfig: e2eConfig, - ClusterctlConfigPath: clusterctlConfigPath, - BootstrapClusterProxy: bootstrapClusterProxy, - ArtifactFolder: artifactFolder, - SkipCleanup: skipCleanup, - Flavor: ptr.To("topology"), + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + Flavor: ptr.To("topology"), + PostMachinesProvisioned: watchVSphereComponentLogsFunc(ctx, artifactFolder), } }) }) @@ -50,12 +51,13 @@ var _ = Describe("ClusterClass Creation using Cluster API quick-start test [PR-B var _ = Describe("Cluster creation with [Ignition] bootstrap [PR-Blocking]", func() { capi_e2e.QuickStartSpec(ctx, func() capi_e2e.QuickStartSpecInput { return capi_e2e.QuickStartSpecInput{ - E2EConfig: e2eConfig, - ClusterctlConfigPath: clusterctlConfigPath, - BootstrapClusterProxy: bootstrapClusterProxy, - ArtifactFolder: artifactFolder, - SkipCleanup: skipCleanup, - Flavor: ptr.To("ignition"), + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + Flavor: ptr.To("ignition"), + PostMachinesProvisioned: watchVSphereComponentLogsFunc(ctx, artifactFolder), } }) }) diff --git a/test/go.mod b/test/go.mod index 759e133e6d..611134a463 100644 --- a/test/go.mod +++ b/test/go.mod @@ -4,7 +4,7 @@ go 1.20 replace sigs.k8s.io/cluster-api => sigs.k8s.io/cluster-api v1.6.0 -replace sigs.k8s.io/cluster-api/test => sigs.k8s.io/cluster-api/test v1.6.1-0.20240108181656-eb46d086f417 +replace sigs.k8s.io/cluster-api/test => sigs.k8s.io/cluster-api/test v1.6.1-0.20240112150242-70716332b487 replace sigs.k8s.io/cluster-api-provider-vsphere => ../ diff --git a/test/go.sum b/test/go.sum index 860e10c4d6..3a8add6147 100644 --- a/test/go.sum +++ b/test/go.sum @@ -1115,8 +1115,8 @@ rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/cluster-api v1.6.0 h1:2bhVSnUbtWI8taCjd9lGiHExsRUpKf7Z1fXqi/IwYx4= sigs.k8s.io/cluster-api v1.6.0/go.mod h1:LB7u/WxiWj4/bbpHNOa1oQ8nq0MQ5iYlD0pGfRSBGLI= -sigs.k8s.io/cluster-api/test v1.6.1-0.20240108181656-eb46d086f417 h1:aI5QL5q0eT20HyA36LQfpkTVu7jdM9ssmNzBnA+gRpo= -sigs.k8s.io/cluster-api/test v1.6.1-0.20240108181656-eb46d086f417/go.mod h1:hsaaOsAftllhnRZ6l6izUvgGswlqO8Gm4EQS96+mnC0= +sigs.k8s.io/cluster-api/test v1.6.1-0.20240112150242-70716332b487 h1:72l+q8XVnAtW74R0slvFDljpjjtK+PJwZYSPphTlNRg= +sigs.k8s.io/cluster-api/test v1.6.1-0.20240112150242-70716332b487/go.mod h1:hsaaOsAftllhnRZ6l6izUvgGswlqO8Gm4EQS96+mnC0= sigs.k8s.io/controller-runtime v0.9.0/go.mod h1:TgkfvrhhEw3PlI0BRL/5xM+89y3/yc0ZDfdbTl84si8= sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= From 2698764363c3a5f0d1cc2cc1bfd358358ea0cc50 Mon Sep 17 00:00:00 2001 From: Christian Schlotter Date: Mon, 15 Jan 2024 14:05:14 +0100 Subject: [PATCH 2/2] fix --- test/e2e/anti_affinity_test.go | 3 ++ test/e2e/dhcp_overrides_test.go | 4 +- test/e2e/gpu_pci_passthrough_test.go | 3 ++ test/e2e/hardware_upgrade_test.go | 3 ++ test/e2e/log_collector.go | 70 +++++++++++++++------------- test/e2e/node_labeling_test.go | 3 ++ test/e2e/ownerreference_test.go | 2 + test/e2e/storage_policy_test.go | 5 +- 8 files changed, 58 insertions(+), 35 deletions(-) diff --git a/test/e2e/anti_affinity_test.go b/test/e2e/anti_affinity_test.go index aedf2b68ca..3a3468cadc 100644 --- a/test/e2e/anti_affinity_test.go +++ b/test/e2e/anti_affinity_test.go @@ -105,6 +105,9 @@ func VerifyAntiAffinity(ctx context.Context, input AntiAffinitySpecInput) { WaitForClusterIntervals: input.Global.E2EConfig.GetIntervals("", "wait-cluster"), WaitForControlPlaneIntervals: input.Global.E2EConfig.GetIntervals("", "wait-control-plane"), WaitForMachineDeployments: input.Global.E2EConfig.GetIntervals("", "wait-worker-nodes"), + PostMachinesProvisioned: func() { + watchVSphereComponentLogs(ctx, artifactFolder, input.Global.BootstrapClusterProxy, namespace.Name, clusterName) + }, }, clusterResources) vsphereCluster := FetchVSphereClusterObject(ctx, input.Global.BootstrapClusterProxy, client.ObjectKey{ diff --git a/test/e2e/dhcp_overrides_test.go b/test/e2e/dhcp_overrides_test.go index 3111eb1819..bd1f7e9268 100644 --- a/test/e2e/dhcp_overrides_test.go +++ b/test/e2e/dhcp_overrides_test.go @@ -85,7 +85,9 @@ var _ = Describe("DHCPOverrides configuration test", func() { WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), - }, clusterResources) + PostMachinesProvisioned: func() { + watchVSphereComponentLogs(ctx, artifactFolder, bootstrapClusterProxy, namespace.Name, clusterName) + }}, clusterResources) list := getVSphereVMsForCluster(clusterName, namespace.Name) for _, vm := range list.Items { diff --git a/test/e2e/gpu_pci_passthrough_test.go b/test/e2e/gpu_pci_passthrough_test.go index 2d730860b1..8b7b430e8c 100644 --- a/test/e2e/gpu_pci_passthrough_test.go +++ b/test/e2e/gpu_pci_passthrough_test.go @@ -61,6 +61,9 @@ var _ = Describe("Cluster creation with GPU devices as PCI passthrough [speciali WaitForClusterIntervals: e2eConfig.GetIntervals("", "wait-cluster"), WaitForControlPlaneIntervals: e2eConfig.GetIntervals("", "wait-control-plane"), WaitForMachineDeployments: e2eConfig.GetIntervals("", "wait-worker-nodes"), + PostMachinesProvisioned: func() { + watchVSphereComponentLogs(ctx, artifactFolder, bootstrapClusterProxy, namespace.Name, clusterName) + }, }, &clusterctl.ApplyClusterTemplateAndWaitResult{}) By("Verifying that the PCI device is attached to the worker node") diff --git a/test/e2e/hardware_upgrade_test.go b/test/e2e/hardware_upgrade_test.go index e1c4f7d140..a2b3d4f65d 100644 --- a/test/e2e/hardware_upgrade_test.go +++ b/test/e2e/hardware_upgrade_test.go @@ -96,6 +96,9 @@ func VerifyHardwareUpgrade(ctx context.Context, input HardwareUpgradeSpecInput) WaitForClusterIntervals: input.Global.E2EConfig.GetIntervals(specName, "wait-cluster"), WaitForControlPlaneIntervals: input.Global.E2EConfig.GetIntervals(specName, "wait-control-plane"), WaitForMachineDeployments: input.Global.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), + PostMachinesProvisioned: func() { + watchVSphereComponentLogs(ctx, artifactFolder, bootstrapClusterProxy, namespace.Name, clusterName) + }, }, clusterResources) Byf("Fetching the VSphereVM objects for the cluster %s", clusterName) diff --git a/test/e2e/log_collector.go b/test/e2e/log_collector.go index 10b5c4bbfe..f1b07dd755 100644 --- a/test/e2e/log_collector.go +++ b/test/e2e/log_collector.go @@ -156,40 +156,44 @@ func readPrivateKey() ([]byte, error) { return os.ReadFile(filepath.Clean(privateKeyFilePath)) } +func watchVSphereComponentLogs(ctx context.Context, artifactFolder string, managementClusterProxy framework.ClusterProxy, workloadClusterNamespace, workloadClusterName string) { + workloadClusterProxy := managementClusterProxy.GetWorkloadCluster(ctx, workloadClusterNamespace, workloadClusterName) + // CPI + framework.WatchDaemonSetLogsByLabelSelector(ctx, framework.WatchDaemonSetLogsByLabelSelectorInput{ + GetLister: workloadClusterProxy.GetClient(), + Cache: workloadClusterProxy.GetCache(ctx), + ClientSet: workloadClusterProxy.GetClientSet(), + Labels: map[string]string{ + "component": "cloud-controller-manager", + }, + LogPath: filepath.Join(artifactFolder, "clusters", workloadClusterName, "logs"), + }) + + // CSI Deployment + framework.WatchDeploymentLogsByLabelSelector(ctx, framework.WatchDeploymentLogsByLabelSelectorInput{ + GetLister: workloadClusterProxy.GetClient(), + Cache: workloadClusterProxy.GetCache(ctx), + ClientSet: workloadClusterProxy.GetClientSet(), + Labels: map[string]string{ + "component": "vsphere-csi", + }, + LogPath: filepath.Join(artifactFolder, "clusters", workloadClusterName, "logs"), + }) + + // CSI Daemonset + framework.WatchDaemonSetLogsByLabelSelector(ctx, framework.WatchDaemonSetLogsByLabelSelectorInput{ + GetLister: workloadClusterProxy.GetClient(), + Cache: workloadClusterProxy.GetCache(ctx), + ClientSet: workloadClusterProxy.GetClientSet(), + Labels: map[string]string{ + "component": "vsphere-csi", + }, + LogPath: filepath.Join(artifactFolder, "clusters", workloadClusterName, "logs"), + }) +} + func watchVSphereComponentLogsFunc(ctx context.Context, artifactFolder string) func(managementClusterProxy framework.ClusterProxy, workloadClusterNamespace, workloadClusterName string) { return func(managementClusterProxy framework.ClusterProxy, workloadClusterNamespace, workloadClusterName string) { - workloadClusterProxy := managementClusterProxy.GetWorkloadCluster(ctx, workloadClusterNamespace, workloadClusterName) - // CPI - framework.WatchDaemonSetLogsByLabelSelector(ctx, framework.WatchDaemonSetLogsByLabelSelectorInput{ - GetLister: workloadClusterProxy.GetClient(), - Cache: workloadClusterProxy.GetCache(ctx), - ClientSet: workloadClusterProxy.GetClientSet(), - Labels: map[string]string{ - "component": "cloud-controller-manager", - }, - LogPath: filepath.Join(artifactFolder, "clusters", workloadClusterName, "logs"), - }) - - // CSI Deployment - framework.WatchDeploymentLogsByLabelSelector(ctx, framework.WatchDeploymentLogsByLabelSelectorInput{ - GetLister: workloadClusterProxy.GetClient(), - Cache: workloadClusterProxy.GetCache(ctx), - ClientSet: workloadClusterProxy.GetClientSet(), - Labels: map[string]string{ - "component": "vsphere-csi", - }, - LogPath: filepath.Join(artifactFolder, "clusters", workloadClusterName, "logs"), - }) - - // CSI Daemonset - framework.WatchDaemonSetLogsByLabelSelector(ctx, framework.WatchDaemonSetLogsByLabelSelectorInput{ - GetLister: workloadClusterProxy.GetClient(), - Cache: workloadClusterProxy.GetCache(ctx), - ClientSet: workloadClusterProxy.GetClientSet(), - Labels: map[string]string{ - "component": "vsphere-csi", - }, - LogPath: filepath.Join(artifactFolder, "clusters", workloadClusterName, "logs"), - }) + watchVSphereComponentLogs(ctx, artifactFolder, managementClusterProxy, workloadClusterNamespace, workloadClusterName) } } diff --git a/test/e2e/node_labeling_test.go b/test/e2e/node_labeling_test.go index 97c3d45e54..b059f2c9db 100644 --- a/test/e2e/node_labeling_test.go +++ b/test/e2e/node_labeling_test.go @@ -84,6 +84,9 @@ func VerifyNodeLabeling(ctx context.Context, input NodeLabelingSpecInput) { WaitForClusterIntervals: input.Global.E2EConfig.GetIntervals("", "wait-cluster"), WaitForControlPlaneIntervals: input.Global.E2EConfig.GetIntervals("", "wait-control-plane"), WaitForMachineDeployments: input.Global.E2EConfig.GetIntervals("", "wait-worker-nodes"), + PostMachinesProvisioned: func() { + watchVSphereComponentLogs(ctx, artifactFolder, input.Global.BootstrapClusterProxy, namespace.Name, clusterName) + }, }, clusterResources) workloadProxy := input.Global.BootstrapClusterProxy.GetWorkloadCluster(ctx, namespace.Name, clusterResources.Cluster.Name) diff --git a/test/e2e/ownerreference_test.go b/test/e2e/ownerreference_test.go index 89af392ebb..810ff71287 100644 --- a/test/e2e/ownerreference_test.go +++ b/test/e2e/ownerreference_test.go @@ -80,6 +80,8 @@ var _ = Describe("OwnerReference checks with FailureDomains and ClusterIdentity" VSphereKubernetesReferenceAssertions, VSphereReferenceAssertions, ) + + watchVSphereComponentLogs(ctx, artifactFolder, bootstrapClusterProxy, namespace, clusterName) }, } }) diff --git a/test/e2e/storage_policy_test.go b/test/e2e/storage_policy_test.go index 4875fa1ec7..ac98d8f80c 100644 --- a/test/e2e/storage_policy_test.go +++ b/test/e2e/storage_policy_test.go @@ -85,7 +85,7 @@ func VerifyStoragePolicy(ctx context.Context, input StoragePolicySpecInput) { By("creating a workload cluster") configCluster := defaultConfigCluster(clusterName, namespace.Name, specName, 1, 0, GlobalInput{ - BootstrapClusterProxy: bootstrapClusterProxy, + BootstrapClusterProxy: input.Global.BootstrapClusterProxy, ClusterctlConfigPath: clusterctlConfigPath, E2EConfig: e2eConfig, ArtifactFolder: artifactFolder, @@ -97,6 +97,9 @@ func VerifyStoragePolicy(ctx context.Context, input StoragePolicySpecInput) { WaitForClusterIntervals: input.Global.E2EConfig.GetIntervals("", "wait-cluster"), WaitForControlPlaneIntervals: input.Global.E2EConfig.GetIntervals("", "wait-control-plane"), WaitForMachineDeployments: input.Global.E2EConfig.GetIntervals("", "wait-worker-nodes"), + PostMachinesProvisioned: func() { + watchVSphereComponentLogs(ctx, artifactFolder, input.Global.BootstrapClusterProxy, namespace.Name, clusterName) + }, }, clusterResources) pbmClient, err := pbm.NewClient(ctx, input.Client.Client)