From e3b2dc3bd7457c986f9c40ccb623d5f96ad71ff6 Mon Sep 17 00:00:00 2001 From: Zhongcheng Lao Date: Wed, 13 Sep 2023 13:35:10 +0800 Subject: [PATCH] Update CSI driver to v3.1.0 --- packaging/flavorgen/cloudprovider/csi.go | 710 +++++++++++++++--- packaging/flavorgen/flavors/crs/csi.go | 99 ++- .../flavors/crs/types/cloudprovider_types.go | 2 + ...luster-template-external-loadbalancer.yaml | 598 +++++++++++---- templates/cluster-template-ignition.yaml | 598 +++++++++++---- templates/cluster-template-node-ipam.yaml | 598 +++++++++++---- templates/cluster-template-topology.yaml | 598 +++++++++++---- templates/cluster-template.yaml | 598 +++++++++++---- 8 files changed, 2892 insertions(+), 909 deletions(-) diff --git a/packaging/flavorgen/cloudprovider/csi.go b/packaging/flavorgen/cloudprovider/csi.go index 0d317f6188..42701d31b4 100644 --- a/packaging/flavorgen/cloudprovider/csi.go +++ b/packaging/flavorgen/cloudprovider/csi.go @@ -20,30 +20,60 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" - storagev1beta1 "k8s.io/api/storage/v1beta1" + storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/crs/types" ) -// NOTE: the contents of this file are derived from https://github.com/kubernetes-sigs/vsphere-csi-driver/tree/master/manifests/1.14 +// NOTE: the contents of this file are derived from https://github.com/kubernetes-sigs/vsphere-csi-driver/tree/master/manifests/vanilla const ( - DefaultCSIControllerImage = "gcr.io/cloud-provider-vsphere/csi/release/driver:v2.1.0" - DefaultCSINodeDriverImage = "gcr.io/cloud-provider-vsphere/csi/release/driver:v2.1.0" - DefaultCSIAttacherImage = "quay.io/k8scsi/csi-attacher:v3.0.0" - DefaultCSIProvisionerImage = "quay.io/k8scsi/csi-provisioner:v2.0.0" - DefaultCSIMetadataSyncerImage = "gcr.io/cloud-provider-vsphere/csi/release/syncer:v2.1.0" - DefaultCSILivenessProbeImage = "quay.io/k8scsi/livenessprobe:v2.1.0" - DefaultCSIRegistrarImage = "quay.io/k8scsi/csi-node-driver-registrar:v2.0.1" - CSINamespace = metav1.NamespaceSystem - CSIControllerName = "vsphere-csi-controller" - CSIFeatureStateConfigMapName = "internal-feature-states.csi.vsphere.vmware.com" + DefaultCSIControllerImage = "gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0" + DefaultCSINodeDriverImage = "gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0" + DefaultCSIAttacherImage = "registry.k8s.io/sig-storage/csi-attacher:v4.3.0" + DefaultCSIResizerImage = "registry.k8s.io/sig-storage/csi-resizer:v1.8.0" + DefaultCSIProvisionerImage = "registry.k8s.io/sig-storage/csi-provisioner:v3.5.0" + DefaultCSISnapshotterContainerImage = "registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2" + DefaultCSIMetadataSyncerImage = "gcr.io/cloud-provider-vsphere/csi/release/syncer:v3.1.0" + DefaultCSILivenessProbeImage = "registry.k8s.io/sig-storage/livenessprobe:v2.10.0" + DefaultCSIRegistrarImage = "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0" + CSINamespace = "vmware-system-csi" + CSIControllerName = "vsphere-csi-controller" + CSINodeDaemonSetName = "vsphere-csi-node" + CSIFeatureStateConfigMapName = "internal-feature-states.csi.vsphere.vmware.com" + + clusterRoleSuffix = "-cluster-role" + clusterRoleBindingSuffix = clusterRoleSuffix + "-binding" + roleSuffix = "-role" + roleBindingSuffix = roleSuffix + "-binding" + CSIControllerClusterRoleName = CSIControllerName + clusterRoleSuffix + CSIControllerRoleBindingName = CSIControllerName + clusterRoleBindingSuffix + CSINodeRoleName = CSINodeDaemonSetName + roleSuffix + CSINodeRoleBindingName = CSINodeDaemonSetName + roleBindingSuffix + CSINodeClusterRoleName = CSINodeDaemonSetName + clusterRoleSuffix + CSINodeClusterRoleBindingName = CSINodeDaemonSetName + clusterRoleBindingSuffix ) +func CSIDriverNamespace() *corev1.Namespace { + return &corev1.Namespace{ + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: CSINamespace, + }, + } +} + func CSIControllerServiceAccount() *corev1.ServiceAccount { return &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{ + Kind: "ServiceAccount", + APIVersion: corev1.SchemeGroupVersion.String(), + }, ObjectMeta: metav1.ObjectMeta{ Name: CSIControllerName, Namespace: CSINamespace, @@ -53,64 +83,113 @@ func CSIControllerServiceAccount() *corev1.ServiceAccount { func CSIControllerClusterRole() *rbacv1.ClusterRole { return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{ + Kind: "ClusterRole", + APIVersion: rbacv1.SchemeGroupVersion.String(), + }, ObjectMeta: metav1.ObjectMeta{ - Name: "vsphere-csi-controller-role", + Name: CSIControllerClusterRoleName, }, Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"storage.k8s.io"}, - Resources: []string{"csidrivers"}, - Verbs: []string{"create", "delete"}, - }, { APIGroups: []string{""}, - Resources: []string{"nodes", "pods", "secrets", "configmaps"}, + Resources: []string{"nodes", "pods"}, Verbs: []string{"get", "list", "watch"}, }, { APIGroups: []string{""}, - Resources: []string{"persistentvolumes"}, - Verbs: []string{"get", "list", "watch", "update", "create", "delete", "patch"}, + Resources: []string{"configmaps"}, + Verbs: []string{"get", "list", "watch", "create"}, }, { - APIGroups: []string{"storage.k8s.io"}, - Resources: []string{"volumeattachments"}, - Verbs: []string{"get", "list", "watch", "update", "patch"}, + APIGroups: []string{""}, + Resources: []string{"persistentvolumeclaims"}, + Verbs: []string{"get", "list", "watch", "update"}, }, { - APIGroups: []string{"storage.k8s.io"}, - Resources: []string{"volumeattachments/status"}, + APIGroups: []string{""}, + Resources: []string{"persistentvolumeclaims/status"}, Verbs: []string{"patch"}, }, { APIGroups: []string{""}, - Resources: []string{"persistentvolumeclaims"}, - Verbs: []string{"get", "list", "watch", "update"}, - }, - { - APIGroups: []string{"storage.k8s.io"}, - Resources: []string{"storageclasses", "csinodes"}, - Verbs: []string{"get", "list", "watch"}, + Resources: []string{"persistentvolumes"}, + Verbs: []string{"get", "list", "watch", "create", "update", "delete", "patch"}, }, { APIGroups: []string{""}, Resources: []string{"events"}, - Verbs: []string{"list", "watch", "create", "update", "patch"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch"}, }, { APIGroups: []string{"coordination.k8s.io"}, Resources: []string{"leases"}, Verbs: []string{"get", "watch", "list", "delete", "update", "create"}, }, + { + APIGroups: []string{"storage.k8s.io"}, + Resources: []string{"storageclasses", "csinodes"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"storage.k8s.io"}, + Resources: []string{"volumeattachments"}, + Verbs: []string{"get", "list", "watch", "patch"}, + }, + { + APIGroups: []string{"cns.vmware.com"}, + Resources: []string{"triggercsifullsyncs"}, + Verbs: []string{"create", "get", "update", "watch", "list"}, + }, + { + APIGroups: []string{"cns.vmware.com"}, + Resources: []string{"cnsvspherevolumemigrations"}, + Verbs: []string{"create", "get", "list", "watch", "update", "delete"}, + }, + { + APIGroups: []string{"cns.vmware.com"}, + Resources: []string{"cnsvolumeinfoes"}, + Verbs: []string{"create", "get", "list", "watch", "delete"}, + }, + { + APIGroups: []string{"apiextensions.k8s.io"}, + Resources: []string{"customresourcedefinitions"}, + Verbs: []string{"get", "create", "update"}, + }, + { + APIGroups: []string{"storage.k8s.io"}, + Resources: []string{"volumeattachments/status"}, + Verbs: []string{"patch"}, + }, + { + APIGroups: []string{"cns.vmware.com"}, + Resources: []string{"cnsvolumeoperationrequests"}, + Verbs: []string{"create", "get", "list", "update", "delete"}, + }, { APIGroups: []string{"snapshot.storage.k8s.io"}, Resources: []string{"volumesnapshots"}, Verbs: []string{"get", "list"}, }, + { + APIGroups: []string{"snapshot.storage.k8s.io"}, + Resources: []string{"volumesnapshotclasses"}, + Verbs: []string{"watch", "get", "list"}, + }, { APIGroups: []string{"snapshot.storage.k8s.io"}, Resources: []string{"volumesnapshotcontents"}, - Verbs: []string{"get", "list"}, + Verbs: []string{"create", "get", "list", "watch", "update", "delete", "patch"}, + }, + { + APIGroups: []string{"snapshot.storage.k8s.io"}, + Resources: []string{"volumesnapshotcontents/status"}, + Verbs: []string{"update", "patch"}, + }, + { + APIGroups: []string{"cns.vmware.com"}, + Resources: []string{"csinodetopologies"}, + Verbs: []string{"get", "update", "watch", "list"}, }, }, } @@ -118,8 +197,12 @@ func CSIControllerClusterRole() *rbacv1.ClusterRole { func CSIControllerClusterRoleBinding() *rbacv1.ClusterRoleBinding { return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{ + Kind: "ClusterRoleBinding", + APIVersion: rbacv1.SchemeGroupVersion.String(), + }, ObjectMeta: metav1.ObjectMeta{ - Name: "vsphere-csi-controller-binding", + Name: CSIControllerRoleBindingName, }, Subjects: []rbacv1.Subject{ { @@ -130,18 +213,128 @@ func CSIControllerClusterRoleBinding() *rbacv1.ClusterRoleBinding { }, RoleRef: rbacv1.RoleRef{ Kind: "ClusterRole", - Name: "vsphere-csi-controller-role", + Name: CSIControllerClusterRoleName, + APIGroup: "rbac.authorization.k8s.io", + }, + } +} + +func CSINodeDaemonSetServiceAccount() *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{ + Kind: "ServiceAccount", + APIVersion: corev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: CSINodeDaemonSetName, + Namespace: CSINamespace, + }, + } +} + +func CSINodeRole() *rbacv1.Role { + return &rbacv1.Role{ + TypeMeta: metav1.TypeMeta{ + Kind: "Role", + APIVersion: rbacv1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: CSINodeRoleName, + Namespace: CSINamespace, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"get", "list", "watch"}, + }, + }, + } +} + +func CSINodeRoleBinding() *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{ + Kind: "RoleBinding", + APIVersion: rbacv1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: CSINodeRoleBindingName, + Namespace: CSINamespace, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: CSINodeDaemonSetName, + Namespace: CSINamespace, + }, + }, + RoleRef: rbacv1.RoleRef{ + Kind: "Role", + Name: CSINodeRoleName, + APIGroup: "rbac.authorization.k8s.io", + }, + } +} + +func CSINodeClusterRole() *rbacv1.ClusterRole { + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{ + Kind: "ClusterRole", + APIVersion: rbacv1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: CSINodeClusterRoleName, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"cns.vmware.com"}, + Resources: []string{"csinodetopologies"}, + Verbs: []string{"create", "watch", "get", "patch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"nodes"}, + Verbs: []string{"get"}, + }, + }, + } +} + +func CSINodeClusterRoleBinding() *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{ + Kind: "ClusterRoleBinding", + APIVersion: rbacv1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: CSINodeClusterRoleBindingName, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: CSINodeDaemonSetName, + Namespace: CSINamespace, + }, + }, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: CSINodeClusterRoleName, APIGroup: "rbac.authorization.k8s.io", }, } } -func CSIDriver() *storagev1beta1.CSIDriver { - return &storagev1beta1.CSIDriver{ +func CSIDriver() *storagev1.CSIDriver { + return &storagev1.CSIDriver{ + TypeMeta: metav1.TypeMeta{ + Kind: "CSIDriver", + APIVersion: storagev1.SchemeGroupVersion.String(), + }, ObjectMeta: metav1.ObjectMeta{ Name: "csi.vsphere.vmware.com", }, - Spec: storagev1beta1.CSIDriverSpec{ + Spec: storagev1.CSIDriverSpec{ AttachRequired: boolPtr(true), PodInfoOnMount: boolPtr(false), }, @@ -150,28 +343,41 @@ func CSIDriver() *storagev1beta1.CSIDriver { func VSphereCSINodeDaemonSet(storageConfig *types.CPIStorageConfig) *appsv1.DaemonSet { return &appsv1.DaemonSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "DaemonSet", + APIVersion: appsv1.SchemeGroupVersion.String(), + }, ObjectMeta: metav1.ObjectMeta{ - Name: "vsphere-csi-node", + Name: CSINodeDaemonSetName, Namespace: CSINamespace, }, Spec: appsv1.DaemonSetSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - "app": "vsphere-csi-node", + "app": CSINodeDaemonSetName, }, }, UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ Type: appsv1.RollingUpdateDaemonSetStrategyType, + RollingUpdate: &appsv1.RollingUpdateDaemonSet{ + MaxUnavailable: newIntOrStringFromInt(1), + }, }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - "app": "vsphere-csi-node", + "app": CSINodeDaemonSetName, "role": "vsphere-csi", }, }, Spec: corev1.PodSpec{ - DNSPolicy: corev1.DNSDefault, + PriorityClassName: "system-node-critical", + NodeSelector: map[string]string{ + "kubernetes.io/os": "linux", + }, + ServiceAccountName: CSINodeDaemonSetName, + HostNetwork: true, + DNSPolicy: corev1.DNSClusterFirstWithHostNet, Containers: []corev1.Container{ NodeDriverRegistrarContainer(storageConfig.RegistrarImage), VSphereCSINodeContainer(storageConfig.NodeDriverImage), @@ -188,14 +394,6 @@ func VSphereCSINodeDaemonSet(storageConfig *types.CPIStorageConfig) *appsv1.Daem }, }, Volumes: []corev1.Volume{ - { - Name: "vsphere-config-volume", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "csi-vsphere-config", - }, - }, - }, { Name: "registration-dir", VolumeSource: corev1.VolumeSource{ @@ -209,7 +407,7 @@ func VSphereCSINodeDaemonSet(storageConfig *types.CPIStorageConfig) *appsv1.Daem Name: "plugin-dir", VolumeSource: corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ - Path: "/var/lib/kubelet/plugins/csi.vsphere.vmware.com/", + Path: "/var/lib/kubelet/plugins/csi.vsphere.vmware.com", Type: newHostPathType(string(corev1.HostPathDirectoryOrCreate)), }, }, @@ -231,6 +429,24 @@ func VSphereCSINodeDaemonSet(storageConfig *types.CPIStorageConfig) *appsv1.Daem }, }, }, + { + Name: "blocks-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/sys/block", + Type: newHostPathType(string(corev1.HostPathDirectory)), + }, + }, + }, + { + Name: "sys-devices-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/sys/devices", + Type: newHostPathType(string(corev1.HostPathDirectory)), + }, + }, + }, }, }, }, @@ -242,17 +458,6 @@ func NodeDriverRegistrarContainer(image string) corev1.Container { return corev1.Container{ Name: "node-driver-registrar", Image: image, - Lifecycle: &corev1.Lifecycle{ - PreStop: &corev1.LifecycleHandler{ - Exec: &corev1.ExecAction{ - Command: []string{ - "/bin/sh", - "-c", - "rm -rf /registration/csi.vsphere.vmware.com-reg.sock /csi/csi.sock", - }, - }, - }, - }, Args: []string{ "--v=5", "--csi-address=$(ADDRESS)", @@ -268,8 +473,17 @@ func NodeDriverRegistrarContainer(image string) corev1.Container { Value: "/var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock", }, }, - SecurityContext: &corev1.SecurityContext{ - Privileged: boolPtr(true), + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{ + "/csi-node-driver-registrar", + "--kubelet-registration-path=/var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock", + "--mode=kubelet-registration-probe", + }, + }, + }, + InitialDelaySeconds: 3, }, VolumeMounts: []corev1.VolumeMount{ { @@ -288,11 +502,27 @@ func VSphereCSINodeContainer(image string) corev1.Container { return corev1.Container{ Name: "vsphere-csi-node", Image: image, + Args: []string{ + "--fss-name=internal-feature-states.csi.vsphere.vmware.com", + "--fss-namespace=$(CSI_NAMESPACE)", + }, Env: []corev1.EnvVar{ + { + Name: "NODE_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, { Name: "CSI_ENDPOINT", Value: "unix:///csi/csi.sock", }, + { + Name: "MAX_VOLUMES_PER_NODE", + Value: "59", + }, { Name: "X_CSI_MODE", Value: "node", @@ -302,25 +532,29 @@ func VSphereCSINodeContainer(image string) corev1.Container { Value: "false", }, { - Name: "VSPHERE_CSI_CONFIG", - Value: "/etc/cloud/csi-vsphere.conf", + Name: "X_CSI_SPEC_DISABLE_LEN_CHECK", + Value: "true", }, { Name: "LOGGER_LEVEL", Value: "PRODUCTION", }, { - Name: "X_CSI_LOG_LEVEL", - Value: "INFO", + Name: "GODEBUG", + Value: "x509sha1=1", }, { - Name: "NODE_NAME", + Name: "CSI_NAMESPACE", ValueFrom: &corev1.EnvVarSource{ FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "spec.nodeName", + FieldPath: "metadata.namespace", }, }, }, + { + Name: "NODEGETINFO_WATCH_TIMEOUT_MINUTES", + Value: "1", + }, }, Ports: []corev1.ContainerPort{ { @@ -337,22 +571,18 @@ func VSphereCSINodeContainer(image string) corev1.Container { }, }, InitialDelaySeconds: 10, - TimeoutSeconds: 3, + TimeoutSeconds: 5, PeriodSeconds: 5, FailureThreshold: 3, }, SecurityContext: &corev1.SecurityContext{ Privileged: boolPtr(true), Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{corev1.Capability("SYS_ADMIN")}, + Add: []corev1.Capability{"SYS_ADMIN"}, }, AllowPrivilegeEscalation: boolPtr(true), }, VolumeMounts: []corev1.VolumeMount{ - { - Name: "vsphere-config-volume", - MountPath: "/etc/cloud", - }, { Name: "plugin-dir", MountPath: "/csi", @@ -366,6 +596,14 @@ func VSphereCSINodeContainer(image string) corev1.Container { Name: "device-dir", MountPath: "/dev", }, + { + Name: "blocks-dir", + MountPath: "/sys/block", + }, + { + Name: "sys-devices-dir", + MountPath: "/sys/devices", + }, }, } } @@ -374,7 +612,10 @@ func LivenessProbeForNodeContainer(image string) corev1.Container { return corev1.Container{ Name: "liveness-probe", Image: image, - Args: []string{"--csi-address=/csi/csi.sock"}, + Args: []string{ + "--v=4", + "--csi-address=/csi/csi.sock", + }, VolumeMounts: []corev1.VolumeMount{ { Name: "plugin-dir", @@ -386,12 +627,23 @@ func LivenessProbeForNodeContainer(image string) corev1.Container { func CSIControllerDeployment(storageConfig *types.CPIStorageConfig) *appsv1.Deployment { return &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: appsv1.SchemeGroupVersion.String(), + }, ObjectMeta: metav1.ObjectMeta{ Name: CSIControllerName, Namespace: CSINamespace, }, Spec: appsv1.DeploymentSpec{ Replicas: boolInt32(1), + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxUnavailable: newIntOrStringFromInt(1), + MaxSurge: newIntOrStringFromInt(0), + }, + }, Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "app": CSIControllerName, @@ -405,7 +657,29 @@ func CSIControllerDeployment(storageConfig *types.CPIStorageConfig) *appsv1.Depl }, }, Spec: corev1.PodSpec{ + PriorityClassName: "system-cluster-critical", + Affinity: &corev1.Affinity{ + PodAntiAffinity: &corev1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ + { + TopologyKey: "kubernetes.io/hostname", + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "app", + Operator: metav1.LabelSelectorOpIn, + Values: []string{CSIControllerName}, + }, + }, + }, + }, + }, + }, + }, ServiceAccountName: CSIControllerName, + NodeSelector: map[string]string{ + "node-role.kubernetes.io/control-plane": "", + }, Tolerations: []corev1.Toleration{ { Key: "node-role.kubernetes.io/master", @@ -421,10 +695,12 @@ func CSIControllerDeployment(storageConfig *types.CPIStorageConfig) *appsv1.Depl DNSPolicy: corev1.DNSDefault, Containers: []corev1.Container{ CSIAttacherContainer(storageConfig.AttacherImage), + CSIResizerContainer(storageConfig.ResizerImage), VSphereCSIControllerContainer(storageConfig.ControllerImage), LivenessProbeForCSIControllerContainer(storageConfig.LivenessProbeImage), VSphereSyncerContainer(storageConfig.MetadataSyncerImage), CSIProvisionerContainer(storageConfig.ProvisionerImage), + CSISnapshotterContainer(storageConfig.SnapshotterImage), }, Volumes: []corev1.Volume{ { @@ -452,7 +728,17 @@ func CSIAttacherContainer(image string) corev1.Container { return corev1.Container{ Name: "csi-attacher", Image: image, - Args: []string{"--v=4", "--timeout=300s", "--csi-address=$(ADDRESS)", "--leader-election"}, + Args: []string{ + "--v=4", + "--timeout=300s", + "--csi-address=$(ADDRESS)", + "--leader-election", + "--leader-election-lease-duration=120s", + "--leader-election-renew-deadline=60s", + "--leader-election-retry-period=30s", + "--kube-api-qps=100", + "--kube-api-burst=100", + }, Env: []corev1.EnvVar{ { Name: "ADDRESS", @@ -468,38 +754,62 @@ func CSIAttacherContainer(image string) corev1.Container { } } -func VSphereCSIControllerContainer(image string) corev1.Container { +func CSIResizerContainer(image string) corev1.Container { return corev1.Container{ - Name: CSIControllerName, + Name: "csi-resizer", Image: image, - Ports: []corev1.ContainerPort{ + Args: []string{ + "--v=4", + "--timeout=300s", + "--handle-volume-inuse-error=false", + "--csi-address=$(ADDRESS)", + "--kube-api-qps=100", + "--kube-api-burst=100", + "--leader-election", + "--leader-election-lease-duration=120s", + "--leader-election-renew-deadline=60s", + "--leader-election-retry-period=30s", + }, + Env: []corev1.EnvVar{ { - Name: "healthz", - ContainerPort: 9808, - Protocol: corev1.ProtocolTCP, + Name: "ADDRESS", + Value: "/csi/csi.sock", }, }, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/healthz", - Port: intstr.Parse("healthz"), - }, + VolumeMounts: []corev1.VolumeMount{ + { + MountPath: "/csi", + Name: "socket-dir", }, - InitialDelaySeconds: 10, - TimeoutSeconds: 3, - PeriodSeconds: 5, - FailureThreshold: 3, + }, + } +} + +func VSphereCSIControllerContainer(image string) corev1.Container { + return corev1.Container{ + Name: CSIControllerName, + Image: image, + Args: []string{ + "--fss-name=internal-feature-states.csi.vsphere.vmware.com", + "--fss-namespace=$(CSI_NAMESPACE)", }, Env: []corev1.EnvVar{ { Name: "CSI_ENDPOINT", - Value: "unix:///var/lib/csi/sockets/pluginproxy/csi.sock", + Value: "unix:///csi/csi.sock", }, { Name: "X_CSI_MODE", Value: "controller", }, + { + Name: "X_CSI_SPEC_DISABLE_LEN_CHECK", + Value: "true", + }, + { + Name: "X_CSI_SERIAL_VOL_ACCESS_TIMEOUT", + Value: "3m", + }, { Name: "VSPHERE_CSI_CONFIG", Value: "/etc/cloud/csi-vsphere.conf", @@ -509,10 +819,27 @@ func VSphereCSIControllerContainer(image string) corev1.Container { Value: "PRODUCTION", }, { - Name: "X_CSI_LOG_LEVEL", - Value: "INFO", + Name: "INCLUSTER_CLIENT_QPS", + Value: "100", + }, + { + Name: "INCLUSTER_CLIENT_BURST", + Value: "100", + }, + { + Name: "CSI_NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }, }, }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: int64ptr(65532), + RunAsGroup: int64ptr(65532), + RunAsNonRoot: boolPtr(true), + }, VolumeMounts: []corev1.VolumeMount{ { MountPath: "/etc/cloud", @@ -520,10 +847,34 @@ func VSphereCSIControllerContainer(image string) corev1.Container { ReadOnly: true, }, { - MountPath: "/var/lib/csi/sockets/pluginproxy/", + MountPath: "/csi", Name: "socket-dir", }, }, + Ports: []corev1.ContainerPort{ + { + Name: "healthz", + ContainerPort: 9808, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "prometheus", + ContainerPort: 2112, + Protocol: corev1.ProtocolTCP, + }, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.Parse("healthz"), + }, + }, + InitialDelaySeconds: 30, + TimeoutSeconds: 10, + PeriodSeconds: 180, + FailureThreshold: 3, + }, } } @@ -531,16 +882,13 @@ func LivenessProbeForCSIControllerContainer(image string) corev1.Container { return corev1.Container{ Name: "liveness-probe", Image: image, - Args: []string{"--csi-address=$(ADDRESS)"}, - Env: []corev1.EnvVar{ - { - Name: "ADDRESS", - Value: "/var/lib/csi/sockets/pluginproxy/csi.sock", - }, + Args: []string{ + "--v=4", + "--csi-address=/csi/csi.sock", }, VolumeMounts: []corev1.VolumeMount{ { - MountPath: "/var/lib/csi/sockets/pluginproxy/", + MountPath: "/csi", Name: "socket-dir", }, }, @@ -551,21 +899,60 @@ func VSphereSyncerContainer(image string) corev1.Container { return corev1.Container{ Name: "vsphere-syncer", Image: image, - Args: []string{"--leader-election"}, + Args: []string{ + "--leader-election", + "--leader-election-lease-duration=30s", + "--leader-election-renew-deadline=20s", + "--leader-election-retry-period=10s", + "--fss-name=internal-feature-states.csi.vsphere.vmware.com", + "--fss-namespace=$(CSI_NAMESPACE)", + }, + Ports: []corev1.ContainerPort{ + { + Name: "prometheus", + ContainerPort: 2113, + Protocol: corev1.ProtocolTCP, + }, + }, Env: []corev1.EnvVar{ { - Name: "X_CSI_FULL_SYNC_INTERVAL_MINUTES", + Name: "FULL_SYNC_INTERVAL_MINUTES", Value: "30", }, + { + Name: "VSPHERE_CSI_CONFIG", + Value: "/etc/cloud/csi-vsphere.conf", + }, { Name: "LOGGER_LEVEL", Value: "PRODUCTION", }, { - Name: "VSPHERE_CSI_CONFIG", - Value: "/etc/cloud/csi-vsphere.conf", + Name: "INCLUSTER_CLIENT_QPS", + Value: "100", + }, + { + Name: "INCLUSTER_CLIENT_BURST", + Value: "100", + }, + { + Name: "GODEBUG", + Value: "x509sha1=1", + }, + { + Name: "CSI_NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }, }, }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: int64ptr(65532), + RunAsGroup: int64ptr(65532), + RunAsNonRoot: boolPtr(true), + }, VolumeMounts: []corev1.VolumeMount{ { MountPath: "/etc/cloud", @@ -584,8 +971,16 @@ func CSIProvisionerContainer(image string) corev1.Container { "--v=4", "--timeout=300s", "--csi-address=$(ADDRESS)", + "--kube-api-qps=100", + "--kube-api-burst=100", "--leader-election", + "--leader-election-lease-duration=120s", + "--leader-election-renew-deadline=60s", + "--leader-election-retry-period=30s", "--default-fstype=ext4", + // needed only for topology aware setup + // "--feature-gates=Topology=true", + // "--strict-topology" }, Env: []corev1.EnvVar{ { @@ -602,23 +997,79 @@ func CSIProvisionerContainer(image string) corev1.Container { } } -func CSICloudConfigSecret(data string) *corev1.Secret { - return &corev1.Secret{ +func CSISnapshotterContainer(image string) corev1.Container { + return corev1.Container{ + Name: "csi-snapshotter", + Image: image, + Args: []string{ + "--v=4", + "--kube-api-qps=100", + "--kube-api-burst=100", + "--timeout=300s", + "--csi-address=$(ADDRESS)", + "--leader-election", + "--leader-election-lease-duration=120s", + "--leader-election-renew-deadline=60s", + "--leader-election-retry-period=30s", + }, + Env: []corev1.EnvVar{ + { + Name: "ADDRESS", + Value: "/csi/csi.sock", + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + MountPath: "/csi", + Name: "socket-dir", + }, + }, + } +} + +func CSIControllerService() *corev1.Service { + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Service", + }, ObjectMeta: metav1.ObjectMeta{ - Name: "csi-vsphere-config", + Name: CSIControllerName, Namespace: CSINamespace, + Labels: map[string]string{ + "app": CSIControllerName, + }, }, - Type: corev1.SecretTypeOpaque, - StringData: map[string]string{ - "csi-vsphere.conf": data, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "app": CSIControllerName, + }, + Ports: []corev1.ServicePort{ + { + Name: "ctlr", + Protocol: corev1.ProtocolTCP, + Port: 2112, + TargetPort: intstr.FromInt(2112), + }, + { + Name: "syncer", + Protocol: corev1.ProtocolTCP, + Port: 2113, + TargetPort: intstr.FromInt(2113), + }, + }, }, } } -func CSIComponentConfigSecret(secretName string, data string) *corev1.Secret { +func CSICloudConfigSecret(data string) *corev1.Secret { return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: corev1.SchemeGroupVersion.String(), + }, ObjectMeta: metav1.ObjectMeta{ - Name: secretName, + Name: "csi-vsphere-config", Namespace: CSINamespace, }, Type: corev1.SecretTypeOpaque, @@ -639,7 +1090,21 @@ func CSIFeatureStatesConfigMap() *corev1.ConfigMap { Namespace: CSINamespace, }, Data: map[string]string{ - "csi-migration": "false", + "csi-migration": "true", + "csi-auth-check": "true", + "online-volume-extend": "true", + "trigger-csi-fullsync": "false", + "async-query-volume": "true", + "block-volume-snapshot": "true", + "csi-windows-support": "true", + "list-volumes": "true", + "pv-to-backingdiskobjectid-mapping": "false", + "cnsmgr-suspend-create-volume": "true", + "topology-preferential-datastores": "true", + "max-pvscsi-targets-per-vm": "true", + "multi-vcenter-csi-topology": "true", + "csi-internal-generated-cluster-id": "true", + "listview-tasks": "true", }, } } @@ -658,6 +1123,11 @@ func newHostPathType(pathType string) *corev1.HostPathType { return hostPathType } +func newIntOrStringFromInt(val int) *intstr.IntOrString { + v := intstr.FromInt(val) + return &v +} + func newMountPropagation(propagation string) *corev1.MountPropagationMode { propagationMode := new(corev1.MountPropagationMode) *propagationMode = corev1.MountPropagationMode(propagation) diff --git a/packaging/flavorgen/flavors/crs/csi.go b/packaging/flavorgen/flavors/crs/csi.go index 7974bb1a03..04d20c07c7 100644 --- a/packaging/flavorgen/flavors/crs/csi.go +++ b/packaging/flavorgen/flavors/crs/csi.go @@ -20,11 +20,6 @@ import ( "fmt" "github.com/pkg/errors" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - rbac "k8s.io/api/rbac/v1" - storagev1 "k8s.io/api/storage/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" addonsv1 "sigs.k8s.io/cluster-api/exp/addons/api/v1beta1" @@ -36,29 +31,36 @@ import ( // CreateCrsResourceObjectsCSI creates the api objects necessary for CSI to function. // Also appends the resources to the CRS. func CreateCrsResourceObjectsCSI(crs *addonsv1.ClusterResourceSet) []runtime.Object { - serviceAccount := cloudprovider.CSIControllerServiceAccount() - serviceAccount.TypeMeta = metav1.TypeMeta{ - Kind: "ServiceAccount", - APIVersion: corev1.SchemeGroupVersion.String(), - } - serviceAccountSecret := newSecret(serviceAccount.Name, serviceAccount) - appendSecretToCrsResource(crs, serviceAccountSecret) + var objs []runtime.Object - clusterRole := cloudprovider.CSIControllerClusterRole() - clusterRole.TypeMeta = metav1.TypeMeta{ - Kind: "ClusterRole", - APIVersion: rbac.SchemeGroupVersion.String(), - } - clusterRoleConfigMap := newConfigMap(clusterRole.Name, clusterRole) - appendConfigMapToCrsResource(crs, clusterRoleConfigMap) + csiNamespace := cloudprovider.CSIDriverNamespace() + objs = append(objs, csiNamespace) - clusterRoleBinding := cloudprovider.CSIControllerClusterRoleBinding() - clusterRoleBinding.TypeMeta = metav1.TypeMeta{ - Kind: "ClusterRoleBinding", - APIVersion: rbac.SchemeGroupVersion.String(), - } - clusterRoleBindingConfigMap := newConfigMap(clusterRoleBinding.Name, clusterRoleBinding) - appendConfigMapToCrsResource(crs, clusterRoleBindingConfigMap) + controllerServiceAccount := cloudprovider.CSIControllerServiceAccount() + controllerServiceAccountSecret := newSecret(controllerServiceAccount.Name, controllerServiceAccount) + appendSecretToCrsResource(crs, controllerServiceAccountSecret) + + controllerClusterRole := cloudprovider.CSIControllerClusterRole() + objs = append(objs, controllerClusterRole) + + controllerClusterRoleBinding := cloudprovider.CSIControllerClusterRoleBinding() + objs = append(objs, controllerClusterRoleBinding) + + nodeDaemonSetServiceAccount := cloudprovider.CSINodeDaemonSetServiceAccount() + nodeDaemonSetServiceAccountSecret := newSecret(nodeDaemonSetServiceAccount.Name, nodeDaemonSetServiceAccount) + appendSecretToCrsResource(crs, nodeDaemonSetServiceAccountSecret) + + nodeRole := cloudprovider.CSINodeRole() + objs = append(objs, nodeRole) + + nodeRoleBinding := cloudprovider.CSINodeRoleBinding() + objs = append(objs, nodeRoleBinding) + + nodeClusterRole := cloudprovider.CSINodeClusterRole() + objs = append(objs, nodeClusterRole) + + nodeClusterRoleBinding := cloudprovider.CSINodeClusterRoleBinding() + objs = append(objs, nodeClusterRoleBinding) cloudConfig, err := ConfigForCSI().MarshalINI() if err != nil { @@ -66,46 +68,33 @@ func CreateCrsResourceObjectsCSI(crs *addonsv1.ClusterResourceSet) []runtime.Obj } // cloud config secret is wrapped in another secret so it could be injected via CRS cloudConfigSecret := cloudprovider.CSICloudConfigSecret(string(cloudConfig)) - cloudConfigSecret.TypeMeta = metav1.TypeMeta{ - Kind: "Secret", - APIVersion: corev1.SchemeGroupVersion.String(), - } cloudConfigSecretWrapper := newSecret(cloudConfigSecret.Name, cloudConfigSecret) appendSecretToCrsResource(crs, cloudConfigSecretWrapper) csiDriver := cloudprovider.CSIDriver() - csiDriver.TypeMeta = metav1.TypeMeta{ - Kind: "CSIDriver", - APIVersion: storagev1.SchemeGroupVersion.String(), - } - csiDriverConfigMap := newConfigMap(csiDriver.Name, csiDriver) - appendConfigMapToCrsResource(crs, csiDriverConfigMap) + objs = append(objs, csiDriver) storageConfig := createStorageConfig() daemonSet := cloudprovider.VSphereCSINodeDaemonSet(storageConfig) - daemonSet.TypeMeta = metav1.TypeMeta{ - Kind: "DaemonSet", - APIVersion: appsv1.SchemeGroupVersion.String(), - } - daemonSetConfigMap := newConfigMap(daemonSet.Name, daemonSet) - appendConfigMapToCrsResource(crs, daemonSetConfigMap) + objs = append(objs, daemonSet) deployment := cloudprovider.CSIControllerDeployment(storageConfig) - deployment.TypeMeta = metav1.TypeMeta{ - Kind: "Deployment", - APIVersion: appsv1.SchemeGroupVersion.String(), - } - deploymentConfigMap := newConfigMap(deployment.Name, deployment) - appendConfigMapToCrsResource(crs, deploymentConfigMap) + objs = append(objs, deployment) + + csiService := cloudprovider.CSIControllerService() + objs = append(objs, csiService) + + csiFeatureStates := cloudprovider.CSIFeatureStatesConfigMap() + objs = append(objs, csiFeatureStates) + + manifestConfigMap := newConfigMapManifests("csi-manifests", objs) + appendConfigMapToCrsResource(crs, manifestConfigMap) return []runtime.Object{ - serviceAccountSecret, - clusterRoleConfigMap, - clusterRoleBindingConfigMap, + controllerServiceAccountSecret, + nodeDaemonSetServiceAccountSecret, cloudConfigSecretWrapper, - csiDriverConfigMap, - daemonSetConfigMap, - deploymentConfigMap, + manifestConfigMap, } } @@ -115,7 +104,9 @@ func createStorageConfig() *types.CPIStorageConfig { ControllerImage: cloudprovider.DefaultCSIControllerImage, NodeDriverImage: cloudprovider.DefaultCSINodeDriverImage, AttacherImage: cloudprovider.DefaultCSIAttacherImage, + ResizerImage: cloudprovider.DefaultCSIResizerImage, ProvisionerImage: cloudprovider.DefaultCSIProvisionerImage, + SnapshotterImage: cloudprovider.DefaultCSISnapshotterContainerImage, MetadataSyncerImage: cloudprovider.DefaultCSIMetadataSyncerImage, LivenessProbeImage: cloudprovider.DefaultCSILivenessProbeImage, RegistrarImage: cloudprovider.DefaultCSIRegistrarImage, diff --git a/packaging/flavorgen/flavors/crs/types/cloudprovider_types.go b/packaging/flavorgen/flavors/crs/types/cloudprovider_types.go index e541ad27c1..2970b2213e 100644 --- a/packaging/flavorgen/flavors/crs/types/cloudprovider_types.go +++ b/packaging/flavorgen/flavors/crs/types/cloudprovider_types.go @@ -75,7 +75,9 @@ type CPIStorageConfig struct { ControllerImage string `json:"controllerImage,omitempty"` NodeDriverImage string `json:"nodeDriverImage,omitempty"` AttacherImage string `json:"attacherImage,omitempty"` + ResizerImage string `json:"resizerImage,omitempty"` ProvisionerImage string `json:"provisionerImage,omitempty"` + SnapshotterImage string `json:"snapshotterImage,omitempty"` MetadataSyncerImage string `json:"metadataSyncerImage,omitempty"` LivenessProbeImage string `json:"livenessProbeImage,omitempty"` RegistrarImage string `json:"registrarImage,omitempty"` diff --git a/templates/cluster-template-external-loadbalancer.yaml b/templates/cluster-template-external-loadbalancer.yaml index 41447e49e9..2d43a7a378 100644 --- a/templates/cluster-template-external-loadbalancer.yaml +++ b/templates/cluster-template-external-loadbalancer.yaml @@ -203,18 +203,12 @@ spec: resources: - kind: Secret name: vsphere-csi-controller - - kind: ConfigMap - name: vsphere-csi-controller-role - - kind: ConfigMap - name: vsphere-csi-controller-binding - kind: Secret - name: csi-vsphere-config - - kind: ConfigMap - name: csi.vsphere.vmware.com - - kind: ConfigMap name: vsphere-csi-node + - kind: Secret + name: csi-vsphere-config - kind: ConfigMap - name: vsphere-csi-controller + name: csi-manifests - kind: Secret name: cloud-controller-manager - kind: Secret @@ -242,72 +236,133 @@ stringData: kind: ServiceAccount metadata: name: vsphere-csi-controller - namespace: kube-system + namespace: vmware-system-csi +type: addons.cluster.x-k8s.io/resource-set +--- +apiVersion: v1 +kind: Secret +metadata: + name: vsphere-csi-node + namespace: '${NAMESPACE}' +stringData: + data: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: vsphere-csi-node + namespace: vmware-system-csi +type: addons.cluster.x-k8s.io/resource-set +--- +apiVersion: v1 +kind: Secret +metadata: + name: csi-vsphere-config + namespace: '${NAMESPACE}' +stringData: + data: | + apiVersion: v1 + kind: Secret + metadata: + name: csi-vsphere-config + namespace: vmware-system-csi + stringData: + csi-vsphere.conf: |+ + [Global] + thumbprint = "${VSPHERE_TLS_THUMBPRINT}" + cluster-id = "${NAMESPACE}/${CLUSTER_NAME}" + + [VirtualCenter "${VSPHERE_SERVER}"] + user = "${VSPHERE_USERNAME}" + password = "${VSPHERE_PASSWORD}" + datacenters = "${VSPHERE_DATACENTER}" + + [Network] + public-network = "${VSPHERE_NETWORK}" + + type: Opaque type: addons.cluster.x-k8s.io/resource-set --- apiVersion: v1 data: data: | + --- + apiVersion: v1 + kind: Namespace + metadata: + name: vmware-system-csi + --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role rules: - apiGroups: - - storage.k8s.io + - "" resources: - - csidrivers + - nodes + - pods verbs: - - create - - delete + - get + - list + - watch - apiGroups: - "" resources: - - nodes - - pods - - secrets - configmaps verbs: - get - list - watch + - create - apiGroups: - "" resources: - - persistentvolumes + - persistentvolumeclaims verbs: - get - list - watch - update - - create - - delete + - apiGroups: + - "" + resources: + - persistentvolumeclaims/status + verbs: - patch - apiGroups: - - storage.k8s.io + - "" resources: - - volumeattachments + - persistentvolumes verbs: - get - list - watch + - create - update + - delete - patch - apiGroups: - - storage.k8s.io + - "" resources: - - volumeattachments/status + - events verbs: + - get + - list + - watch + - create + - update - patch - apiGroups: - - "" + - coordination.k8s.io resources: - - persistentvolumeclaims + - leases verbs: - get - - list - watch + - list + - delete - update + - create - apiGroups: - storage.k8s.io resources: @@ -318,26 +373,69 @@ data: - list - watch - apiGroups: - - "" + - storage.k8s.io resources: - - events + - volumeattachments verbs: + - get - list - watch + - patch + - apiGroups: + - cns.vmware.com + resources: + - triggercsifullsyncs + verbs: - create + - get - update - - patch + - watch + - list - apiGroups: - - coordination.k8s.io + - cns.vmware.com resources: - - leases + - cnsvspherevolumemigrations verbs: + - create - get + - list - watch + - update + - delete + - apiGroups: + - cns.vmware.com + resources: + - cnsvolumeinfoes + verbs: + - create + - get - list + - watch - delete + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - create - update + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + - apiGroups: + - cns.vmware.com + resources: + - cnsvolumeoperationrequests + verbs: - create + - get + - list + - update + - delete - apiGroups: - snapshot.storage.k8s.io resources: @@ -345,89 +443,131 @@ data: verbs: - get - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - watch + - get + - list - apiGroups: - snapshot.storage.k8s.io resources: - volumesnapshotcontents verbs: + - create - get - list -kind: ConfigMap -metadata: - name: vsphere-csi-controller-role - namespace: '${NAMESPACE}' ---- -apiVersion: v1 -data: - data: | + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + - patch + - apiGroups: + - cns.vmware.com + resources: + - csinodetopologies + verbs: + - get + - update + - watch + - list + --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: vsphere-csi-controller-binding + name: vsphere-csi-controller-cluster-role-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role subjects: - kind: ServiceAccount name: vsphere-csi-controller - namespace: kube-system -kind: ConfigMap -metadata: - name: vsphere-csi-controller-binding - namespace: '${NAMESPACE}' ---- -apiVersion: v1 -kind: Secret -metadata: - name: csi-vsphere-config - namespace: '${NAMESPACE}' -stringData: - data: | - apiVersion: v1 - kind: Secret + namespace: vmware-system-csi + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role metadata: - name: csi-vsphere-config - namespace: kube-system - stringData: - csi-vsphere.conf: |+ - [Global] - thumbprint = "${VSPHERE_TLS_THUMBPRINT}" - cluster-id = "${NAMESPACE}/${CLUSTER_NAME}" - - [VirtualCenter "${VSPHERE_SERVER}"] - user = "${VSPHERE_USERNAME}" - password = "${VSPHERE_PASSWORD}" - datacenters = "${VSPHERE_DATACENTER}" - - [Network] - public-network = "${VSPHERE_NETWORK}" - - type: Opaque -type: addons.cluster.x-k8s.io/resource-set ---- -apiVersion: v1 -data: - data: | + name: vsphere-csi-node-role + namespace: vmware-system-csi + rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: vsphere-csi-node-role-binding + namespace: vmware-system-csi + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: vsphere-csi-node-role + subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: vmware-system-csi + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: vsphere-csi-node-cluster-role + rules: + - apiGroups: + - cns.vmware.com + resources: + - csinodetopologies + verbs: + - create + - watch + - get + - patch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: vsphere-csi-node-cluster-role-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: vsphere-csi-node-cluster-role + subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: vmware-system-csi + --- apiVersion: storage.k8s.io/v1 kind: CSIDriver metadata: name: csi.vsphere.vmware.com spec: attachRequired: true -kind: ConfigMap -metadata: - name: csi.vsphere.vmware.com - namespace: '${NAMESPACE}' ---- -apiVersion: v1 -data: - data: | + --- apiVersion: apps/v1 kind: DaemonSet metadata: name: vsphere-csi-node - namespace: kube-system + namespace: vmware-system-csi spec: selector: matchLabels: @@ -448,41 +588,50 @@ data: value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock - image: quay.io/k8scsi/csi-node-driver-registrar:v2.0.1 - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - rm -rf /registration/csi.vsphere.vmware.com-reg.sock /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock + - --mode=kubelet-registration-probe + initialDelaySeconds: 3 name: node-driver-registrar resources: {} - securityContext: - privileged: true volumeMounts: - mountPath: /csi name: plugin-dir - mountPath: /registration name: registration-dir - - env: + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName - name: CSI_ENDPOINT value: unix:///csi/csi.sock + - name: MAX_VOLUMES_PER_NODE + value: "59" - name: X_CSI_MODE value: node - name: X_CSI_SPEC_REQ_VALIDATION value: "false" - - name: VSPHERE_CSI_CONFIG - value: /etc/cloud/csi-vsphere.conf + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" - name: LOGGER_LEVEL value: PRODUCTION - - name: X_CSI_LOG_LEVEL - value: INFO - - name: NODE_NAME + - name: GODEBUG + value: x509sha1=1 + - name: CSI_NAMESPACE valueFrom: fieldRef: - fieldPath: spec.nodeName - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v2.1.0 + fieldPath: metadata.namespace + - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES + value: "1" + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0 livenessProbe: failureThreshold: 3 httpGet: @@ -490,7 +639,7 @@ data: port: healthz initialDelaySeconds: 10 periodSeconds: 5 - timeoutSeconds: 3 + timeoutSeconds: 5 name: vsphere-csi-node ports: - containerPort: 9808 @@ -504,8 +653,6 @@ data: - SYS_ADMIN privileged: true volumeMounts: - - mountPath: /etc/cloud - name: vsphere-config-volume - mountPath: /csi name: plugin-dir - mountPath: /var/lib/kubelet @@ -513,30 +660,37 @@ data: name: pods-mount-dir - mountPath: /dev name: device-dir + - mountPath: /sys/block + name: blocks-dir + - mountPath: /sys/devices + name: sys-devices-dir - args: + - --v=4 - --csi-address=/csi/csi.sock - image: quay.io/k8scsi/livenessprobe:v2.1.0 + image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 name: liveness-probe resources: {} volumeMounts: - mountPath: /csi name: plugin-dir - dnsPolicy: Default + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + serviceAccountName: vsphere-csi-node tolerations: - effect: NoSchedule operator: Exists - effect: NoExecute operator: Exists volumes: - - name: vsphere-config-volume - secret: - secretName: csi-vsphere-config - hostPath: path: /var/lib/kubelet/plugins_registry type: Directory name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/ + path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com type: DirectoryOrCreate name: plugin-dir - hostPath: @@ -546,102 +700,185 @@ data: - hostPath: path: /dev name: device-dir + - hostPath: + path: /sys/block + type: Directory + name: blocks-dir + - hostPath: + path: /sys/devices + type: Directory + name: sys-devices-dir updateStrategy: + rollingUpdate: + maxUnavailable: 1 type: RollingUpdate -kind: ConfigMap -metadata: - name: vsphere-csi-node - namespace: '${NAMESPACE}' ---- -apiVersion: v1 -data: - data: | + --- apiVersion: apps/v1 kind: Deployment metadata: name: vsphere-csi-controller - namespace: kube-system + namespace: vmware-system-csi spec: replicas: 1 selector: matchLabels: app: vsphere-csi-controller + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate template: metadata: labels: app: vsphere-csi-controller role: vsphere-csi spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - vsphere-csi-controller + topologyKey: kubernetes.io/hostname containers: - args: - --v=4 - --timeout=300s - --csi-address=$(ADDRESS) - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + - --kube-api-qps=100 + - --kube-api-burst=100 env: - name: ADDRESS value: /csi/csi.sock - image: quay.io/k8scsi/csi-attacher:v3.0.0 + image: registry.k8s.io/sig-storage/csi-attacher:v4.3.0 name: csi-attacher resources: {} volumeMounts: - mountPath: /csi name: socket-dir - - env: + - args: + - --v=4 + - --timeout=300s + - --handle-volume-inuse-error=false + - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-resizer:v1.8.0 + name: csi-resizer + resources: {} + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: - name: CSI_ENDPOINT - value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + value: unix:///csi/csi.sock - name: X_CSI_MODE value: controller + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" + - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT + value: 3m - name: VSPHERE_CSI_CONFIG value: /etc/cloud/csi-vsphere.conf - name: LOGGER_LEVEL value: PRODUCTION - - name: X_CSI_LOG_LEVEL - value: INFO - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v2.1.0 + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0 livenessProbe: failureThreshold: 3 httpGet: path: /healthz port: healthz - initialDelaySeconds: 10 - periodSeconds: 5 - timeoutSeconds: 3 + initialDelaySeconds: 30 + periodSeconds: 180 + timeoutSeconds: 10 name: vsphere-csi-controller ports: - containerPort: 9808 name: healthz protocol: TCP + - containerPort: 2112 + name: prometheus + protocol: TCP resources: {} + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 volumeMounts: - mountPath: /etc/cloud name: vsphere-config-volume readOnly: true - - mountPath: /var/lib/csi/sockets/pluginproxy/ + - mountPath: /csi name: socket-dir - args: - - --csi-address=$(ADDRESS) - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: quay.io/k8scsi/livenessprobe:v2.1.0 + - --v=4 + - --csi-address=/csi/csi.sock + image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 name: liveness-probe resources: {} volumeMounts: - - mountPath: /var/lib/csi/sockets/pluginproxy/ + - mountPath: /csi name: socket-dir - args: - --leader-election + - --leader-election-lease-duration=30s + - --leader-election-renew-deadline=20s + - --leader-election-retry-period=10s + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) env: - - name: X_CSI_FULL_SYNC_INTERVAL_MINUTES + - name: FULL_SYNC_INTERVAL_MINUTES value: "30" - - name: LOGGER_LEVEL - value: PRODUCTION - name: VSPHERE_CSI_CONFIG value: /etc/cloud/csi-vsphere.conf - image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v2.1.0 + - name: LOGGER_LEVEL + value: PRODUCTION + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: GODEBUG + value: x509sha1=1 + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v3.1.0 name: vsphere-syncer + ports: + - containerPort: 2113 + name: prometheus + protocol: TCP resources: {} + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 volumeMounts: - mountPath: /etc/cloud name: vsphere-config-volume @@ -650,18 +887,43 @@ data: - --v=4 - --timeout=300s - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s - --default-fstype=ext4 env: - name: ADDRESS value: /csi/csi.sock - image: quay.io/k8scsi/csi-provisioner:v2.0.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0 name: csi-provisioner resources: {} volumeMounts: - mountPath: /csi name: socket-dir + - args: + - --v=4 + - --kube-api-qps=100 + - --kube-api-burst=100 + - --timeout=300s + - --csi-address=$(ADDRESS) + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2 + name: csi-snapshotter + resources: {} + volumeMounts: + - mountPath: /csi + name: socket-dir dnsPolicy: Default + priorityClassName: system-cluster-critical serviceAccountName: vsphere-csi-controller tolerations: - effect: NoSchedule @@ -676,9 +938,51 @@ data: secretName: csi-vsphere-config - emptyDir: {} name: socket-dir + --- + apiVersion: v1 + kind: Service + metadata: + labels: + app: vsphere-csi-controller + name: vsphere-csi-controller + namespace: vmware-system-csi + spec: + ports: + - name: ctlr + port: 2112 + protocol: TCP + targetPort: 2112 + - name: syncer + port: 2113 + protocol: TCP + targetPort: 2113 + selector: + app: vsphere-csi-controller + --- + apiVersion: v1 + data: + async-query-volume: "true" + block-volume-snapshot: "true" + cnsmgr-suspend-create-volume: "true" + csi-auth-check: "true" + csi-internal-generated-cluster-id: "true" + csi-migration: "true" + csi-windows-support: "true" + list-volumes: "true" + listview-tasks: "true" + max-pvscsi-targets-per-vm: "true" + multi-vcenter-csi-topology: "true" + online-volume-extend: "true" + pv-to-backingdiskobjectid-mapping: "false" + topology-preferential-datastores: "true" + trigger-csi-fullsync: "false" + kind: ConfigMap + metadata: + name: internal-feature-states.csi.vsphere.vmware.com + namespace: vmware-system-csi kind: ConfigMap metadata: - name: vsphere-csi-controller + name: csi-manifests namespace: '${NAMESPACE}' --- apiVersion: v1 diff --git a/templates/cluster-template-ignition.yaml b/templates/cluster-template-ignition.yaml index d4747e4f4e..f6c523fb4e 100644 --- a/templates/cluster-template-ignition.yaml +++ b/templates/cluster-template-ignition.yaml @@ -345,18 +345,12 @@ spec: resources: - kind: Secret name: vsphere-csi-controller - - kind: ConfigMap - name: vsphere-csi-controller-role - - kind: ConfigMap - name: vsphere-csi-controller-binding - kind: Secret - name: csi-vsphere-config - - kind: ConfigMap - name: csi.vsphere.vmware.com - - kind: ConfigMap name: vsphere-csi-node + - kind: Secret + name: csi-vsphere-config - kind: ConfigMap - name: vsphere-csi-controller + name: csi-manifests - kind: Secret name: cloud-controller-manager - kind: Secret @@ -384,72 +378,133 @@ stringData: kind: ServiceAccount metadata: name: vsphere-csi-controller - namespace: kube-system + namespace: vmware-system-csi +type: addons.cluster.x-k8s.io/resource-set +--- +apiVersion: v1 +kind: Secret +metadata: + name: vsphere-csi-node + namespace: '${NAMESPACE}' +stringData: + data: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: vsphere-csi-node + namespace: vmware-system-csi +type: addons.cluster.x-k8s.io/resource-set +--- +apiVersion: v1 +kind: Secret +metadata: + name: csi-vsphere-config + namespace: '${NAMESPACE}' +stringData: + data: | + apiVersion: v1 + kind: Secret + metadata: + name: csi-vsphere-config + namespace: vmware-system-csi + stringData: + csi-vsphere.conf: |+ + [Global] + thumbprint = "${VSPHERE_TLS_THUMBPRINT}" + cluster-id = "${NAMESPACE}/${CLUSTER_NAME}" + + [VirtualCenter "${VSPHERE_SERVER}"] + user = "${VSPHERE_USERNAME}" + password = "${VSPHERE_PASSWORD}" + datacenters = "${VSPHERE_DATACENTER}" + + [Network] + public-network = "${VSPHERE_NETWORK}" + + type: Opaque type: addons.cluster.x-k8s.io/resource-set --- apiVersion: v1 data: data: | + --- + apiVersion: v1 + kind: Namespace + metadata: + name: vmware-system-csi + --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role rules: - apiGroups: - - storage.k8s.io + - "" resources: - - csidrivers + - nodes + - pods verbs: - - create - - delete + - get + - list + - watch - apiGroups: - "" resources: - - nodes - - pods - - secrets - configmaps verbs: - get - list - watch + - create - apiGroups: - "" resources: - - persistentvolumes + - persistentvolumeclaims verbs: - get - list - watch - update - - create - - delete + - apiGroups: + - "" + resources: + - persistentvolumeclaims/status + verbs: - patch - apiGroups: - - storage.k8s.io + - "" resources: - - volumeattachments + - persistentvolumes verbs: - get - list - watch + - create - update + - delete - patch - apiGroups: - - storage.k8s.io + - "" resources: - - volumeattachments/status + - events verbs: + - get + - list + - watch + - create + - update - patch - apiGroups: - - "" + - coordination.k8s.io resources: - - persistentvolumeclaims + - leases verbs: - get - - list - watch + - list + - delete - update + - create - apiGroups: - storage.k8s.io resources: @@ -460,26 +515,69 @@ data: - list - watch - apiGroups: - - "" + - storage.k8s.io resources: - - events + - volumeattachments verbs: + - get - list - watch + - patch + - apiGroups: + - cns.vmware.com + resources: + - triggercsifullsyncs + verbs: - create + - get - update - - patch + - watch + - list - apiGroups: - - coordination.k8s.io + - cns.vmware.com resources: - - leases + - cnsvspherevolumemigrations verbs: + - create - get + - list - watch + - update + - delete + - apiGroups: + - cns.vmware.com + resources: + - cnsvolumeinfoes + verbs: + - create + - get - list + - watch - delete + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - create - update + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + - apiGroups: + - cns.vmware.com + resources: + - cnsvolumeoperationrequests + verbs: - create + - get + - list + - update + - delete - apiGroups: - snapshot.storage.k8s.io resources: @@ -487,89 +585,131 @@ data: verbs: - get - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - watch + - get + - list - apiGroups: - snapshot.storage.k8s.io resources: - volumesnapshotcontents verbs: + - create - get - list -kind: ConfigMap -metadata: - name: vsphere-csi-controller-role - namespace: '${NAMESPACE}' ---- -apiVersion: v1 -data: - data: | + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + - patch + - apiGroups: + - cns.vmware.com + resources: + - csinodetopologies + verbs: + - get + - update + - watch + - list + --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: vsphere-csi-controller-binding + name: vsphere-csi-controller-cluster-role-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role subjects: - kind: ServiceAccount name: vsphere-csi-controller - namespace: kube-system -kind: ConfigMap -metadata: - name: vsphere-csi-controller-binding - namespace: '${NAMESPACE}' ---- -apiVersion: v1 -kind: Secret -metadata: - name: csi-vsphere-config - namespace: '${NAMESPACE}' -stringData: - data: | - apiVersion: v1 - kind: Secret + namespace: vmware-system-csi + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role metadata: - name: csi-vsphere-config - namespace: kube-system - stringData: - csi-vsphere.conf: |+ - [Global] - thumbprint = "${VSPHERE_TLS_THUMBPRINT}" - cluster-id = "${NAMESPACE}/${CLUSTER_NAME}" - - [VirtualCenter "${VSPHERE_SERVER}"] - user = "${VSPHERE_USERNAME}" - password = "${VSPHERE_PASSWORD}" - datacenters = "${VSPHERE_DATACENTER}" - - [Network] - public-network = "${VSPHERE_NETWORK}" - - type: Opaque -type: addons.cluster.x-k8s.io/resource-set ---- -apiVersion: v1 -data: - data: | + name: vsphere-csi-node-role + namespace: vmware-system-csi + rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: vsphere-csi-node-role-binding + namespace: vmware-system-csi + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: vsphere-csi-node-role + subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: vmware-system-csi + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: vsphere-csi-node-cluster-role + rules: + - apiGroups: + - cns.vmware.com + resources: + - csinodetopologies + verbs: + - create + - watch + - get + - patch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: vsphere-csi-node-cluster-role-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: vsphere-csi-node-cluster-role + subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: vmware-system-csi + --- apiVersion: storage.k8s.io/v1 kind: CSIDriver metadata: name: csi.vsphere.vmware.com spec: attachRequired: true -kind: ConfigMap -metadata: - name: csi.vsphere.vmware.com - namespace: '${NAMESPACE}' ---- -apiVersion: v1 -data: - data: | + --- apiVersion: apps/v1 kind: DaemonSet metadata: name: vsphere-csi-node - namespace: kube-system + namespace: vmware-system-csi spec: selector: matchLabels: @@ -590,41 +730,50 @@ data: value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock - image: quay.io/k8scsi/csi-node-driver-registrar:v2.0.1 - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - rm -rf /registration/csi.vsphere.vmware.com-reg.sock /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock + - --mode=kubelet-registration-probe + initialDelaySeconds: 3 name: node-driver-registrar resources: {} - securityContext: - privileged: true volumeMounts: - mountPath: /csi name: plugin-dir - mountPath: /registration name: registration-dir - - env: + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName - name: CSI_ENDPOINT value: unix:///csi/csi.sock + - name: MAX_VOLUMES_PER_NODE + value: "59" - name: X_CSI_MODE value: node - name: X_CSI_SPEC_REQ_VALIDATION value: "false" - - name: VSPHERE_CSI_CONFIG - value: /etc/cloud/csi-vsphere.conf + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" - name: LOGGER_LEVEL value: PRODUCTION - - name: X_CSI_LOG_LEVEL - value: INFO - - name: NODE_NAME + - name: GODEBUG + value: x509sha1=1 + - name: CSI_NAMESPACE valueFrom: fieldRef: - fieldPath: spec.nodeName - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v2.1.0 + fieldPath: metadata.namespace + - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES + value: "1" + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0 livenessProbe: failureThreshold: 3 httpGet: @@ -632,7 +781,7 @@ data: port: healthz initialDelaySeconds: 10 periodSeconds: 5 - timeoutSeconds: 3 + timeoutSeconds: 5 name: vsphere-csi-node ports: - containerPort: 9808 @@ -646,8 +795,6 @@ data: - SYS_ADMIN privileged: true volumeMounts: - - mountPath: /etc/cloud - name: vsphere-config-volume - mountPath: /csi name: plugin-dir - mountPath: /var/lib/kubelet @@ -655,30 +802,37 @@ data: name: pods-mount-dir - mountPath: /dev name: device-dir + - mountPath: /sys/block + name: blocks-dir + - mountPath: /sys/devices + name: sys-devices-dir - args: + - --v=4 - --csi-address=/csi/csi.sock - image: quay.io/k8scsi/livenessprobe:v2.1.0 + image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 name: liveness-probe resources: {} volumeMounts: - mountPath: /csi name: plugin-dir - dnsPolicy: Default + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + serviceAccountName: vsphere-csi-node tolerations: - effect: NoSchedule operator: Exists - effect: NoExecute operator: Exists volumes: - - name: vsphere-config-volume - secret: - secretName: csi-vsphere-config - hostPath: path: /var/lib/kubelet/plugins_registry type: Directory name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/ + path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com type: DirectoryOrCreate name: plugin-dir - hostPath: @@ -688,102 +842,185 @@ data: - hostPath: path: /dev name: device-dir + - hostPath: + path: /sys/block + type: Directory + name: blocks-dir + - hostPath: + path: /sys/devices + type: Directory + name: sys-devices-dir updateStrategy: + rollingUpdate: + maxUnavailable: 1 type: RollingUpdate -kind: ConfigMap -metadata: - name: vsphere-csi-node - namespace: '${NAMESPACE}' ---- -apiVersion: v1 -data: - data: | + --- apiVersion: apps/v1 kind: Deployment metadata: name: vsphere-csi-controller - namespace: kube-system + namespace: vmware-system-csi spec: replicas: 1 selector: matchLabels: app: vsphere-csi-controller + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate template: metadata: labels: app: vsphere-csi-controller role: vsphere-csi spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - vsphere-csi-controller + topologyKey: kubernetes.io/hostname containers: - args: - --v=4 - --timeout=300s - --csi-address=$(ADDRESS) - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + - --kube-api-qps=100 + - --kube-api-burst=100 env: - name: ADDRESS value: /csi/csi.sock - image: quay.io/k8scsi/csi-attacher:v3.0.0 + image: registry.k8s.io/sig-storage/csi-attacher:v4.3.0 name: csi-attacher resources: {} volumeMounts: - mountPath: /csi name: socket-dir - - env: + - args: + - --v=4 + - --timeout=300s + - --handle-volume-inuse-error=false + - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-resizer:v1.8.0 + name: csi-resizer + resources: {} + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: - name: CSI_ENDPOINT - value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + value: unix:///csi/csi.sock - name: X_CSI_MODE value: controller + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" + - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT + value: 3m - name: VSPHERE_CSI_CONFIG value: /etc/cloud/csi-vsphere.conf - name: LOGGER_LEVEL value: PRODUCTION - - name: X_CSI_LOG_LEVEL - value: INFO - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v2.1.0 + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0 livenessProbe: failureThreshold: 3 httpGet: path: /healthz port: healthz - initialDelaySeconds: 10 - periodSeconds: 5 - timeoutSeconds: 3 + initialDelaySeconds: 30 + periodSeconds: 180 + timeoutSeconds: 10 name: vsphere-csi-controller ports: - containerPort: 9808 name: healthz protocol: TCP + - containerPort: 2112 + name: prometheus + protocol: TCP resources: {} + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 volumeMounts: - mountPath: /etc/cloud name: vsphere-config-volume readOnly: true - - mountPath: /var/lib/csi/sockets/pluginproxy/ + - mountPath: /csi name: socket-dir - args: - - --csi-address=$(ADDRESS) - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: quay.io/k8scsi/livenessprobe:v2.1.0 + - --v=4 + - --csi-address=/csi/csi.sock + image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 name: liveness-probe resources: {} volumeMounts: - - mountPath: /var/lib/csi/sockets/pluginproxy/ + - mountPath: /csi name: socket-dir - args: - --leader-election + - --leader-election-lease-duration=30s + - --leader-election-renew-deadline=20s + - --leader-election-retry-period=10s + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) env: - - name: X_CSI_FULL_SYNC_INTERVAL_MINUTES + - name: FULL_SYNC_INTERVAL_MINUTES value: "30" - - name: LOGGER_LEVEL - value: PRODUCTION - name: VSPHERE_CSI_CONFIG value: /etc/cloud/csi-vsphere.conf - image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v2.1.0 + - name: LOGGER_LEVEL + value: PRODUCTION + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: GODEBUG + value: x509sha1=1 + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v3.1.0 name: vsphere-syncer + ports: + - containerPort: 2113 + name: prometheus + protocol: TCP resources: {} + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 volumeMounts: - mountPath: /etc/cloud name: vsphere-config-volume @@ -792,18 +1029,43 @@ data: - --v=4 - --timeout=300s - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s - --default-fstype=ext4 env: - name: ADDRESS value: /csi/csi.sock - image: quay.io/k8scsi/csi-provisioner:v2.0.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0 name: csi-provisioner resources: {} volumeMounts: - mountPath: /csi name: socket-dir + - args: + - --v=4 + - --kube-api-qps=100 + - --kube-api-burst=100 + - --timeout=300s + - --csi-address=$(ADDRESS) + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2 + name: csi-snapshotter + resources: {} + volumeMounts: + - mountPath: /csi + name: socket-dir dnsPolicy: Default + priorityClassName: system-cluster-critical serviceAccountName: vsphere-csi-controller tolerations: - effect: NoSchedule @@ -818,9 +1080,51 @@ data: secretName: csi-vsphere-config - emptyDir: {} name: socket-dir + --- + apiVersion: v1 + kind: Service + metadata: + labels: + app: vsphere-csi-controller + name: vsphere-csi-controller + namespace: vmware-system-csi + spec: + ports: + - name: ctlr + port: 2112 + protocol: TCP + targetPort: 2112 + - name: syncer + port: 2113 + protocol: TCP + targetPort: 2113 + selector: + app: vsphere-csi-controller + --- + apiVersion: v1 + data: + async-query-volume: "true" + block-volume-snapshot: "true" + cnsmgr-suspend-create-volume: "true" + csi-auth-check: "true" + csi-internal-generated-cluster-id: "true" + csi-migration: "true" + csi-windows-support: "true" + list-volumes: "true" + listview-tasks: "true" + max-pvscsi-targets-per-vm: "true" + multi-vcenter-csi-topology: "true" + online-volume-extend: "true" + pv-to-backingdiskobjectid-mapping: "false" + topology-preferential-datastores: "true" + trigger-csi-fullsync: "false" + kind: ConfigMap + metadata: + name: internal-feature-states.csi.vsphere.vmware.com + namespace: vmware-system-csi kind: ConfigMap metadata: - name: vsphere-csi-controller + name: csi-manifests namespace: '${NAMESPACE}' --- apiVersion: v1 diff --git a/templates/cluster-template-node-ipam.yaml b/templates/cluster-template-node-ipam.yaml index b0b4bdc97f..4fc3ed9f1f 100644 --- a/templates/cluster-template-node-ipam.yaml +++ b/templates/cluster-template-node-ipam.yaml @@ -269,18 +269,12 @@ spec: resources: - kind: Secret name: vsphere-csi-controller - - kind: ConfigMap - name: vsphere-csi-controller-role - - kind: ConfigMap - name: vsphere-csi-controller-binding - kind: Secret - name: csi-vsphere-config - - kind: ConfigMap - name: csi.vsphere.vmware.com - - kind: ConfigMap name: vsphere-csi-node + - kind: Secret + name: csi-vsphere-config - kind: ConfigMap - name: vsphere-csi-controller + name: csi-manifests - kind: Secret name: cloud-controller-manager - kind: Secret @@ -308,72 +302,133 @@ stringData: kind: ServiceAccount metadata: name: vsphere-csi-controller - namespace: kube-system + namespace: vmware-system-csi +type: addons.cluster.x-k8s.io/resource-set +--- +apiVersion: v1 +kind: Secret +metadata: + name: vsphere-csi-node + namespace: '${NAMESPACE}' +stringData: + data: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: vsphere-csi-node + namespace: vmware-system-csi +type: addons.cluster.x-k8s.io/resource-set +--- +apiVersion: v1 +kind: Secret +metadata: + name: csi-vsphere-config + namespace: '${NAMESPACE}' +stringData: + data: | + apiVersion: v1 + kind: Secret + metadata: + name: csi-vsphere-config + namespace: vmware-system-csi + stringData: + csi-vsphere.conf: |+ + [Global] + thumbprint = "${VSPHERE_TLS_THUMBPRINT}" + cluster-id = "${NAMESPACE}/${CLUSTER_NAME}" + + [VirtualCenter "${VSPHERE_SERVER}"] + user = "${VSPHERE_USERNAME}" + password = "${VSPHERE_PASSWORD}" + datacenters = "${VSPHERE_DATACENTER}" + + [Network] + public-network = "${VSPHERE_NETWORK}" + + type: Opaque type: addons.cluster.x-k8s.io/resource-set --- apiVersion: v1 data: data: | + --- + apiVersion: v1 + kind: Namespace + metadata: + name: vmware-system-csi + --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role rules: - apiGroups: - - storage.k8s.io + - "" resources: - - csidrivers + - nodes + - pods verbs: - - create - - delete + - get + - list + - watch - apiGroups: - "" resources: - - nodes - - pods - - secrets - configmaps verbs: - get - list - watch + - create - apiGroups: - "" resources: - - persistentvolumes + - persistentvolumeclaims verbs: - get - list - watch - update - - create - - delete + - apiGroups: + - "" + resources: + - persistentvolumeclaims/status + verbs: - patch - apiGroups: - - storage.k8s.io + - "" resources: - - volumeattachments + - persistentvolumes verbs: - get - list - watch + - create - update + - delete - patch - apiGroups: - - storage.k8s.io + - "" resources: - - volumeattachments/status + - events verbs: + - get + - list + - watch + - create + - update - patch - apiGroups: - - "" + - coordination.k8s.io resources: - - persistentvolumeclaims + - leases verbs: - get - - list - watch + - list + - delete - update + - create - apiGroups: - storage.k8s.io resources: @@ -384,26 +439,69 @@ data: - list - watch - apiGroups: - - "" + - storage.k8s.io resources: - - events + - volumeattachments verbs: + - get - list - watch + - patch + - apiGroups: + - cns.vmware.com + resources: + - triggercsifullsyncs + verbs: - create + - get - update - - patch + - watch + - list - apiGroups: - - coordination.k8s.io + - cns.vmware.com resources: - - leases + - cnsvspherevolumemigrations verbs: + - create - get + - list - watch + - update + - delete + - apiGroups: + - cns.vmware.com + resources: + - cnsvolumeinfoes + verbs: + - create + - get - list + - watch - delete + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - create - update + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + - apiGroups: + - cns.vmware.com + resources: + - cnsvolumeoperationrequests + verbs: - create + - get + - list + - update + - delete - apiGroups: - snapshot.storage.k8s.io resources: @@ -411,89 +509,131 @@ data: verbs: - get - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - watch + - get + - list - apiGroups: - snapshot.storage.k8s.io resources: - volumesnapshotcontents verbs: + - create - get - list -kind: ConfigMap -metadata: - name: vsphere-csi-controller-role - namespace: '${NAMESPACE}' ---- -apiVersion: v1 -data: - data: | + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + - patch + - apiGroups: + - cns.vmware.com + resources: + - csinodetopologies + verbs: + - get + - update + - watch + - list + --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: vsphere-csi-controller-binding + name: vsphere-csi-controller-cluster-role-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role subjects: - kind: ServiceAccount name: vsphere-csi-controller - namespace: kube-system -kind: ConfigMap -metadata: - name: vsphere-csi-controller-binding - namespace: '${NAMESPACE}' ---- -apiVersion: v1 -kind: Secret -metadata: - name: csi-vsphere-config - namespace: '${NAMESPACE}' -stringData: - data: | - apiVersion: v1 - kind: Secret + namespace: vmware-system-csi + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role metadata: - name: csi-vsphere-config - namespace: kube-system - stringData: - csi-vsphere.conf: |+ - [Global] - thumbprint = "${VSPHERE_TLS_THUMBPRINT}" - cluster-id = "${NAMESPACE}/${CLUSTER_NAME}" - - [VirtualCenter "${VSPHERE_SERVER}"] - user = "${VSPHERE_USERNAME}" - password = "${VSPHERE_PASSWORD}" - datacenters = "${VSPHERE_DATACENTER}" - - [Network] - public-network = "${VSPHERE_NETWORK}" - - type: Opaque -type: addons.cluster.x-k8s.io/resource-set ---- -apiVersion: v1 -data: - data: | + name: vsphere-csi-node-role + namespace: vmware-system-csi + rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: vsphere-csi-node-role-binding + namespace: vmware-system-csi + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: vsphere-csi-node-role + subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: vmware-system-csi + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: vsphere-csi-node-cluster-role + rules: + - apiGroups: + - cns.vmware.com + resources: + - csinodetopologies + verbs: + - create + - watch + - get + - patch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: vsphere-csi-node-cluster-role-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: vsphere-csi-node-cluster-role + subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: vmware-system-csi + --- apiVersion: storage.k8s.io/v1 kind: CSIDriver metadata: name: csi.vsphere.vmware.com spec: attachRequired: true -kind: ConfigMap -metadata: - name: csi.vsphere.vmware.com - namespace: '${NAMESPACE}' ---- -apiVersion: v1 -data: - data: | + --- apiVersion: apps/v1 kind: DaemonSet metadata: name: vsphere-csi-node - namespace: kube-system + namespace: vmware-system-csi spec: selector: matchLabels: @@ -514,41 +654,50 @@ data: value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock - image: quay.io/k8scsi/csi-node-driver-registrar:v2.0.1 - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - rm -rf /registration/csi.vsphere.vmware.com-reg.sock /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock + - --mode=kubelet-registration-probe + initialDelaySeconds: 3 name: node-driver-registrar resources: {} - securityContext: - privileged: true volumeMounts: - mountPath: /csi name: plugin-dir - mountPath: /registration name: registration-dir - - env: + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName - name: CSI_ENDPOINT value: unix:///csi/csi.sock + - name: MAX_VOLUMES_PER_NODE + value: "59" - name: X_CSI_MODE value: node - name: X_CSI_SPEC_REQ_VALIDATION value: "false" - - name: VSPHERE_CSI_CONFIG - value: /etc/cloud/csi-vsphere.conf + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" - name: LOGGER_LEVEL value: PRODUCTION - - name: X_CSI_LOG_LEVEL - value: INFO - - name: NODE_NAME + - name: GODEBUG + value: x509sha1=1 + - name: CSI_NAMESPACE valueFrom: fieldRef: - fieldPath: spec.nodeName - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v2.1.0 + fieldPath: metadata.namespace + - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES + value: "1" + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0 livenessProbe: failureThreshold: 3 httpGet: @@ -556,7 +705,7 @@ data: port: healthz initialDelaySeconds: 10 periodSeconds: 5 - timeoutSeconds: 3 + timeoutSeconds: 5 name: vsphere-csi-node ports: - containerPort: 9808 @@ -570,8 +719,6 @@ data: - SYS_ADMIN privileged: true volumeMounts: - - mountPath: /etc/cloud - name: vsphere-config-volume - mountPath: /csi name: plugin-dir - mountPath: /var/lib/kubelet @@ -579,30 +726,37 @@ data: name: pods-mount-dir - mountPath: /dev name: device-dir + - mountPath: /sys/block + name: blocks-dir + - mountPath: /sys/devices + name: sys-devices-dir - args: + - --v=4 - --csi-address=/csi/csi.sock - image: quay.io/k8scsi/livenessprobe:v2.1.0 + image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 name: liveness-probe resources: {} volumeMounts: - mountPath: /csi name: plugin-dir - dnsPolicy: Default + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + serviceAccountName: vsphere-csi-node tolerations: - effect: NoSchedule operator: Exists - effect: NoExecute operator: Exists volumes: - - name: vsphere-config-volume - secret: - secretName: csi-vsphere-config - hostPath: path: /var/lib/kubelet/plugins_registry type: Directory name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/ + path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com type: DirectoryOrCreate name: plugin-dir - hostPath: @@ -612,102 +766,185 @@ data: - hostPath: path: /dev name: device-dir + - hostPath: + path: /sys/block + type: Directory + name: blocks-dir + - hostPath: + path: /sys/devices + type: Directory + name: sys-devices-dir updateStrategy: + rollingUpdate: + maxUnavailable: 1 type: RollingUpdate -kind: ConfigMap -metadata: - name: vsphere-csi-node - namespace: '${NAMESPACE}' ---- -apiVersion: v1 -data: - data: | + --- apiVersion: apps/v1 kind: Deployment metadata: name: vsphere-csi-controller - namespace: kube-system + namespace: vmware-system-csi spec: replicas: 1 selector: matchLabels: app: vsphere-csi-controller + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate template: metadata: labels: app: vsphere-csi-controller role: vsphere-csi spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - vsphere-csi-controller + topologyKey: kubernetes.io/hostname containers: - args: - --v=4 - --timeout=300s - --csi-address=$(ADDRESS) - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + - --kube-api-qps=100 + - --kube-api-burst=100 env: - name: ADDRESS value: /csi/csi.sock - image: quay.io/k8scsi/csi-attacher:v3.0.0 + image: registry.k8s.io/sig-storage/csi-attacher:v4.3.0 name: csi-attacher resources: {} volumeMounts: - mountPath: /csi name: socket-dir - - env: + - args: + - --v=4 + - --timeout=300s + - --handle-volume-inuse-error=false + - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-resizer:v1.8.0 + name: csi-resizer + resources: {} + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: - name: CSI_ENDPOINT - value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + value: unix:///csi/csi.sock - name: X_CSI_MODE value: controller + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" + - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT + value: 3m - name: VSPHERE_CSI_CONFIG value: /etc/cloud/csi-vsphere.conf - name: LOGGER_LEVEL value: PRODUCTION - - name: X_CSI_LOG_LEVEL - value: INFO - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v2.1.0 + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0 livenessProbe: failureThreshold: 3 httpGet: path: /healthz port: healthz - initialDelaySeconds: 10 - periodSeconds: 5 - timeoutSeconds: 3 + initialDelaySeconds: 30 + periodSeconds: 180 + timeoutSeconds: 10 name: vsphere-csi-controller ports: - containerPort: 9808 name: healthz protocol: TCP + - containerPort: 2112 + name: prometheus + protocol: TCP resources: {} + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 volumeMounts: - mountPath: /etc/cloud name: vsphere-config-volume readOnly: true - - mountPath: /var/lib/csi/sockets/pluginproxy/ + - mountPath: /csi name: socket-dir - args: - - --csi-address=$(ADDRESS) - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: quay.io/k8scsi/livenessprobe:v2.1.0 + - --v=4 + - --csi-address=/csi/csi.sock + image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 name: liveness-probe resources: {} volumeMounts: - - mountPath: /var/lib/csi/sockets/pluginproxy/ + - mountPath: /csi name: socket-dir - args: - --leader-election + - --leader-election-lease-duration=30s + - --leader-election-renew-deadline=20s + - --leader-election-retry-period=10s + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) env: - - name: X_CSI_FULL_SYNC_INTERVAL_MINUTES + - name: FULL_SYNC_INTERVAL_MINUTES value: "30" - - name: LOGGER_LEVEL - value: PRODUCTION - name: VSPHERE_CSI_CONFIG value: /etc/cloud/csi-vsphere.conf - image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v2.1.0 + - name: LOGGER_LEVEL + value: PRODUCTION + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: GODEBUG + value: x509sha1=1 + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v3.1.0 name: vsphere-syncer + ports: + - containerPort: 2113 + name: prometheus + protocol: TCP resources: {} + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 volumeMounts: - mountPath: /etc/cloud name: vsphere-config-volume @@ -716,18 +953,43 @@ data: - --v=4 - --timeout=300s - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s - --default-fstype=ext4 env: - name: ADDRESS value: /csi/csi.sock - image: quay.io/k8scsi/csi-provisioner:v2.0.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0 name: csi-provisioner resources: {} volumeMounts: - mountPath: /csi name: socket-dir + - args: + - --v=4 + - --kube-api-qps=100 + - --kube-api-burst=100 + - --timeout=300s + - --csi-address=$(ADDRESS) + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2 + name: csi-snapshotter + resources: {} + volumeMounts: + - mountPath: /csi + name: socket-dir dnsPolicy: Default + priorityClassName: system-cluster-critical serviceAccountName: vsphere-csi-controller tolerations: - effect: NoSchedule @@ -742,9 +1004,51 @@ data: secretName: csi-vsphere-config - emptyDir: {} name: socket-dir + --- + apiVersion: v1 + kind: Service + metadata: + labels: + app: vsphere-csi-controller + name: vsphere-csi-controller + namespace: vmware-system-csi + spec: + ports: + - name: ctlr + port: 2112 + protocol: TCP + targetPort: 2112 + - name: syncer + port: 2113 + protocol: TCP + targetPort: 2113 + selector: + app: vsphere-csi-controller + --- + apiVersion: v1 + data: + async-query-volume: "true" + block-volume-snapshot: "true" + cnsmgr-suspend-create-volume: "true" + csi-auth-check: "true" + csi-internal-generated-cluster-id: "true" + csi-migration: "true" + csi-windows-support: "true" + list-volumes: "true" + listview-tasks: "true" + max-pvscsi-targets-per-vm: "true" + multi-vcenter-csi-topology: "true" + online-volume-extend: "true" + pv-to-backingdiskobjectid-mapping: "false" + topology-preferential-datastores: "true" + trigger-csi-fullsync: "false" + kind: ConfigMap + metadata: + name: internal-feature-states.csi.vsphere.vmware.com + namespace: vmware-system-csi kind: ConfigMap metadata: - name: vsphere-csi-controller + name: csi-manifests namespace: '${NAMESPACE}' --- apiVersion: v1 diff --git a/templates/cluster-template-topology.yaml b/templates/cluster-template-topology.yaml index f886449ee7..b16d1046be 100644 --- a/templates/cluster-template-topology.yaml +++ b/templates/cluster-template-topology.yaml @@ -105,18 +105,12 @@ spec: resources: - kind: Secret name: vsphere-csi-controller - - kind: ConfigMap - name: vsphere-csi-controller-role - - kind: ConfigMap - name: vsphere-csi-controller-binding - kind: Secret - name: csi-vsphere-config - - kind: ConfigMap - name: csi.vsphere.vmware.com - - kind: ConfigMap name: vsphere-csi-node + - kind: Secret + name: csi-vsphere-config - kind: ConfigMap - name: vsphere-csi-controller + name: csi-manifests - kind: Secret name: cloud-controller-manager - kind: Secret @@ -135,72 +129,133 @@ stringData: kind: ServiceAccount metadata: name: vsphere-csi-controller - namespace: kube-system + namespace: vmware-system-csi +type: addons.cluster.x-k8s.io/resource-set +--- +apiVersion: v1 +kind: Secret +metadata: + name: vsphere-csi-node + namespace: '${NAMESPACE}' +stringData: + data: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: vsphere-csi-node + namespace: vmware-system-csi +type: addons.cluster.x-k8s.io/resource-set +--- +apiVersion: v1 +kind: Secret +metadata: + name: csi-vsphere-config + namespace: '${NAMESPACE}' +stringData: + data: | + apiVersion: v1 + kind: Secret + metadata: + name: csi-vsphere-config + namespace: vmware-system-csi + stringData: + csi-vsphere.conf: |+ + [Global] + thumbprint = "${VSPHERE_TLS_THUMBPRINT}" + cluster-id = "${NAMESPACE}/${CLUSTER_NAME}" + + [VirtualCenter "${VSPHERE_SERVER}"] + user = "${VSPHERE_USERNAME}" + password = "${VSPHERE_PASSWORD}" + datacenters = "${VSPHERE_DATACENTER}" + + [Network] + public-network = "${VSPHERE_NETWORK}" + + type: Opaque type: addons.cluster.x-k8s.io/resource-set --- apiVersion: v1 data: data: | + --- + apiVersion: v1 + kind: Namespace + metadata: + name: vmware-system-csi + --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role rules: - apiGroups: - - storage.k8s.io + - "" resources: - - csidrivers + - nodes + - pods verbs: - - create - - delete + - get + - list + - watch - apiGroups: - "" resources: - - nodes - - pods - - secrets - configmaps verbs: - get - list - watch + - create - apiGroups: - "" resources: - - persistentvolumes + - persistentvolumeclaims verbs: - get - list - watch - update - - create - - delete + - apiGroups: + - "" + resources: + - persistentvolumeclaims/status + verbs: - patch - apiGroups: - - storage.k8s.io + - "" resources: - - volumeattachments + - persistentvolumes verbs: - get - list - watch + - create - update + - delete - patch - apiGroups: - - storage.k8s.io + - "" resources: - - volumeattachments/status + - events verbs: + - get + - list + - watch + - create + - update - patch - apiGroups: - - "" + - coordination.k8s.io resources: - - persistentvolumeclaims + - leases verbs: - get - - list - watch + - list + - delete - update + - create - apiGroups: - storage.k8s.io resources: @@ -211,26 +266,69 @@ data: - list - watch - apiGroups: - - "" + - storage.k8s.io resources: - - events + - volumeattachments verbs: + - get - list - watch + - patch + - apiGroups: + - cns.vmware.com + resources: + - triggercsifullsyncs + verbs: - create + - get - update - - patch + - watch + - list - apiGroups: - - coordination.k8s.io + - cns.vmware.com resources: - - leases + - cnsvspherevolumemigrations verbs: + - create - get + - list - watch + - update + - delete + - apiGroups: + - cns.vmware.com + resources: + - cnsvolumeinfoes + verbs: + - create + - get - list + - watch - delete + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - create - update + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + - apiGroups: + - cns.vmware.com + resources: + - cnsvolumeoperationrequests + verbs: - create + - get + - list + - update + - delete - apiGroups: - snapshot.storage.k8s.io resources: @@ -238,89 +336,131 @@ data: verbs: - get - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - watch + - get + - list - apiGroups: - snapshot.storage.k8s.io resources: - volumesnapshotcontents verbs: + - create - get - list -kind: ConfigMap -metadata: - name: vsphere-csi-controller-role - namespace: '${NAMESPACE}' ---- -apiVersion: v1 -data: - data: | + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + - patch + - apiGroups: + - cns.vmware.com + resources: + - csinodetopologies + verbs: + - get + - update + - watch + - list + --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: vsphere-csi-controller-binding + name: vsphere-csi-controller-cluster-role-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role subjects: - kind: ServiceAccount name: vsphere-csi-controller - namespace: kube-system -kind: ConfigMap -metadata: - name: vsphere-csi-controller-binding - namespace: '${NAMESPACE}' ---- -apiVersion: v1 -kind: Secret -metadata: - name: csi-vsphere-config - namespace: '${NAMESPACE}' -stringData: - data: | - apiVersion: v1 - kind: Secret + namespace: vmware-system-csi + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role metadata: - name: csi-vsphere-config - namespace: kube-system - stringData: - csi-vsphere.conf: |+ - [Global] - thumbprint = "${VSPHERE_TLS_THUMBPRINT}" - cluster-id = "${NAMESPACE}/${CLUSTER_NAME}" - - [VirtualCenter "${VSPHERE_SERVER}"] - user = "${VSPHERE_USERNAME}" - password = "${VSPHERE_PASSWORD}" - datacenters = "${VSPHERE_DATACENTER}" - - [Network] - public-network = "${VSPHERE_NETWORK}" - - type: Opaque -type: addons.cluster.x-k8s.io/resource-set ---- -apiVersion: v1 -data: - data: | + name: vsphere-csi-node-role + namespace: vmware-system-csi + rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: vsphere-csi-node-role-binding + namespace: vmware-system-csi + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: vsphere-csi-node-role + subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: vmware-system-csi + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: vsphere-csi-node-cluster-role + rules: + - apiGroups: + - cns.vmware.com + resources: + - csinodetopologies + verbs: + - create + - watch + - get + - patch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: vsphere-csi-node-cluster-role-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: vsphere-csi-node-cluster-role + subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: vmware-system-csi + --- apiVersion: storage.k8s.io/v1 kind: CSIDriver metadata: name: csi.vsphere.vmware.com spec: attachRequired: true -kind: ConfigMap -metadata: - name: csi.vsphere.vmware.com - namespace: '${NAMESPACE}' ---- -apiVersion: v1 -data: - data: | + --- apiVersion: apps/v1 kind: DaemonSet metadata: name: vsphere-csi-node - namespace: kube-system + namespace: vmware-system-csi spec: selector: matchLabels: @@ -341,41 +481,50 @@ data: value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock - image: quay.io/k8scsi/csi-node-driver-registrar:v2.0.1 - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - rm -rf /registration/csi.vsphere.vmware.com-reg.sock /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock + - --mode=kubelet-registration-probe + initialDelaySeconds: 3 name: node-driver-registrar resources: {} - securityContext: - privileged: true volumeMounts: - mountPath: /csi name: plugin-dir - mountPath: /registration name: registration-dir - - env: + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName - name: CSI_ENDPOINT value: unix:///csi/csi.sock + - name: MAX_VOLUMES_PER_NODE + value: "59" - name: X_CSI_MODE value: node - name: X_CSI_SPEC_REQ_VALIDATION value: "false" - - name: VSPHERE_CSI_CONFIG - value: /etc/cloud/csi-vsphere.conf + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" - name: LOGGER_LEVEL value: PRODUCTION - - name: X_CSI_LOG_LEVEL - value: INFO - - name: NODE_NAME + - name: GODEBUG + value: x509sha1=1 + - name: CSI_NAMESPACE valueFrom: fieldRef: - fieldPath: spec.nodeName - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v2.1.0 + fieldPath: metadata.namespace + - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES + value: "1" + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0 livenessProbe: failureThreshold: 3 httpGet: @@ -383,7 +532,7 @@ data: port: healthz initialDelaySeconds: 10 periodSeconds: 5 - timeoutSeconds: 3 + timeoutSeconds: 5 name: vsphere-csi-node ports: - containerPort: 9808 @@ -397,8 +546,6 @@ data: - SYS_ADMIN privileged: true volumeMounts: - - mountPath: /etc/cloud - name: vsphere-config-volume - mountPath: /csi name: plugin-dir - mountPath: /var/lib/kubelet @@ -406,30 +553,37 @@ data: name: pods-mount-dir - mountPath: /dev name: device-dir + - mountPath: /sys/block + name: blocks-dir + - mountPath: /sys/devices + name: sys-devices-dir - args: + - --v=4 - --csi-address=/csi/csi.sock - image: quay.io/k8scsi/livenessprobe:v2.1.0 + image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 name: liveness-probe resources: {} volumeMounts: - mountPath: /csi name: plugin-dir - dnsPolicy: Default + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + serviceAccountName: vsphere-csi-node tolerations: - effect: NoSchedule operator: Exists - effect: NoExecute operator: Exists volumes: - - name: vsphere-config-volume - secret: - secretName: csi-vsphere-config - hostPath: path: /var/lib/kubelet/plugins_registry type: Directory name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/ + path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com type: DirectoryOrCreate name: plugin-dir - hostPath: @@ -439,102 +593,185 @@ data: - hostPath: path: /dev name: device-dir + - hostPath: + path: /sys/block + type: Directory + name: blocks-dir + - hostPath: + path: /sys/devices + type: Directory + name: sys-devices-dir updateStrategy: + rollingUpdate: + maxUnavailable: 1 type: RollingUpdate -kind: ConfigMap -metadata: - name: vsphere-csi-node - namespace: '${NAMESPACE}' ---- -apiVersion: v1 -data: - data: | + --- apiVersion: apps/v1 kind: Deployment metadata: name: vsphere-csi-controller - namespace: kube-system + namespace: vmware-system-csi spec: replicas: 1 selector: matchLabels: app: vsphere-csi-controller + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate template: metadata: labels: app: vsphere-csi-controller role: vsphere-csi spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - vsphere-csi-controller + topologyKey: kubernetes.io/hostname containers: - args: - --v=4 - --timeout=300s - --csi-address=$(ADDRESS) - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + - --kube-api-qps=100 + - --kube-api-burst=100 env: - name: ADDRESS value: /csi/csi.sock - image: quay.io/k8scsi/csi-attacher:v3.0.0 + image: registry.k8s.io/sig-storage/csi-attacher:v4.3.0 name: csi-attacher resources: {} volumeMounts: - mountPath: /csi name: socket-dir - - env: + - args: + - --v=4 + - --timeout=300s + - --handle-volume-inuse-error=false + - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-resizer:v1.8.0 + name: csi-resizer + resources: {} + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: - name: CSI_ENDPOINT - value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + value: unix:///csi/csi.sock - name: X_CSI_MODE value: controller + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" + - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT + value: 3m - name: VSPHERE_CSI_CONFIG value: /etc/cloud/csi-vsphere.conf - name: LOGGER_LEVEL value: PRODUCTION - - name: X_CSI_LOG_LEVEL - value: INFO - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v2.1.0 + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0 livenessProbe: failureThreshold: 3 httpGet: path: /healthz port: healthz - initialDelaySeconds: 10 - periodSeconds: 5 - timeoutSeconds: 3 + initialDelaySeconds: 30 + periodSeconds: 180 + timeoutSeconds: 10 name: vsphere-csi-controller ports: - containerPort: 9808 name: healthz protocol: TCP + - containerPort: 2112 + name: prometheus + protocol: TCP resources: {} + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 volumeMounts: - mountPath: /etc/cloud name: vsphere-config-volume readOnly: true - - mountPath: /var/lib/csi/sockets/pluginproxy/ + - mountPath: /csi name: socket-dir - args: - - --csi-address=$(ADDRESS) - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: quay.io/k8scsi/livenessprobe:v2.1.0 + - --v=4 + - --csi-address=/csi/csi.sock + image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 name: liveness-probe resources: {} volumeMounts: - - mountPath: /var/lib/csi/sockets/pluginproxy/ + - mountPath: /csi name: socket-dir - args: - --leader-election + - --leader-election-lease-duration=30s + - --leader-election-renew-deadline=20s + - --leader-election-retry-period=10s + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) env: - - name: X_CSI_FULL_SYNC_INTERVAL_MINUTES + - name: FULL_SYNC_INTERVAL_MINUTES value: "30" - - name: LOGGER_LEVEL - value: PRODUCTION - name: VSPHERE_CSI_CONFIG value: /etc/cloud/csi-vsphere.conf - image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v2.1.0 + - name: LOGGER_LEVEL + value: PRODUCTION + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: GODEBUG + value: x509sha1=1 + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v3.1.0 name: vsphere-syncer + ports: + - containerPort: 2113 + name: prometheus + protocol: TCP resources: {} + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 volumeMounts: - mountPath: /etc/cloud name: vsphere-config-volume @@ -543,18 +780,43 @@ data: - --v=4 - --timeout=300s - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s - --default-fstype=ext4 env: - name: ADDRESS value: /csi/csi.sock - image: quay.io/k8scsi/csi-provisioner:v2.0.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0 name: csi-provisioner resources: {} volumeMounts: - mountPath: /csi name: socket-dir + - args: + - --v=4 + - --kube-api-qps=100 + - --kube-api-burst=100 + - --timeout=300s + - --csi-address=$(ADDRESS) + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2 + name: csi-snapshotter + resources: {} + volumeMounts: + - mountPath: /csi + name: socket-dir dnsPolicy: Default + priorityClassName: system-cluster-critical serviceAccountName: vsphere-csi-controller tolerations: - effect: NoSchedule @@ -569,9 +831,51 @@ data: secretName: csi-vsphere-config - emptyDir: {} name: socket-dir + --- + apiVersion: v1 + kind: Service + metadata: + labels: + app: vsphere-csi-controller + name: vsphere-csi-controller + namespace: vmware-system-csi + spec: + ports: + - name: ctlr + port: 2112 + protocol: TCP + targetPort: 2112 + - name: syncer + port: 2113 + protocol: TCP + targetPort: 2113 + selector: + app: vsphere-csi-controller + --- + apiVersion: v1 + data: + async-query-volume: "true" + block-volume-snapshot: "true" + cnsmgr-suspend-create-volume: "true" + csi-auth-check: "true" + csi-internal-generated-cluster-id: "true" + csi-migration: "true" + csi-windows-support: "true" + list-volumes: "true" + listview-tasks: "true" + max-pvscsi-targets-per-vm: "true" + multi-vcenter-csi-topology: "true" + online-volume-extend: "true" + pv-to-backingdiskobjectid-mapping: "false" + topology-preferential-datastores: "true" + trigger-csi-fullsync: "false" + kind: ConfigMap + metadata: + name: internal-feature-states.csi.vsphere.vmware.com + namespace: vmware-system-csi kind: ConfigMap metadata: - name: vsphere-csi-controller + name: csi-manifests namespace: '${NAMESPACE}' --- apiVersion: v1 diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index 2916bb99a4..614218c68e 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -259,18 +259,12 @@ spec: resources: - kind: Secret name: vsphere-csi-controller - - kind: ConfigMap - name: vsphere-csi-controller-role - - kind: ConfigMap - name: vsphere-csi-controller-binding - kind: Secret - name: csi-vsphere-config - - kind: ConfigMap - name: csi.vsphere.vmware.com - - kind: ConfigMap name: vsphere-csi-node + - kind: Secret + name: csi-vsphere-config - kind: ConfigMap - name: vsphere-csi-controller + name: csi-manifests - kind: Secret name: cloud-controller-manager - kind: Secret @@ -298,72 +292,133 @@ stringData: kind: ServiceAccount metadata: name: vsphere-csi-controller - namespace: kube-system + namespace: vmware-system-csi +type: addons.cluster.x-k8s.io/resource-set +--- +apiVersion: v1 +kind: Secret +metadata: + name: vsphere-csi-node + namespace: '${NAMESPACE}' +stringData: + data: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: vsphere-csi-node + namespace: vmware-system-csi +type: addons.cluster.x-k8s.io/resource-set +--- +apiVersion: v1 +kind: Secret +metadata: + name: csi-vsphere-config + namespace: '${NAMESPACE}' +stringData: + data: | + apiVersion: v1 + kind: Secret + metadata: + name: csi-vsphere-config + namespace: vmware-system-csi + stringData: + csi-vsphere.conf: |+ + [Global] + thumbprint = "${VSPHERE_TLS_THUMBPRINT}" + cluster-id = "${NAMESPACE}/${CLUSTER_NAME}" + + [VirtualCenter "${VSPHERE_SERVER}"] + user = "${VSPHERE_USERNAME}" + password = "${VSPHERE_PASSWORD}" + datacenters = "${VSPHERE_DATACENTER}" + + [Network] + public-network = "${VSPHERE_NETWORK}" + + type: Opaque type: addons.cluster.x-k8s.io/resource-set --- apiVersion: v1 data: data: | + --- + apiVersion: v1 + kind: Namespace + metadata: + name: vmware-system-csi + --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role rules: - apiGroups: - - storage.k8s.io + - "" resources: - - csidrivers + - nodes + - pods verbs: - - create - - delete + - get + - list + - watch - apiGroups: - "" resources: - - nodes - - pods - - secrets - configmaps verbs: - get - list - watch + - create - apiGroups: - "" resources: - - persistentvolumes + - persistentvolumeclaims verbs: - get - list - watch - update - - create - - delete + - apiGroups: + - "" + resources: + - persistentvolumeclaims/status + verbs: - patch - apiGroups: - - storage.k8s.io + - "" resources: - - volumeattachments + - persistentvolumes verbs: - get - list - watch + - create - update + - delete - patch - apiGroups: - - storage.k8s.io + - "" resources: - - volumeattachments/status + - events verbs: + - get + - list + - watch + - create + - update - patch - apiGroups: - - "" + - coordination.k8s.io resources: - - persistentvolumeclaims + - leases verbs: - get - - list - watch + - list + - delete - update + - create - apiGroups: - storage.k8s.io resources: @@ -374,26 +429,69 @@ data: - list - watch - apiGroups: - - "" + - storage.k8s.io resources: - - events + - volumeattachments verbs: + - get - list - watch + - patch + - apiGroups: + - cns.vmware.com + resources: + - triggercsifullsyncs + verbs: - create + - get - update - - patch + - watch + - list - apiGroups: - - coordination.k8s.io + - cns.vmware.com resources: - - leases + - cnsvspherevolumemigrations verbs: + - create - get + - list - watch + - update + - delete + - apiGroups: + - cns.vmware.com + resources: + - cnsvolumeinfoes + verbs: + - create + - get - list + - watch - delete + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - create - update + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + - apiGroups: + - cns.vmware.com + resources: + - cnsvolumeoperationrequests + verbs: - create + - get + - list + - update + - delete - apiGroups: - snapshot.storage.k8s.io resources: @@ -401,89 +499,131 @@ data: verbs: - get - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - watch + - get + - list - apiGroups: - snapshot.storage.k8s.io resources: - volumesnapshotcontents verbs: + - create - get - list -kind: ConfigMap -metadata: - name: vsphere-csi-controller-role - namespace: '${NAMESPACE}' ---- -apiVersion: v1 -data: - data: | + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + - patch + - apiGroups: + - cns.vmware.com + resources: + - csinodetopologies + verbs: + - get + - update + - watch + - list + --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: vsphere-csi-controller-binding + name: vsphere-csi-controller-cluster-role-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role subjects: - kind: ServiceAccount name: vsphere-csi-controller - namespace: kube-system -kind: ConfigMap -metadata: - name: vsphere-csi-controller-binding - namespace: '${NAMESPACE}' ---- -apiVersion: v1 -kind: Secret -metadata: - name: csi-vsphere-config - namespace: '${NAMESPACE}' -stringData: - data: | - apiVersion: v1 - kind: Secret + namespace: vmware-system-csi + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role metadata: - name: csi-vsphere-config - namespace: kube-system - stringData: - csi-vsphere.conf: |+ - [Global] - thumbprint = "${VSPHERE_TLS_THUMBPRINT}" - cluster-id = "${NAMESPACE}/${CLUSTER_NAME}" - - [VirtualCenter "${VSPHERE_SERVER}"] - user = "${VSPHERE_USERNAME}" - password = "${VSPHERE_PASSWORD}" - datacenters = "${VSPHERE_DATACENTER}" - - [Network] - public-network = "${VSPHERE_NETWORK}" - - type: Opaque -type: addons.cluster.x-k8s.io/resource-set ---- -apiVersion: v1 -data: - data: | + name: vsphere-csi-node-role + namespace: vmware-system-csi + rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: vsphere-csi-node-role-binding + namespace: vmware-system-csi + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: vsphere-csi-node-role + subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: vmware-system-csi + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: vsphere-csi-node-cluster-role + rules: + - apiGroups: + - cns.vmware.com + resources: + - csinodetopologies + verbs: + - create + - watch + - get + - patch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: vsphere-csi-node-cluster-role-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: vsphere-csi-node-cluster-role + subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: vmware-system-csi + --- apiVersion: storage.k8s.io/v1 kind: CSIDriver metadata: name: csi.vsphere.vmware.com spec: attachRequired: true -kind: ConfigMap -metadata: - name: csi.vsphere.vmware.com - namespace: '${NAMESPACE}' ---- -apiVersion: v1 -data: - data: | + --- apiVersion: apps/v1 kind: DaemonSet metadata: name: vsphere-csi-node - namespace: kube-system + namespace: vmware-system-csi spec: selector: matchLabels: @@ -504,41 +644,50 @@ data: value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock - image: quay.io/k8scsi/csi-node-driver-registrar:v2.0.1 - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - rm -rf /registration/csi.vsphere.vmware.com-reg.sock /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock + - --mode=kubelet-registration-probe + initialDelaySeconds: 3 name: node-driver-registrar resources: {} - securityContext: - privileged: true volumeMounts: - mountPath: /csi name: plugin-dir - mountPath: /registration name: registration-dir - - env: + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName - name: CSI_ENDPOINT value: unix:///csi/csi.sock + - name: MAX_VOLUMES_PER_NODE + value: "59" - name: X_CSI_MODE value: node - name: X_CSI_SPEC_REQ_VALIDATION value: "false" - - name: VSPHERE_CSI_CONFIG - value: /etc/cloud/csi-vsphere.conf + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" - name: LOGGER_LEVEL value: PRODUCTION - - name: X_CSI_LOG_LEVEL - value: INFO - - name: NODE_NAME + - name: GODEBUG + value: x509sha1=1 + - name: CSI_NAMESPACE valueFrom: fieldRef: - fieldPath: spec.nodeName - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v2.1.0 + fieldPath: metadata.namespace + - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES + value: "1" + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0 livenessProbe: failureThreshold: 3 httpGet: @@ -546,7 +695,7 @@ data: port: healthz initialDelaySeconds: 10 periodSeconds: 5 - timeoutSeconds: 3 + timeoutSeconds: 5 name: vsphere-csi-node ports: - containerPort: 9808 @@ -560,8 +709,6 @@ data: - SYS_ADMIN privileged: true volumeMounts: - - mountPath: /etc/cloud - name: vsphere-config-volume - mountPath: /csi name: plugin-dir - mountPath: /var/lib/kubelet @@ -569,30 +716,37 @@ data: name: pods-mount-dir - mountPath: /dev name: device-dir + - mountPath: /sys/block + name: blocks-dir + - mountPath: /sys/devices + name: sys-devices-dir - args: + - --v=4 - --csi-address=/csi/csi.sock - image: quay.io/k8scsi/livenessprobe:v2.1.0 + image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 name: liveness-probe resources: {} volumeMounts: - mountPath: /csi name: plugin-dir - dnsPolicy: Default + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + serviceAccountName: vsphere-csi-node tolerations: - effect: NoSchedule operator: Exists - effect: NoExecute operator: Exists volumes: - - name: vsphere-config-volume - secret: - secretName: csi-vsphere-config - hostPath: path: /var/lib/kubelet/plugins_registry type: Directory name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/ + path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com type: DirectoryOrCreate name: plugin-dir - hostPath: @@ -602,102 +756,185 @@ data: - hostPath: path: /dev name: device-dir + - hostPath: + path: /sys/block + type: Directory + name: blocks-dir + - hostPath: + path: /sys/devices + type: Directory + name: sys-devices-dir updateStrategy: + rollingUpdate: + maxUnavailable: 1 type: RollingUpdate -kind: ConfigMap -metadata: - name: vsphere-csi-node - namespace: '${NAMESPACE}' ---- -apiVersion: v1 -data: - data: | + --- apiVersion: apps/v1 kind: Deployment metadata: name: vsphere-csi-controller - namespace: kube-system + namespace: vmware-system-csi spec: replicas: 1 selector: matchLabels: app: vsphere-csi-controller + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate template: metadata: labels: app: vsphere-csi-controller role: vsphere-csi spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - vsphere-csi-controller + topologyKey: kubernetes.io/hostname containers: - args: - --v=4 - --timeout=300s - --csi-address=$(ADDRESS) - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + - --kube-api-qps=100 + - --kube-api-burst=100 env: - name: ADDRESS value: /csi/csi.sock - image: quay.io/k8scsi/csi-attacher:v3.0.0 + image: registry.k8s.io/sig-storage/csi-attacher:v4.3.0 name: csi-attacher resources: {} volumeMounts: - mountPath: /csi name: socket-dir - - env: + - args: + - --v=4 + - --timeout=300s + - --handle-volume-inuse-error=false + - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-resizer:v1.8.0 + name: csi-resizer + resources: {} + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: - name: CSI_ENDPOINT - value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + value: unix:///csi/csi.sock - name: X_CSI_MODE value: controller + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" + - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT + value: 3m - name: VSPHERE_CSI_CONFIG value: /etc/cloud/csi-vsphere.conf - name: LOGGER_LEVEL value: PRODUCTION - - name: X_CSI_LOG_LEVEL - value: INFO - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v2.1.0 + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0 livenessProbe: failureThreshold: 3 httpGet: path: /healthz port: healthz - initialDelaySeconds: 10 - periodSeconds: 5 - timeoutSeconds: 3 + initialDelaySeconds: 30 + periodSeconds: 180 + timeoutSeconds: 10 name: vsphere-csi-controller ports: - containerPort: 9808 name: healthz protocol: TCP + - containerPort: 2112 + name: prometheus + protocol: TCP resources: {} + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 volumeMounts: - mountPath: /etc/cloud name: vsphere-config-volume readOnly: true - - mountPath: /var/lib/csi/sockets/pluginproxy/ + - mountPath: /csi name: socket-dir - args: - - --csi-address=$(ADDRESS) - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: quay.io/k8scsi/livenessprobe:v2.1.0 + - --v=4 + - --csi-address=/csi/csi.sock + image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 name: liveness-probe resources: {} volumeMounts: - - mountPath: /var/lib/csi/sockets/pluginproxy/ + - mountPath: /csi name: socket-dir - args: - --leader-election + - --leader-election-lease-duration=30s + - --leader-election-renew-deadline=20s + - --leader-election-retry-period=10s + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) env: - - name: X_CSI_FULL_SYNC_INTERVAL_MINUTES + - name: FULL_SYNC_INTERVAL_MINUTES value: "30" - - name: LOGGER_LEVEL - value: PRODUCTION - name: VSPHERE_CSI_CONFIG value: /etc/cloud/csi-vsphere.conf - image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v2.1.0 + - name: LOGGER_LEVEL + value: PRODUCTION + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: GODEBUG + value: x509sha1=1 + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v3.1.0 name: vsphere-syncer + ports: + - containerPort: 2113 + name: prometheus + protocol: TCP resources: {} + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 volumeMounts: - mountPath: /etc/cloud name: vsphere-config-volume @@ -706,18 +943,43 @@ data: - --v=4 - --timeout=300s - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s - --default-fstype=ext4 env: - name: ADDRESS value: /csi/csi.sock - image: quay.io/k8scsi/csi-provisioner:v2.0.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0 name: csi-provisioner resources: {} volumeMounts: - mountPath: /csi name: socket-dir + - args: + - --v=4 + - --kube-api-qps=100 + - --kube-api-burst=100 + - --timeout=300s + - --csi-address=$(ADDRESS) + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2 + name: csi-snapshotter + resources: {} + volumeMounts: + - mountPath: /csi + name: socket-dir dnsPolicy: Default + priorityClassName: system-cluster-critical serviceAccountName: vsphere-csi-controller tolerations: - effect: NoSchedule @@ -732,9 +994,51 @@ data: secretName: csi-vsphere-config - emptyDir: {} name: socket-dir + --- + apiVersion: v1 + kind: Service + metadata: + labels: + app: vsphere-csi-controller + name: vsphere-csi-controller + namespace: vmware-system-csi + spec: + ports: + - name: ctlr + port: 2112 + protocol: TCP + targetPort: 2112 + - name: syncer + port: 2113 + protocol: TCP + targetPort: 2113 + selector: + app: vsphere-csi-controller + --- + apiVersion: v1 + data: + async-query-volume: "true" + block-volume-snapshot: "true" + cnsmgr-suspend-create-volume: "true" + csi-auth-check: "true" + csi-internal-generated-cluster-id: "true" + csi-migration: "true" + csi-windows-support: "true" + list-volumes: "true" + listview-tasks: "true" + max-pvscsi-targets-per-vm: "true" + multi-vcenter-csi-topology: "true" + online-volume-extend: "true" + pv-to-backingdiskobjectid-mapping: "false" + topology-preferential-datastores: "true" + trigger-csi-fullsync: "false" + kind: ConfigMap + metadata: + name: internal-feature-states.csi.vsphere.vmware.com + namespace: vmware-system-csi kind: ConfigMap metadata: - name: vsphere-csi-controller + name: csi-manifests namespace: '${NAMESPACE}' --- apiVersion: v1