diff --git a/packaging/flavorgen/cloudprovider/csi.go b/packaging/flavorgen/cloudprovider/csi.go index 0d317f6188..42701d31b4 100644 --- a/packaging/flavorgen/cloudprovider/csi.go +++ b/packaging/flavorgen/cloudprovider/csi.go @@ -20,30 +20,60 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" - storagev1beta1 "k8s.io/api/storage/v1beta1" + storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/crs/types" ) -// NOTE: the contents of this file are derived from https://github.com/kubernetes-sigs/vsphere-csi-driver/tree/master/manifests/1.14 +// NOTE: the contents of this file are derived from https://github.com/kubernetes-sigs/vsphere-csi-driver/tree/master/manifests/vanilla const ( - DefaultCSIControllerImage = "gcr.io/cloud-provider-vsphere/csi/release/driver:v2.1.0" - DefaultCSINodeDriverImage = "gcr.io/cloud-provider-vsphere/csi/release/driver:v2.1.0" - DefaultCSIAttacherImage = "quay.io/k8scsi/csi-attacher:v3.0.0" - DefaultCSIProvisionerImage = "quay.io/k8scsi/csi-provisioner:v2.0.0" - DefaultCSIMetadataSyncerImage = "gcr.io/cloud-provider-vsphere/csi/release/syncer:v2.1.0" - DefaultCSILivenessProbeImage = "quay.io/k8scsi/livenessprobe:v2.1.0" - DefaultCSIRegistrarImage = "quay.io/k8scsi/csi-node-driver-registrar:v2.0.1" - CSINamespace = metav1.NamespaceSystem - CSIControllerName = "vsphere-csi-controller" - CSIFeatureStateConfigMapName = "internal-feature-states.csi.vsphere.vmware.com" + DefaultCSIControllerImage = "gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0" + DefaultCSINodeDriverImage = "gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0" + DefaultCSIAttacherImage = "registry.k8s.io/sig-storage/csi-attacher:v4.3.0" + DefaultCSIResizerImage = "registry.k8s.io/sig-storage/csi-resizer:v1.8.0" + DefaultCSIProvisionerImage = "registry.k8s.io/sig-storage/csi-provisioner:v3.5.0" + DefaultCSISnapshotterContainerImage = "registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2" + DefaultCSIMetadataSyncerImage = "gcr.io/cloud-provider-vsphere/csi/release/syncer:v3.1.0" + DefaultCSILivenessProbeImage = "registry.k8s.io/sig-storage/livenessprobe:v2.10.0" + DefaultCSIRegistrarImage = "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0" + CSINamespace = "vmware-system-csi" + CSIControllerName = "vsphere-csi-controller" + CSINodeDaemonSetName = "vsphere-csi-node" + CSIFeatureStateConfigMapName = "internal-feature-states.csi.vsphere.vmware.com" + + clusterRoleSuffix = "-cluster-role" + clusterRoleBindingSuffix = clusterRoleSuffix + "-binding" + roleSuffix = "-role" + roleBindingSuffix = roleSuffix + "-binding" + CSIControllerClusterRoleName = CSIControllerName + clusterRoleSuffix + CSIControllerRoleBindingName = CSIControllerName + clusterRoleBindingSuffix + CSINodeRoleName = CSINodeDaemonSetName + roleSuffix + CSINodeRoleBindingName = CSINodeDaemonSetName + roleBindingSuffix + CSINodeClusterRoleName = CSINodeDaemonSetName + clusterRoleSuffix + CSINodeClusterRoleBindingName = CSINodeDaemonSetName + clusterRoleBindingSuffix ) +func CSIDriverNamespace() *corev1.Namespace { + return &corev1.Namespace{ + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: CSINamespace, + }, + } +} + func CSIControllerServiceAccount() *corev1.ServiceAccount { return &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{ + Kind: "ServiceAccount", + APIVersion: corev1.SchemeGroupVersion.String(), + }, ObjectMeta: metav1.ObjectMeta{ Name: CSIControllerName, Namespace: CSINamespace, @@ -53,64 +83,113 @@ func CSIControllerServiceAccount() *corev1.ServiceAccount { func CSIControllerClusterRole() *rbacv1.ClusterRole { return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{ + Kind: "ClusterRole", + APIVersion: rbacv1.SchemeGroupVersion.String(), + }, ObjectMeta: metav1.ObjectMeta{ - Name: "vsphere-csi-controller-role", + Name: CSIControllerClusterRoleName, }, Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"storage.k8s.io"}, - Resources: []string{"csidrivers"}, - Verbs: []string{"create", "delete"}, - }, { APIGroups: []string{""}, - Resources: []string{"nodes", "pods", "secrets", "configmaps"}, + Resources: []string{"nodes", "pods"}, Verbs: []string{"get", "list", "watch"}, }, { APIGroups: []string{""}, - Resources: []string{"persistentvolumes"}, - Verbs: []string{"get", "list", "watch", "update", "create", "delete", "patch"}, + Resources: []string{"configmaps"}, + Verbs: []string{"get", "list", "watch", "create"}, }, { - APIGroups: []string{"storage.k8s.io"}, - Resources: []string{"volumeattachments"}, - Verbs: []string{"get", "list", "watch", "update", "patch"}, + APIGroups: []string{""}, + Resources: []string{"persistentvolumeclaims"}, + Verbs: []string{"get", "list", "watch", "update"}, }, { - APIGroups: []string{"storage.k8s.io"}, - Resources: []string{"volumeattachments/status"}, + APIGroups: []string{""}, + Resources: []string{"persistentvolumeclaims/status"}, Verbs: []string{"patch"}, }, { APIGroups: []string{""}, - Resources: []string{"persistentvolumeclaims"}, - Verbs: []string{"get", "list", "watch", "update"}, - }, - { - APIGroups: []string{"storage.k8s.io"}, - Resources: []string{"storageclasses", "csinodes"}, - Verbs: []string{"get", "list", "watch"}, + Resources: []string{"persistentvolumes"}, + Verbs: []string{"get", "list", "watch", "create", "update", "delete", "patch"}, }, { APIGroups: []string{""}, Resources: []string{"events"}, - Verbs: []string{"list", "watch", "create", "update", "patch"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch"}, }, { APIGroups: []string{"coordination.k8s.io"}, Resources: []string{"leases"}, Verbs: []string{"get", "watch", "list", "delete", "update", "create"}, }, + { + APIGroups: []string{"storage.k8s.io"}, + Resources: []string{"storageclasses", "csinodes"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"storage.k8s.io"}, + Resources: []string{"volumeattachments"}, + Verbs: []string{"get", "list", "watch", "patch"}, + }, + { + APIGroups: []string{"cns.vmware.com"}, + Resources: []string{"triggercsifullsyncs"}, + Verbs: []string{"create", "get", "update", "watch", "list"}, + }, + { + APIGroups: []string{"cns.vmware.com"}, + Resources: []string{"cnsvspherevolumemigrations"}, + Verbs: []string{"create", "get", "list", "watch", "update", "delete"}, + }, + { + APIGroups: []string{"cns.vmware.com"}, + Resources: []string{"cnsvolumeinfoes"}, + Verbs: []string{"create", "get", "list", "watch", "delete"}, + }, + { + APIGroups: []string{"apiextensions.k8s.io"}, + Resources: []string{"customresourcedefinitions"}, + Verbs: []string{"get", "create", "update"}, + }, + { + APIGroups: []string{"storage.k8s.io"}, + Resources: []string{"volumeattachments/status"}, + Verbs: []string{"patch"}, + }, + { + APIGroups: []string{"cns.vmware.com"}, + Resources: []string{"cnsvolumeoperationrequests"}, + Verbs: []string{"create", "get", "list", "update", "delete"}, + }, { APIGroups: []string{"snapshot.storage.k8s.io"}, Resources: []string{"volumesnapshots"}, Verbs: []string{"get", "list"}, }, + { + APIGroups: []string{"snapshot.storage.k8s.io"}, + Resources: []string{"volumesnapshotclasses"}, + Verbs: []string{"watch", "get", "list"}, + }, { APIGroups: []string{"snapshot.storage.k8s.io"}, Resources: []string{"volumesnapshotcontents"}, - Verbs: []string{"get", "list"}, + Verbs: []string{"create", "get", "list", "watch", "update", "delete", "patch"}, + }, + { + APIGroups: []string{"snapshot.storage.k8s.io"}, + Resources: []string{"volumesnapshotcontents/status"}, + Verbs: []string{"update", "patch"}, + }, + { + APIGroups: []string{"cns.vmware.com"}, + Resources: []string{"csinodetopologies"}, + Verbs: []string{"get", "update", "watch", "list"}, }, }, } @@ -118,8 +197,12 @@ func CSIControllerClusterRole() *rbacv1.ClusterRole { func CSIControllerClusterRoleBinding() *rbacv1.ClusterRoleBinding { return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{ + Kind: "ClusterRoleBinding", + APIVersion: rbacv1.SchemeGroupVersion.String(), + }, ObjectMeta: metav1.ObjectMeta{ - Name: "vsphere-csi-controller-binding", + Name: CSIControllerRoleBindingName, }, Subjects: []rbacv1.Subject{ { @@ -130,18 +213,128 @@ func CSIControllerClusterRoleBinding() *rbacv1.ClusterRoleBinding { }, RoleRef: rbacv1.RoleRef{ Kind: "ClusterRole", - Name: "vsphere-csi-controller-role", + Name: CSIControllerClusterRoleName, + APIGroup: "rbac.authorization.k8s.io", + }, + } +} + +func CSINodeDaemonSetServiceAccount() *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{ + Kind: "ServiceAccount", + APIVersion: corev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: CSINodeDaemonSetName, + Namespace: CSINamespace, + }, + } +} + +func CSINodeRole() *rbacv1.Role { + return &rbacv1.Role{ + TypeMeta: metav1.TypeMeta{ + Kind: "Role", + APIVersion: rbacv1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: CSINodeRoleName, + Namespace: CSINamespace, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"get", "list", "watch"}, + }, + }, + } +} + +func CSINodeRoleBinding() *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{ + Kind: "RoleBinding", + APIVersion: rbacv1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: CSINodeRoleBindingName, + Namespace: CSINamespace, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: CSINodeDaemonSetName, + Namespace: CSINamespace, + }, + }, + RoleRef: rbacv1.RoleRef{ + Kind: "Role", + Name: CSINodeRoleName, + APIGroup: "rbac.authorization.k8s.io", + }, + } +} + +func CSINodeClusterRole() *rbacv1.ClusterRole { + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{ + Kind: "ClusterRole", + APIVersion: rbacv1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: CSINodeClusterRoleName, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"cns.vmware.com"}, + Resources: []string{"csinodetopologies"}, + Verbs: []string{"create", "watch", "get", "patch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"nodes"}, + Verbs: []string{"get"}, + }, + }, + } +} + +func CSINodeClusterRoleBinding() *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{ + Kind: "ClusterRoleBinding", + APIVersion: rbacv1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: CSINodeClusterRoleBindingName, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: CSINodeDaemonSetName, + Namespace: CSINamespace, + }, + }, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: CSINodeClusterRoleName, APIGroup: "rbac.authorization.k8s.io", }, } } -func CSIDriver() *storagev1beta1.CSIDriver { - return &storagev1beta1.CSIDriver{ +func CSIDriver() *storagev1.CSIDriver { + return &storagev1.CSIDriver{ + TypeMeta: metav1.TypeMeta{ + Kind: "CSIDriver", + APIVersion: storagev1.SchemeGroupVersion.String(), + }, ObjectMeta: metav1.ObjectMeta{ Name: "csi.vsphere.vmware.com", }, - Spec: storagev1beta1.CSIDriverSpec{ + Spec: storagev1.CSIDriverSpec{ AttachRequired: boolPtr(true), PodInfoOnMount: boolPtr(false), }, @@ -150,28 +343,41 @@ func CSIDriver() *storagev1beta1.CSIDriver { func VSphereCSINodeDaemonSet(storageConfig *types.CPIStorageConfig) *appsv1.DaemonSet { return &appsv1.DaemonSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "DaemonSet", + APIVersion: appsv1.SchemeGroupVersion.String(), + }, ObjectMeta: metav1.ObjectMeta{ - Name: "vsphere-csi-node", + Name: CSINodeDaemonSetName, Namespace: CSINamespace, }, Spec: appsv1.DaemonSetSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - "app": "vsphere-csi-node", + "app": CSINodeDaemonSetName, }, }, UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ Type: appsv1.RollingUpdateDaemonSetStrategyType, + RollingUpdate: &appsv1.RollingUpdateDaemonSet{ + MaxUnavailable: newIntOrStringFromInt(1), + }, }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - "app": "vsphere-csi-node", + "app": CSINodeDaemonSetName, "role": "vsphere-csi", }, }, Spec: corev1.PodSpec{ - DNSPolicy: corev1.DNSDefault, + PriorityClassName: "system-node-critical", + NodeSelector: map[string]string{ + "kubernetes.io/os": "linux", + }, + ServiceAccountName: CSINodeDaemonSetName, + HostNetwork: true, + DNSPolicy: corev1.DNSClusterFirstWithHostNet, Containers: []corev1.Container{ NodeDriverRegistrarContainer(storageConfig.RegistrarImage), VSphereCSINodeContainer(storageConfig.NodeDriverImage), @@ -188,14 +394,6 @@ func VSphereCSINodeDaemonSet(storageConfig *types.CPIStorageConfig) *appsv1.Daem }, }, Volumes: []corev1.Volume{ - { - Name: "vsphere-config-volume", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "csi-vsphere-config", - }, - }, - }, { Name: "registration-dir", VolumeSource: corev1.VolumeSource{ @@ -209,7 +407,7 @@ func VSphereCSINodeDaemonSet(storageConfig *types.CPIStorageConfig) *appsv1.Daem Name: "plugin-dir", VolumeSource: corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ - Path: "/var/lib/kubelet/plugins/csi.vsphere.vmware.com/", + Path: "/var/lib/kubelet/plugins/csi.vsphere.vmware.com", Type: newHostPathType(string(corev1.HostPathDirectoryOrCreate)), }, }, @@ -231,6 +429,24 @@ func VSphereCSINodeDaemonSet(storageConfig *types.CPIStorageConfig) *appsv1.Daem }, }, }, + { + Name: "blocks-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/sys/block", + Type: newHostPathType(string(corev1.HostPathDirectory)), + }, + }, + }, + { + Name: "sys-devices-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/sys/devices", + Type: newHostPathType(string(corev1.HostPathDirectory)), + }, + }, + }, }, }, }, @@ -242,17 +458,6 @@ func NodeDriverRegistrarContainer(image string) corev1.Container { return corev1.Container{ Name: "node-driver-registrar", Image: image, - Lifecycle: &corev1.Lifecycle{ - PreStop: &corev1.LifecycleHandler{ - Exec: &corev1.ExecAction{ - Command: []string{ - "/bin/sh", - "-c", - "rm -rf /registration/csi.vsphere.vmware.com-reg.sock /csi/csi.sock", - }, - }, - }, - }, Args: []string{ "--v=5", "--csi-address=$(ADDRESS)", @@ -268,8 +473,17 @@ func NodeDriverRegistrarContainer(image string) corev1.Container { Value: "/var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock", }, }, - SecurityContext: &corev1.SecurityContext{ - Privileged: boolPtr(true), + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{ + "/csi-node-driver-registrar", + "--kubelet-registration-path=/var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock", + "--mode=kubelet-registration-probe", + }, + }, + }, + InitialDelaySeconds: 3, }, VolumeMounts: []corev1.VolumeMount{ { @@ -288,11 +502,27 @@ func VSphereCSINodeContainer(image string) corev1.Container { return corev1.Container{ Name: "vsphere-csi-node", Image: image, + Args: []string{ + "--fss-name=internal-feature-states.csi.vsphere.vmware.com", + "--fss-namespace=$(CSI_NAMESPACE)", + }, Env: []corev1.EnvVar{ + { + Name: "NODE_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, { Name: "CSI_ENDPOINT", Value: "unix:///csi/csi.sock", }, + { + Name: "MAX_VOLUMES_PER_NODE", + Value: "59", + }, { Name: "X_CSI_MODE", Value: "node", @@ -302,25 +532,29 @@ func VSphereCSINodeContainer(image string) corev1.Container { Value: "false", }, { - Name: "VSPHERE_CSI_CONFIG", - Value: "/etc/cloud/csi-vsphere.conf", + Name: "X_CSI_SPEC_DISABLE_LEN_CHECK", + Value: "true", }, { Name: "LOGGER_LEVEL", Value: "PRODUCTION", }, { - Name: "X_CSI_LOG_LEVEL", - Value: "INFO", + Name: "GODEBUG", + Value: "x509sha1=1", }, { - Name: "NODE_NAME", + Name: "CSI_NAMESPACE", ValueFrom: &corev1.EnvVarSource{ FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "spec.nodeName", + FieldPath: "metadata.namespace", }, }, }, + { + Name: "NODEGETINFO_WATCH_TIMEOUT_MINUTES", + Value: "1", + }, }, Ports: []corev1.ContainerPort{ { @@ -337,22 +571,18 @@ func VSphereCSINodeContainer(image string) corev1.Container { }, }, InitialDelaySeconds: 10, - TimeoutSeconds: 3, + TimeoutSeconds: 5, PeriodSeconds: 5, FailureThreshold: 3, }, SecurityContext: &corev1.SecurityContext{ Privileged: boolPtr(true), Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{corev1.Capability("SYS_ADMIN")}, + Add: []corev1.Capability{"SYS_ADMIN"}, }, AllowPrivilegeEscalation: boolPtr(true), }, VolumeMounts: []corev1.VolumeMount{ - { - Name: "vsphere-config-volume", - MountPath: "/etc/cloud", - }, { Name: "plugin-dir", MountPath: "/csi", @@ -366,6 +596,14 @@ func VSphereCSINodeContainer(image string) corev1.Container { Name: "device-dir", MountPath: "/dev", }, + { + Name: "blocks-dir", + MountPath: "/sys/block", + }, + { + Name: "sys-devices-dir", + MountPath: "/sys/devices", + }, }, } } @@ -374,7 +612,10 @@ func LivenessProbeForNodeContainer(image string) corev1.Container { return corev1.Container{ Name: "liveness-probe", Image: image, - Args: []string{"--csi-address=/csi/csi.sock"}, + Args: []string{ + "--v=4", + "--csi-address=/csi/csi.sock", + }, VolumeMounts: []corev1.VolumeMount{ { Name: "plugin-dir", @@ -386,12 +627,23 @@ func LivenessProbeForNodeContainer(image string) corev1.Container { func CSIControllerDeployment(storageConfig *types.CPIStorageConfig) *appsv1.Deployment { return &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: appsv1.SchemeGroupVersion.String(), + }, ObjectMeta: metav1.ObjectMeta{ Name: CSIControllerName, Namespace: CSINamespace, }, Spec: appsv1.DeploymentSpec{ Replicas: boolInt32(1), + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxUnavailable: newIntOrStringFromInt(1), + MaxSurge: newIntOrStringFromInt(0), + }, + }, Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "app": CSIControllerName, @@ -405,7 +657,29 @@ func CSIControllerDeployment(storageConfig *types.CPIStorageConfig) *appsv1.Depl }, }, Spec: corev1.PodSpec{ + PriorityClassName: "system-cluster-critical", + Affinity: &corev1.Affinity{ + PodAntiAffinity: &corev1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ + { + TopologyKey: "kubernetes.io/hostname", + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "app", + Operator: metav1.LabelSelectorOpIn, + Values: []string{CSIControllerName}, + }, + }, + }, + }, + }, + }, + }, ServiceAccountName: CSIControllerName, + NodeSelector: map[string]string{ + "node-role.kubernetes.io/control-plane": "", + }, Tolerations: []corev1.Toleration{ { Key: "node-role.kubernetes.io/master", @@ -421,10 +695,12 @@ func CSIControllerDeployment(storageConfig *types.CPIStorageConfig) *appsv1.Depl DNSPolicy: corev1.DNSDefault, Containers: []corev1.Container{ CSIAttacherContainer(storageConfig.AttacherImage), + CSIResizerContainer(storageConfig.ResizerImage), VSphereCSIControllerContainer(storageConfig.ControllerImage), LivenessProbeForCSIControllerContainer(storageConfig.LivenessProbeImage), VSphereSyncerContainer(storageConfig.MetadataSyncerImage), CSIProvisionerContainer(storageConfig.ProvisionerImage), + CSISnapshotterContainer(storageConfig.SnapshotterImage), }, Volumes: []corev1.Volume{ { @@ -452,7 +728,17 @@ func CSIAttacherContainer(image string) corev1.Container { return corev1.Container{ Name: "csi-attacher", Image: image, - Args: []string{"--v=4", "--timeout=300s", "--csi-address=$(ADDRESS)", "--leader-election"}, + Args: []string{ + "--v=4", + "--timeout=300s", + "--csi-address=$(ADDRESS)", + "--leader-election", + "--leader-election-lease-duration=120s", + "--leader-election-renew-deadline=60s", + "--leader-election-retry-period=30s", + "--kube-api-qps=100", + "--kube-api-burst=100", + }, Env: []corev1.EnvVar{ { Name: "ADDRESS", @@ -468,38 +754,62 @@ func CSIAttacherContainer(image string) corev1.Container { } } -func VSphereCSIControllerContainer(image string) corev1.Container { +func CSIResizerContainer(image string) corev1.Container { return corev1.Container{ - Name: CSIControllerName, + Name: "csi-resizer", Image: image, - Ports: []corev1.ContainerPort{ + Args: []string{ + "--v=4", + "--timeout=300s", + "--handle-volume-inuse-error=false", + "--csi-address=$(ADDRESS)", + "--kube-api-qps=100", + "--kube-api-burst=100", + "--leader-election", + "--leader-election-lease-duration=120s", + "--leader-election-renew-deadline=60s", + "--leader-election-retry-period=30s", + }, + Env: []corev1.EnvVar{ { - Name: "healthz", - ContainerPort: 9808, - Protocol: corev1.ProtocolTCP, + Name: "ADDRESS", + Value: "/csi/csi.sock", }, }, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/healthz", - Port: intstr.Parse("healthz"), - }, + VolumeMounts: []corev1.VolumeMount{ + { + MountPath: "/csi", + Name: "socket-dir", }, - InitialDelaySeconds: 10, - TimeoutSeconds: 3, - PeriodSeconds: 5, - FailureThreshold: 3, + }, + } +} + +func VSphereCSIControllerContainer(image string) corev1.Container { + return corev1.Container{ + Name: CSIControllerName, + Image: image, + Args: []string{ + "--fss-name=internal-feature-states.csi.vsphere.vmware.com", + "--fss-namespace=$(CSI_NAMESPACE)", }, Env: []corev1.EnvVar{ { Name: "CSI_ENDPOINT", - Value: "unix:///var/lib/csi/sockets/pluginproxy/csi.sock", + Value: "unix:///csi/csi.sock", }, { Name: "X_CSI_MODE", Value: "controller", }, + { + Name: "X_CSI_SPEC_DISABLE_LEN_CHECK", + Value: "true", + }, + { + Name: "X_CSI_SERIAL_VOL_ACCESS_TIMEOUT", + Value: "3m", + }, { Name: "VSPHERE_CSI_CONFIG", Value: "/etc/cloud/csi-vsphere.conf", @@ -509,10 +819,27 @@ func VSphereCSIControllerContainer(image string) corev1.Container { Value: "PRODUCTION", }, { - Name: "X_CSI_LOG_LEVEL", - Value: "INFO", + Name: "INCLUSTER_CLIENT_QPS", + Value: "100", + }, + { + Name: "INCLUSTER_CLIENT_BURST", + Value: "100", + }, + { + Name: "CSI_NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }, }, }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: int64ptr(65532), + RunAsGroup: int64ptr(65532), + RunAsNonRoot: boolPtr(true), + }, VolumeMounts: []corev1.VolumeMount{ { MountPath: "/etc/cloud", @@ -520,10 +847,34 @@ func VSphereCSIControllerContainer(image string) corev1.Container { ReadOnly: true, }, { - MountPath: "/var/lib/csi/sockets/pluginproxy/", + MountPath: "/csi", Name: "socket-dir", }, }, + Ports: []corev1.ContainerPort{ + { + Name: "healthz", + ContainerPort: 9808, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "prometheus", + ContainerPort: 2112, + Protocol: corev1.ProtocolTCP, + }, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.Parse("healthz"), + }, + }, + InitialDelaySeconds: 30, + TimeoutSeconds: 10, + PeriodSeconds: 180, + FailureThreshold: 3, + }, } } @@ -531,16 +882,13 @@ func LivenessProbeForCSIControllerContainer(image string) corev1.Container { return corev1.Container{ Name: "liveness-probe", Image: image, - Args: []string{"--csi-address=$(ADDRESS)"}, - Env: []corev1.EnvVar{ - { - Name: "ADDRESS", - Value: "/var/lib/csi/sockets/pluginproxy/csi.sock", - }, + Args: []string{ + "--v=4", + "--csi-address=/csi/csi.sock", }, VolumeMounts: []corev1.VolumeMount{ { - MountPath: "/var/lib/csi/sockets/pluginproxy/", + MountPath: "/csi", Name: "socket-dir", }, }, @@ -551,21 +899,60 @@ func VSphereSyncerContainer(image string) corev1.Container { return corev1.Container{ Name: "vsphere-syncer", Image: image, - Args: []string{"--leader-election"}, + Args: []string{ + "--leader-election", + "--leader-election-lease-duration=30s", + "--leader-election-renew-deadline=20s", + "--leader-election-retry-period=10s", + "--fss-name=internal-feature-states.csi.vsphere.vmware.com", + "--fss-namespace=$(CSI_NAMESPACE)", + }, + Ports: []corev1.ContainerPort{ + { + Name: "prometheus", + ContainerPort: 2113, + Protocol: corev1.ProtocolTCP, + }, + }, Env: []corev1.EnvVar{ { - Name: "X_CSI_FULL_SYNC_INTERVAL_MINUTES", + Name: "FULL_SYNC_INTERVAL_MINUTES", Value: "30", }, + { + Name: "VSPHERE_CSI_CONFIG", + Value: "/etc/cloud/csi-vsphere.conf", + }, { Name: "LOGGER_LEVEL", Value: "PRODUCTION", }, { - Name: "VSPHERE_CSI_CONFIG", - Value: "/etc/cloud/csi-vsphere.conf", + Name: "INCLUSTER_CLIENT_QPS", + Value: "100", + }, + { + Name: "INCLUSTER_CLIENT_BURST", + Value: "100", + }, + { + Name: "GODEBUG", + Value: "x509sha1=1", + }, + { + Name: "CSI_NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }, }, }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: int64ptr(65532), + RunAsGroup: int64ptr(65532), + RunAsNonRoot: boolPtr(true), + }, VolumeMounts: []corev1.VolumeMount{ { MountPath: "/etc/cloud", @@ -584,8 +971,16 @@ func CSIProvisionerContainer(image string) corev1.Container { "--v=4", "--timeout=300s", "--csi-address=$(ADDRESS)", + "--kube-api-qps=100", + "--kube-api-burst=100", "--leader-election", + "--leader-election-lease-duration=120s", + "--leader-election-renew-deadline=60s", + "--leader-election-retry-period=30s", "--default-fstype=ext4", + // needed only for topology aware setup + // "--feature-gates=Topology=true", + // "--strict-topology" }, Env: []corev1.EnvVar{ { @@ -602,23 +997,79 @@ func CSIProvisionerContainer(image string) corev1.Container { } } -func CSICloudConfigSecret(data string) *corev1.Secret { - return &corev1.Secret{ +func CSISnapshotterContainer(image string) corev1.Container { + return corev1.Container{ + Name: "csi-snapshotter", + Image: image, + Args: []string{ + "--v=4", + "--kube-api-qps=100", + "--kube-api-burst=100", + "--timeout=300s", + "--csi-address=$(ADDRESS)", + "--leader-election", + "--leader-election-lease-duration=120s", + "--leader-election-renew-deadline=60s", + "--leader-election-retry-period=30s", + }, + Env: []corev1.EnvVar{ + { + Name: "ADDRESS", + Value: "/csi/csi.sock", + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + MountPath: "/csi", + Name: "socket-dir", + }, + }, + } +} + +func CSIControllerService() *corev1.Service { + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Service", + }, ObjectMeta: metav1.ObjectMeta{ - Name: "csi-vsphere-config", + Name: CSIControllerName, Namespace: CSINamespace, + Labels: map[string]string{ + "app": CSIControllerName, + }, }, - Type: corev1.SecretTypeOpaque, - StringData: map[string]string{ - "csi-vsphere.conf": data, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "app": CSIControllerName, + }, + Ports: []corev1.ServicePort{ + { + Name: "ctlr", + Protocol: corev1.ProtocolTCP, + Port: 2112, + TargetPort: intstr.FromInt(2112), + }, + { + Name: "syncer", + Protocol: corev1.ProtocolTCP, + Port: 2113, + TargetPort: intstr.FromInt(2113), + }, + }, }, } } -func CSIComponentConfigSecret(secretName string, data string) *corev1.Secret { +func CSICloudConfigSecret(data string) *corev1.Secret { return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: corev1.SchemeGroupVersion.String(), + }, ObjectMeta: metav1.ObjectMeta{ - Name: secretName, + Name: "csi-vsphere-config", Namespace: CSINamespace, }, Type: corev1.SecretTypeOpaque, @@ -639,7 +1090,21 @@ func CSIFeatureStatesConfigMap() *corev1.ConfigMap { Namespace: CSINamespace, }, Data: map[string]string{ - "csi-migration": "false", + "csi-migration": "true", + "csi-auth-check": "true", + "online-volume-extend": "true", + "trigger-csi-fullsync": "false", + "async-query-volume": "true", + "block-volume-snapshot": "true", + "csi-windows-support": "true", + "list-volumes": "true", + "pv-to-backingdiskobjectid-mapping": "false", + "cnsmgr-suspend-create-volume": "true", + "topology-preferential-datastores": "true", + "max-pvscsi-targets-per-vm": "true", + "multi-vcenter-csi-topology": "true", + "csi-internal-generated-cluster-id": "true", + "listview-tasks": "true", }, } } @@ -658,6 +1123,11 @@ func newHostPathType(pathType string) *corev1.HostPathType { return hostPathType } +func newIntOrStringFromInt(val int) *intstr.IntOrString { + v := intstr.FromInt(val) + return &v +} + func newMountPropagation(propagation string) *corev1.MountPropagationMode { propagationMode := new(corev1.MountPropagationMode) *propagationMode = corev1.MountPropagationMode(propagation) diff --git a/packaging/flavorgen/flavors/crs/csi.go b/packaging/flavorgen/flavors/crs/csi.go index 7974bb1a03..9bfae3d4d7 100644 --- a/packaging/flavorgen/flavors/crs/csi.go +++ b/packaging/flavorgen/flavors/crs/csi.go @@ -20,11 +20,6 @@ import ( "fmt" "github.com/pkg/errors" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - rbac "k8s.io/api/rbac/v1" - storagev1 "k8s.io/api/storage/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" addonsv1 "sigs.k8s.io/cluster-api/exp/addons/api/v1beta1" @@ -36,29 +31,41 @@ import ( // CreateCrsResourceObjectsCSI creates the api objects necessary for CSI to function. // Also appends the resources to the CRS. func CreateCrsResourceObjectsCSI(crs *addonsv1.ClusterResourceSet) []runtime.Object { - serviceAccount := cloudprovider.CSIControllerServiceAccount() - serviceAccount.TypeMeta = metav1.TypeMeta{ - Kind: "ServiceAccount", - APIVersion: corev1.SchemeGroupVersion.String(), - } - serviceAccountSecret := newSecret(serviceAccount.Name, serviceAccount) - appendSecretToCrsResource(crs, serviceAccountSecret) + csiNamespace := cloudprovider.CSIDriverNamespace() + csiNamespaceConfigMap := newConfigMap("csi-namespace", csiNamespace) + appendConfigMapToCrsResource(crs, csiNamespaceConfigMap) - clusterRole := cloudprovider.CSIControllerClusterRole() - clusterRole.TypeMeta = metav1.TypeMeta{ - Kind: "ClusterRole", - APIVersion: rbac.SchemeGroupVersion.String(), - } - clusterRoleConfigMap := newConfigMap(clusterRole.Name, clusterRole) - appendConfigMapToCrsResource(crs, clusterRoleConfigMap) + controllerServiceAccount := cloudprovider.CSIControllerServiceAccount() + controllerServiceAccountSecret := newSecret(controllerServiceAccount.Name, controllerServiceAccount) + appendSecretToCrsResource(crs, controllerServiceAccountSecret) - clusterRoleBinding := cloudprovider.CSIControllerClusterRoleBinding() - clusterRoleBinding.TypeMeta = metav1.TypeMeta{ - Kind: "ClusterRoleBinding", - APIVersion: rbac.SchemeGroupVersion.String(), - } - clusterRoleBindingConfigMap := newConfigMap(clusterRoleBinding.Name, clusterRoleBinding) - appendConfigMapToCrsResource(crs, clusterRoleBindingConfigMap) + controllerClusterRole := cloudprovider.CSIControllerClusterRole() + controllerClusterRoleConfigMap := newConfigMap(controllerClusterRole.Name, controllerClusterRole) + appendConfigMapToCrsResource(crs, controllerClusterRoleConfigMap) + + controllerClusterRoleBinding := cloudprovider.CSIControllerClusterRoleBinding() + controllerClusterRoleBindingConfigMap := newConfigMap(controllerClusterRoleBinding.Name, controllerClusterRoleBinding) + appendConfigMapToCrsResource(crs, controllerClusterRoleBindingConfigMap) + + nodeDaemonSetServiceAccount := cloudprovider.CSINodeDaemonSetServiceAccount() + nodeDaemonSetServiceAccountSecret := newSecret(nodeDaemonSetServiceAccount.Name, nodeDaemonSetServiceAccount) + appendSecretToCrsResource(crs, nodeDaemonSetServiceAccountSecret) + + nodeRole := cloudprovider.CSINodeRole() + nodeRoleConfigMap := newConfigMap(nodeRole.Name, nodeRole) + appendConfigMapToCrsResource(crs, nodeRoleConfigMap) + + nodeRoleBinding := cloudprovider.CSINodeRoleBinding() + nodeRoleBindingConfigMap := newConfigMap(nodeRoleBinding.Name, nodeRoleBinding) + appendConfigMapToCrsResource(crs, nodeRoleBindingConfigMap) + + nodeClusterRole := cloudprovider.CSINodeClusterRole() + nodeClusterRoleConfigMap := newConfigMap(nodeClusterRole.Name, nodeClusterRole) + appendConfigMapToCrsResource(crs, nodeClusterRoleConfigMap) + + nodeClusterRoleBinding := cloudprovider.CSINodeClusterRoleBinding() + nodeClusterRoleBindingConfigMap := newConfigMap(nodeClusterRoleBinding.Name, nodeClusterRoleBinding) + appendConfigMapToCrsResource(crs, nodeClusterRoleBindingConfigMap) cloudConfig, err := ConfigForCSI().MarshalINI() if err != nil { @@ -66,46 +73,46 @@ func CreateCrsResourceObjectsCSI(crs *addonsv1.ClusterResourceSet) []runtime.Obj } // cloud config secret is wrapped in another secret so it could be injected via CRS cloudConfigSecret := cloudprovider.CSICloudConfigSecret(string(cloudConfig)) - cloudConfigSecret.TypeMeta = metav1.TypeMeta{ - Kind: "Secret", - APIVersion: corev1.SchemeGroupVersion.String(), - } cloudConfigSecretWrapper := newSecret(cloudConfigSecret.Name, cloudConfigSecret) appendSecretToCrsResource(crs, cloudConfigSecretWrapper) csiDriver := cloudprovider.CSIDriver() - csiDriver.TypeMeta = metav1.TypeMeta{ - Kind: "CSIDriver", - APIVersion: storagev1.SchemeGroupVersion.String(), - } csiDriverConfigMap := newConfigMap(csiDriver.Name, csiDriver) appendConfigMapToCrsResource(crs, csiDriverConfigMap) storageConfig := createStorageConfig() daemonSet := cloudprovider.VSphereCSINodeDaemonSet(storageConfig) - daemonSet.TypeMeta = metav1.TypeMeta{ - Kind: "DaemonSet", - APIVersion: appsv1.SchemeGroupVersion.String(), - } daemonSetConfigMap := newConfigMap(daemonSet.Name, daemonSet) appendConfigMapToCrsResource(crs, daemonSetConfigMap) deployment := cloudprovider.CSIControllerDeployment(storageConfig) - deployment.TypeMeta = metav1.TypeMeta{ - Kind: "Deployment", - APIVersion: appsv1.SchemeGroupVersion.String(), - } deploymentConfigMap := newConfigMap(deployment.Name, deployment) appendConfigMapToCrsResource(crs, deploymentConfigMap) + csiService := cloudprovider.CSIControllerService() + csiServiceConfigMap := newConfigMap(csiService.Name+"-service", csiService) + appendConfigMapToCrsResource(crs, csiServiceConfigMap) + + csiFeatureStates := cloudprovider.CSIFeatureStatesConfigMap() + csiFeatureStatesConfigMap := newConfigMap("internal-feature-states", csiFeatureStates) + appendConfigMapToCrsResource(crs, csiFeatureStatesConfigMap) + return []runtime.Object{ - serviceAccountSecret, - clusterRoleConfigMap, - clusterRoleBindingConfigMap, + csiNamespaceConfigMap, + controllerServiceAccountSecret, + controllerClusterRoleConfigMap, + controllerClusterRoleBindingConfigMap, + nodeDaemonSetServiceAccountSecret, + nodeRoleConfigMap, + nodeRoleBindingConfigMap, + nodeClusterRoleConfigMap, + nodeClusterRoleBindingConfigMap, cloudConfigSecretWrapper, csiDriverConfigMap, daemonSetConfigMap, deploymentConfigMap, + csiFeatureStatesConfigMap, + csiServiceConfigMap, } } @@ -115,7 +122,9 @@ func createStorageConfig() *types.CPIStorageConfig { ControllerImage: cloudprovider.DefaultCSIControllerImage, NodeDriverImage: cloudprovider.DefaultCSINodeDriverImage, AttacherImage: cloudprovider.DefaultCSIAttacherImage, + ResizerImage: cloudprovider.DefaultCSIResizerImage, ProvisionerImage: cloudprovider.DefaultCSIProvisionerImage, + SnapshotterImage: cloudprovider.DefaultCSISnapshotterContainerImage, MetadataSyncerImage: cloudprovider.DefaultCSIMetadataSyncerImage, LivenessProbeImage: cloudprovider.DefaultCSILivenessProbeImage, RegistrarImage: cloudprovider.DefaultCSIRegistrarImage, diff --git a/packaging/flavorgen/flavors/crs/types/cloudprovider_types.go b/packaging/flavorgen/flavors/crs/types/cloudprovider_types.go index e541ad27c1..2970b2213e 100644 --- a/packaging/flavorgen/flavors/crs/types/cloudprovider_types.go +++ b/packaging/flavorgen/flavors/crs/types/cloudprovider_types.go @@ -75,7 +75,9 @@ type CPIStorageConfig struct { ControllerImage string `json:"controllerImage,omitempty"` NodeDriverImage string `json:"nodeDriverImage,omitempty"` AttacherImage string `json:"attacherImage,omitempty"` + ResizerImage string `json:"resizerImage,omitempty"` ProvisionerImage string `json:"provisionerImage,omitempty"` + SnapshotterImage string `json:"snapshotterImage,omitempty"` MetadataSyncerImage string `json:"metadataSyncerImage,omitempty"` LivenessProbeImage string `json:"livenessProbeImage,omitempty"` RegistrarImage string `json:"registrarImage,omitempty"` diff --git a/packaging/flavorgen/flavors/flavors.go b/packaging/flavorgen/flavors/flavors.go index dbed2c2777..49641636ee 100644 --- a/packaging/flavorgen/flavors/flavors.go +++ b/packaging/flavorgen/flavors/flavors.go @@ -20,7 +20,6 @@ import ( "fmt" "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/crs" "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/env" ) diff --git a/templates/cluster-template-external-loadbalancer.yaml b/templates/cluster-template-external-loadbalancer.yaml index 41447e49e9..50f2acbb4e 100644 --- a/templates/cluster-template-external-loadbalancer.yaml +++ b/templates/cluster-template-external-loadbalancer.yaml @@ -201,12 +201,24 @@ spec: matchLabels: cluster.x-k8s.io/cluster-name: '${CLUSTER_NAME}' resources: + - kind: ConfigMap + name: csi-namespace - kind: Secret name: vsphere-csi-controller - kind: ConfigMap - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role - kind: ConfigMap - name: vsphere-csi-controller-binding + name: vsphere-csi-controller-cluster-role-binding + - kind: Secret + name: vsphere-csi-node + - kind: ConfigMap + name: vsphere-csi-node-role + - kind: ConfigMap + name: vsphere-csi-node-role-binding + - kind: ConfigMap + name: vsphere-csi-node-cluster-role + - kind: ConfigMap + name: vsphere-csi-node-cluster-role-binding - kind: Secret name: csi-vsphere-config - kind: ConfigMap @@ -215,6 +227,10 @@ spec: name: vsphere-csi-node - kind: ConfigMap name: vsphere-csi-controller + - kind: ConfigMap + name: vsphere-csi-controller-service + - kind: ConfigMap + name: internal-feature-states - kind: Secret name: cloud-controller-manager - kind: Secret @@ -232,6 +248,18 @@ stringData: username: ${VSPHERE_USERNAME} --- apiVersion: v1 +data: + data: | + apiVersion: v1 + kind: Namespace + metadata: + name: vmware-system-csi +kind: ConfigMap +metadata: + name: csi-namespace + namespace: '${NAMESPACE}' +--- +apiVersion: v1 kind: Secret metadata: name: vsphere-csi-controller @@ -242,7 +270,7 @@ stringData: kind: ServiceAccount metadata: name: vsphere-csi-controller - namespace: kube-system + namespace: vmware-system-csi type: addons.cluster.x-k8s.io/resource-set --- apiVersion: v1 @@ -251,63 +279,75 @@ data: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role rules: - apiGroups: - - storage.k8s.io + - "" resources: - - csidrivers + - nodes + - pods verbs: - - create - - delete + - get + - list + - watch - apiGroups: - "" resources: - - nodes - - pods - - secrets - configmaps verbs: - get - list - watch + - create - apiGroups: - "" resources: - - persistentvolumes + - persistentvolumeclaims verbs: - get - list - watch - update - - create - - delete + - apiGroups: + - "" + resources: + - persistentvolumeclaims/status + verbs: - patch - apiGroups: - - storage.k8s.io + - "" resources: - - volumeattachments + - persistentvolumes verbs: - get - list - watch + - create - update + - delete - patch - apiGroups: - - storage.k8s.io + - "" resources: - - volumeattachments/status + - events verbs: + - get + - list + - watch + - create + - update - patch - apiGroups: - - "" + - coordination.k8s.io resources: - - persistentvolumeclaims + - leases verbs: - get - - list - watch + - list + - delete - update + - create - apiGroups: - storage.k8s.io resources: @@ -318,26 +358,69 @@ data: - list - watch - apiGroups: - - "" + - storage.k8s.io resources: - - events + - volumeattachments verbs: + - get - list - watch + - patch + - apiGroups: + - cns.vmware.com + resources: + - triggercsifullsyncs + verbs: - create + - get - update - - patch + - watch + - list - apiGroups: - - coordination.k8s.io + - cns.vmware.com resources: - - leases + - cnsvspherevolumemigrations verbs: + - create - get + - list - watch + - update + - delete + - apiGroups: + - cns.vmware.com + resources: + - cnsvolumeinfoes + verbs: + - create + - get - list + - watch - delete + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - create - update + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + - apiGroups: + - cns.vmware.com + resources: + - cnsvolumeoperationrequests + verbs: - create + - get + - list + - update + - delete - apiGroups: - snapshot.storage.k8s.io resources: @@ -345,16 +428,45 @@ data: verbs: - get - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - watch + - get + - list - apiGroups: - snapshot.storage.k8s.io resources: - volumesnapshotcontents verbs: + - create - get - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + - patch + - apiGroups: + - cns.vmware.com + resources: + - csinodetopologies + verbs: + - get + - update + - watch + - list kind: ConfigMap metadata: - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role namespace: '${NAMESPACE}' --- apiVersion: v1 @@ -363,18 +475,123 @@ data: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: vsphere-csi-controller-binding + name: vsphere-csi-controller-cluster-role-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role subjects: - kind: ServiceAccount name: vsphere-csi-controller - namespace: kube-system + namespace: vmware-system-csi +kind: ConfigMap +metadata: + name: vsphere-csi-controller-cluster-role-binding + namespace: '${NAMESPACE}' +--- +apiVersion: v1 +kind: Secret +metadata: + name: vsphere-csi-node + namespace: '${NAMESPACE}' +stringData: + data: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: vsphere-csi-node + namespace: vmware-system-csi +type: addons.cluster.x-k8s.io/resource-set +--- +apiVersion: v1 +data: + data: | + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: vsphere-csi-node-role + namespace: vmware-system-csi + rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch +kind: ConfigMap +metadata: + name: vsphere-csi-node-role + namespace: '${NAMESPACE}' +--- +apiVersion: v1 +data: + data: | + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: vsphere-csi-node-role-binding + namespace: vmware-system-csi + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: vsphere-csi-node-role + subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: vmware-system-csi +kind: ConfigMap +metadata: + name: vsphere-csi-node-role-binding + namespace: '${NAMESPACE}' +--- +apiVersion: v1 +data: + data: | + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: vsphere-csi-node-cluster-role + rules: + - apiGroups: + - cns.vmware.com + resources: + - csinodetopologies + verbs: + - create + - watch + - get + - patch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get kind: ConfigMap metadata: - name: vsphere-csi-controller-binding + name: vsphere-csi-node-cluster-role + namespace: '${NAMESPACE}' +--- +apiVersion: v1 +data: + data: | + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: vsphere-csi-node-cluster-role-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: vsphere-csi-node-cluster-role + subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: vmware-system-csi +kind: ConfigMap +metadata: + name: vsphere-csi-node-cluster-role-binding namespace: '${NAMESPACE}' --- apiVersion: v1 @@ -388,7 +605,7 @@ stringData: kind: Secret metadata: name: csi-vsphere-config - namespace: kube-system + namespace: vmware-system-csi stringData: csi-vsphere.conf: |+ [Global] @@ -427,7 +644,7 @@ data: kind: DaemonSet metadata: name: vsphere-csi-node - namespace: kube-system + namespace: vmware-system-csi spec: selector: matchLabels: @@ -448,41 +665,50 @@ data: value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock - image: quay.io/k8scsi/csi-node-driver-registrar:v2.0.1 - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - rm -rf /registration/csi.vsphere.vmware.com-reg.sock /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock + - --mode=kubelet-registration-probe + initialDelaySeconds: 3 name: node-driver-registrar resources: {} - securityContext: - privileged: true volumeMounts: - mountPath: /csi name: plugin-dir - mountPath: /registration name: registration-dir - - env: + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName - name: CSI_ENDPOINT value: unix:///csi/csi.sock + - name: MAX_VOLUMES_PER_NODE + value: "59" - name: X_CSI_MODE value: node - name: X_CSI_SPEC_REQ_VALIDATION value: "false" - - name: VSPHERE_CSI_CONFIG - value: /etc/cloud/csi-vsphere.conf + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" - name: LOGGER_LEVEL value: PRODUCTION - - name: X_CSI_LOG_LEVEL - value: INFO - - name: NODE_NAME + - name: GODEBUG + value: x509sha1=1 + - name: CSI_NAMESPACE valueFrom: fieldRef: - fieldPath: spec.nodeName - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v2.1.0 + fieldPath: metadata.namespace + - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES + value: "1" + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0 livenessProbe: failureThreshold: 3 httpGet: @@ -490,7 +716,7 @@ data: port: healthz initialDelaySeconds: 10 periodSeconds: 5 - timeoutSeconds: 3 + timeoutSeconds: 5 name: vsphere-csi-node ports: - containerPort: 9808 @@ -504,8 +730,6 @@ data: - SYS_ADMIN privileged: true volumeMounts: - - mountPath: /etc/cloud - name: vsphere-config-volume - mountPath: /csi name: plugin-dir - mountPath: /var/lib/kubelet @@ -513,30 +737,37 @@ data: name: pods-mount-dir - mountPath: /dev name: device-dir + - mountPath: /sys/block + name: blocks-dir + - mountPath: /sys/devices + name: sys-devices-dir - args: + - --v=4 - --csi-address=/csi/csi.sock - image: quay.io/k8scsi/livenessprobe:v2.1.0 + image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 name: liveness-probe resources: {} volumeMounts: - mountPath: /csi name: plugin-dir - dnsPolicy: Default + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + serviceAccountName: vsphere-csi-node tolerations: - effect: NoSchedule operator: Exists - effect: NoExecute operator: Exists volumes: - - name: vsphere-config-volume - secret: - secretName: csi-vsphere-config - hostPath: path: /var/lib/kubelet/plugins_registry type: Directory name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/ + path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com type: DirectoryOrCreate name: plugin-dir - hostPath: @@ -546,7 +777,17 @@ data: - hostPath: path: /dev name: device-dir + - hostPath: + path: /sys/block + type: Directory + name: blocks-dir + - hostPath: + path: /sys/devices + type: Directory + name: sys-devices-dir updateStrategy: + rollingUpdate: + maxUnavailable: 1 type: RollingUpdate kind: ConfigMap metadata: @@ -560,88 +801,168 @@ data: kind: Deployment metadata: name: vsphere-csi-controller - namespace: kube-system + namespace: vmware-system-csi spec: replicas: 1 selector: matchLabels: app: vsphere-csi-controller + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate template: metadata: labels: app: vsphere-csi-controller role: vsphere-csi spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - vsphere-csi-controller + topologyKey: kubernetes.io/hostname containers: - args: - --v=4 - --timeout=300s - --csi-address=$(ADDRESS) - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + - --kube-api-qps=100 + - --kube-api-burst=100 env: - name: ADDRESS value: /csi/csi.sock - image: quay.io/k8scsi/csi-attacher:v3.0.0 + image: registry.k8s.io/sig-storage/csi-attacher:v4.3.0 name: csi-attacher resources: {} volumeMounts: - mountPath: /csi name: socket-dir - - env: + - args: + - --v=4 + - --timeout=300s + - --handle-volume-inuse-error=false + - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-resizer:v1.8.0 + name: csi-resizer + resources: {} + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: - name: CSI_ENDPOINT - value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + value: unix:///csi/csi.sock - name: X_CSI_MODE value: controller + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" + - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT + value: 3m - name: VSPHERE_CSI_CONFIG value: /etc/cloud/csi-vsphere.conf - name: LOGGER_LEVEL value: PRODUCTION - - name: X_CSI_LOG_LEVEL - value: INFO - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v2.1.0 + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0 livenessProbe: failureThreshold: 3 httpGet: path: /healthz port: healthz - initialDelaySeconds: 10 - periodSeconds: 5 - timeoutSeconds: 3 + initialDelaySeconds: 30 + periodSeconds: 180 + timeoutSeconds: 10 name: vsphere-csi-controller ports: - containerPort: 9808 name: healthz protocol: TCP + - containerPort: 2112 + name: prometheus + protocol: TCP resources: {} + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 volumeMounts: - mountPath: /etc/cloud name: vsphere-config-volume readOnly: true - - mountPath: /var/lib/csi/sockets/pluginproxy/ + - mountPath: /csi name: socket-dir - args: - - --csi-address=$(ADDRESS) - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: quay.io/k8scsi/livenessprobe:v2.1.0 + - --v=4 + - --csi-address=/csi/csi.sock + image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 name: liveness-probe resources: {} volumeMounts: - - mountPath: /var/lib/csi/sockets/pluginproxy/ + - mountPath: /csi name: socket-dir - args: - --leader-election + - --leader-election-lease-duration=30s + - --leader-election-renew-deadline=20s + - --leader-election-retry-period=10s + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) env: - - name: X_CSI_FULL_SYNC_INTERVAL_MINUTES + - name: FULL_SYNC_INTERVAL_MINUTES value: "30" - - name: LOGGER_LEVEL - value: PRODUCTION - name: VSPHERE_CSI_CONFIG value: /etc/cloud/csi-vsphere.conf - image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v2.1.0 + - name: LOGGER_LEVEL + value: PRODUCTION + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: GODEBUG + value: x509sha1=1 + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v3.1.0 name: vsphere-syncer + ports: + - containerPort: 2113 + name: prometheus + protocol: TCP resources: {} + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 volumeMounts: - mountPath: /etc/cloud name: vsphere-config-volume @@ -650,18 +971,43 @@ data: - --v=4 - --timeout=300s - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s - --default-fstype=ext4 env: - name: ADDRESS value: /csi/csi.sock - image: quay.io/k8scsi/csi-provisioner:v2.0.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0 name: csi-provisioner resources: {} volumeMounts: - mountPath: /csi name: socket-dir + - args: + - --v=4 + - --kube-api-qps=100 + - --kube-api-burst=100 + - --timeout=300s + - --csi-address=$(ADDRESS) + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2 + name: csi-snapshotter + resources: {} + volumeMounts: + - mountPath: /csi + name: socket-dir dnsPolicy: Default + priorityClassName: system-cluster-critical serviceAccountName: vsphere-csi-controller tolerations: - effect: NoSchedule @@ -682,6 +1028,62 @@ metadata: namespace: '${NAMESPACE}' --- apiVersion: v1 +data: + data: | + apiVersion: v1 + data: + async-query-volume: "true" + block-volume-snapshot: "true" + cnsmgr-suspend-create-volume: "true" + csi-auth-check: "true" + csi-internal-generated-cluster-id: "true" + csi-migration: "true" + csi-windows-support: "true" + list-volumes: "true" + listview-tasks: "true" + max-pvscsi-targets-per-vm: "true" + multi-vcenter-csi-topology: "true" + online-volume-extend: "true" + pv-to-backingdiskobjectid-mapping: "false" + topology-preferential-datastores: "true" + trigger-csi-fullsync: "false" + kind: ConfigMap + metadata: + name: internal-feature-states.csi.vsphere.vmware.com + namespace: vmware-system-csi +kind: ConfigMap +metadata: + name: internal-feature-states + namespace: '${NAMESPACE}' +--- +apiVersion: v1 +data: + data: | + apiVersion: v1 + kind: Service + metadata: + labels: + app: vsphere-csi-controller + name: vsphere-csi-controller + namespace: vmware-system-csi + spec: + ports: + - name: ctlr + port: 2112 + protocol: TCP + targetPort: 2112 + - name: syncer + port: 2113 + protocol: TCP + targetPort: 2113 + selector: + app: vsphere-csi-controller +kind: ConfigMap +metadata: + name: vsphere-csi-controller-service + namespace: '${NAMESPACE}' +--- +apiVersion: v1 kind: Secret metadata: name: cloud-controller-manager diff --git a/templates/cluster-template-ignition.yaml b/templates/cluster-template-ignition.yaml index d4747e4f4e..4f9198b573 100644 --- a/templates/cluster-template-ignition.yaml +++ b/templates/cluster-template-ignition.yaml @@ -343,12 +343,24 @@ spec: matchLabels: cluster.x-k8s.io/cluster-name: '${CLUSTER_NAME}' resources: + - kind: ConfigMap + name: csi-namespace - kind: Secret name: vsphere-csi-controller - kind: ConfigMap - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role - kind: ConfigMap - name: vsphere-csi-controller-binding + name: vsphere-csi-controller-cluster-role-binding + - kind: Secret + name: vsphere-csi-node + - kind: ConfigMap + name: vsphere-csi-node-role + - kind: ConfigMap + name: vsphere-csi-node-role-binding + - kind: ConfigMap + name: vsphere-csi-node-cluster-role + - kind: ConfigMap + name: vsphere-csi-node-cluster-role-binding - kind: Secret name: csi-vsphere-config - kind: ConfigMap @@ -357,6 +369,10 @@ spec: name: vsphere-csi-node - kind: ConfigMap name: vsphere-csi-controller + - kind: ConfigMap + name: vsphere-csi-controller-service + - kind: ConfigMap + name: internal-feature-states - kind: Secret name: cloud-controller-manager - kind: Secret @@ -374,6 +390,18 @@ stringData: username: ${VSPHERE_USERNAME} --- apiVersion: v1 +data: + data: | + apiVersion: v1 + kind: Namespace + metadata: + name: vmware-system-csi +kind: ConfigMap +metadata: + name: csi-namespace + namespace: '${NAMESPACE}' +--- +apiVersion: v1 kind: Secret metadata: name: vsphere-csi-controller @@ -384,7 +412,7 @@ stringData: kind: ServiceAccount metadata: name: vsphere-csi-controller - namespace: kube-system + namespace: vmware-system-csi type: addons.cluster.x-k8s.io/resource-set --- apiVersion: v1 @@ -393,63 +421,75 @@ data: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role rules: - apiGroups: - - storage.k8s.io + - "" resources: - - csidrivers + - nodes + - pods verbs: - - create - - delete + - get + - list + - watch - apiGroups: - "" resources: - - nodes - - pods - - secrets - configmaps verbs: - get - list - watch + - create - apiGroups: - "" resources: - - persistentvolumes + - persistentvolumeclaims verbs: - get - list - watch - update - - create - - delete + - apiGroups: + - "" + resources: + - persistentvolumeclaims/status + verbs: - patch - apiGroups: - - storage.k8s.io + - "" resources: - - volumeattachments + - persistentvolumes verbs: - get - list - watch + - create - update + - delete - patch - apiGroups: - - storage.k8s.io + - "" resources: - - volumeattachments/status + - events verbs: + - get + - list + - watch + - create + - update - patch - apiGroups: - - "" + - coordination.k8s.io resources: - - persistentvolumeclaims + - leases verbs: - get - - list - watch + - list + - delete - update + - create - apiGroups: - storage.k8s.io resources: @@ -460,26 +500,69 @@ data: - list - watch - apiGroups: - - "" + - storage.k8s.io resources: - - events + - volumeattachments verbs: + - get - list - watch + - patch + - apiGroups: + - cns.vmware.com + resources: + - triggercsifullsyncs + verbs: - create + - get - update - - patch + - watch + - list - apiGroups: - - coordination.k8s.io + - cns.vmware.com resources: - - leases + - cnsvspherevolumemigrations verbs: + - create - get + - list - watch + - update + - delete + - apiGroups: + - cns.vmware.com + resources: + - cnsvolumeinfoes + verbs: + - create + - get - list + - watch - delete + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - create - update + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + - apiGroups: + - cns.vmware.com + resources: + - cnsvolumeoperationrequests + verbs: - create + - get + - list + - update + - delete - apiGroups: - snapshot.storage.k8s.io resources: @@ -487,16 +570,45 @@ data: verbs: - get - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - watch + - get + - list - apiGroups: - snapshot.storage.k8s.io resources: - volumesnapshotcontents verbs: + - create - get - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + - patch + - apiGroups: + - cns.vmware.com + resources: + - csinodetopologies + verbs: + - get + - update + - watch + - list kind: ConfigMap metadata: - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role namespace: '${NAMESPACE}' --- apiVersion: v1 @@ -505,18 +617,123 @@ data: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: vsphere-csi-controller-binding + name: vsphere-csi-controller-cluster-role-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role subjects: - kind: ServiceAccount name: vsphere-csi-controller - namespace: kube-system + namespace: vmware-system-csi +kind: ConfigMap +metadata: + name: vsphere-csi-controller-cluster-role-binding + namespace: '${NAMESPACE}' +--- +apiVersion: v1 +kind: Secret +metadata: + name: vsphere-csi-node + namespace: '${NAMESPACE}' +stringData: + data: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: vsphere-csi-node + namespace: vmware-system-csi +type: addons.cluster.x-k8s.io/resource-set +--- +apiVersion: v1 +data: + data: | + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: vsphere-csi-node-role + namespace: vmware-system-csi + rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch kind: ConfigMap metadata: - name: vsphere-csi-controller-binding + name: vsphere-csi-node-role + namespace: '${NAMESPACE}' +--- +apiVersion: v1 +data: + data: | + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: vsphere-csi-node-role-binding + namespace: vmware-system-csi + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: vsphere-csi-node-role + subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: vmware-system-csi +kind: ConfigMap +metadata: + name: vsphere-csi-node-role-binding + namespace: '${NAMESPACE}' +--- +apiVersion: v1 +data: + data: | + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: vsphere-csi-node-cluster-role + rules: + - apiGroups: + - cns.vmware.com + resources: + - csinodetopologies + verbs: + - create + - watch + - get + - patch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get +kind: ConfigMap +metadata: + name: vsphere-csi-node-cluster-role + namespace: '${NAMESPACE}' +--- +apiVersion: v1 +data: + data: | + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: vsphere-csi-node-cluster-role-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: vsphere-csi-node-cluster-role + subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: vmware-system-csi +kind: ConfigMap +metadata: + name: vsphere-csi-node-cluster-role-binding namespace: '${NAMESPACE}' --- apiVersion: v1 @@ -530,7 +747,7 @@ stringData: kind: Secret metadata: name: csi-vsphere-config - namespace: kube-system + namespace: vmware-system-csi stringData: csi-vsphere.conf: |+ [Global] @@ -569,7 +786,7 @@ data: kind: DaemonSet metadata: name: vsphere-csi-node - namespace: kube-system + namespace: vmware-system-csi spec: selector: matchLabels: @@ -590,41 +807,50 @@ data: value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock - image: quay.io/k8scsi/csi-node-driver-registrar:v2.0.1 - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - rm -rf /registration/csi.vsphere.vmware.com-reg.sock /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock + - --mode=kubelet-registration-probe + initialDelaySeconds: 3 name: node-driver-registrar resources: {} - securityContext: - privileged: true volumeMounts: - mountPath: /csi name: plugin-dir - mountPath: /registration name: registration-dir - - env: + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName - name: CSI_ENDPOINT value: unix:///csi/csi.sock + - name: MAX_VOLUMES_PER_NODE + value: "59" - name: X_CSI_MODE value: node - name: X_CSI_SPEC_REQ_VALIDATION value: "false" - - name: VSPHERE_CSI_CONFIG - value: /etc/cloud/csi-vsphere.conf + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" - name: LOGGER_LEVEL value: PRODUCTION - - name: X_CSI_LOG_LEVEL - value: INFO - - name: NODE_NAME + - name: GODEBUG + value: x509sha1=1 + - name: CSI_NAMESPACE valueFrom: fieldRef: - fieldPath: spec.nodeName - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v2.1.0 + fieldPath: metadata.namespace + - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES + value: "1" + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0 livenessProbe: failureThreshold: 3 httpGet: @@ -632,7 +858,7 @@ data: port: healthz initialDelaySeconds: 10 periodSeconds: 5 - timeoutSeconds: 3 + timeoutSeconds: 5 name: vsphere-csi-node ports: - containerPort: 9808 @@ -646,8 +872,6 @@ data: - SYS_ADMIN privileged: true volumeMounts: - - mountPath: /etc/cloud - name: vsphere-config-volume - mountPath: /csi name: plugin-dir - mountPath: /var/lib/kubelet @@ -655,30 +879,37 @@ data: name: pods-mount-dir - mountPath: /dev name: device-dir + - mountPath: /sys/block + name: blocks-dir + - mountPath: /sys/devices + name: sys-devices-dir - args: + - --v=4 - --csi-address=/csi/csi.sock - image: quay.io/k8scsi/livenessprobe:v2.1.0 + image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 name: liveness-probe resources: {} volumeMounts: - mountPath: /csi name: plugin-dir - dnsPolicy: Default + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + serviceAccountName: vsphere-csi-node tolerations: - effect: NoSchedule operator: Exists - effect: NoExecute operator: Exists volumes: - - name: vsphere-config-volume - secret: - secretName: csi-vsphere-config - hostPath: path: /var/lib/kubelet/plugins_registry type: Directory name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/ + path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com type: DirectoryOrCreate name: plugin-dir - hostPath: @@ -688,7 +919,17 @@ data: - hostPath: path: /dev name: device-dir + - hostPath: + path: /sys/block + type: Directory + name: blocks-dir + - hostPath: + path: /sys/devices + type: Directory + name: sys-devices-dir updateStrategy: + rollingUpdate: + maxUnavailable: 1 type: RollingUpdate kind: ConfigMap metadata: @@ -702,88 +943,168 @@ data: kind: Deployment metadata: name: vsphere-csi-controller - namespace: kube-system + namespace: vmware-system-csi spec: replicas: 1 selector: matchLabels: app: vsphere-csi-controller + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate template: metadata: labels: app: vsphere-csi-controller role: vsphere-csi spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - vsphere-csi-controller + topologyKey: kubernetes.io/hostname containers: - args: - --v=4 - --timeout=300s - --csi-address=$(ADDRESS) - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + - --kube-api-qps=100 + - --kube-api-burst=100 env: - name: ADDRESS value: /csi/csi.sock - image: quay.io/k8scsi/csi-attacher:v3.0.0 + image: registry.k8s.io/sig-storage/csi-attacher:v4.3.0 name: csi-attacher resources: {} volumeMounts: - mountPath: /csi name: socket-dir - - env: + - args: + - --v=4 + - --timeout=300s + - --handle-volume-inuse-error=false + - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-resizer:v1.8.0 + name: csi-resizer + resources: {} + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: - name: CSI_ENDPOINT - value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + value: unix:///csi/csi.sock - name: X_CSI_MODE value: controller + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" + - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT + value: 3m - name: VSPHERE_CSI_CONFIG value: /etc/cloud/csi-vsphere.conf - name: LOGGER_LEVEL value: PRODUCTION - - name: X_CSI_LOG_LEVEL - value: INFO - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v2.1.0 + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0 livenessProbe: failureThreshold: 3 httpGet: path: /healthz port: healthz - initialDelaySeconds: 10 - periodSeconds: 5 - timeoutSeconds: 3 + initialDelaySeconds: 30 + periodSeconds: 180 + timeoutSeconds: 10 name: vsphere-csi-controller ports: - containerPort: 9808 name: healthz protocol: TCP + - containerPort: 2112 + name: prometheus + protocol: TCP resources: {} + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 volumeMounts: - mountPath: /etc/cloud name: vsphere-config-volume readOnly: true - - mountPath: /var/lib/csi/sockets/pluginproxy/ + - mountPath: /csi name: socket-dir - args: - - --csi-address=$(ADDRESS) - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: quay.io/k8scsi/livenessprobe:v2.1.0 + - --v=4 + - --csi-address=/csi/csi.sock + image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 name: liveness-probe resources: {} volumeMounts: - - mountPath: /var/lib/csi/sockets/pluginproxy/ + - mountPath: /csi name: socket-dir - args: - --leader-election + - --leader-election-lease-duration=30s + - --leader-election-renew-deadline=20s + - --leader-election-retry-period=10s + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) env: - - name: X_CSI_FULL_SYNC_INTERVAL_MINUTES + - name: FULL_SYNC_INTERVAL_MINUTES value: "30" - - name: LOGGER_LEVEL - value: PRODUCTION - name: VSPHERE_CSI_CONFIG value: /etc/cloud/csi-vsphere.conf - image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v2.1.0 + - name: LOGGER_LEVEL + value: PRODUCTION + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: GODEBUG + value: x509sha1=1 + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v3.1.0 name: vsphere-syncer + ports: + - containerPort: 2113 + name: prometheus + protocol: TCP resources: {} + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 volumeMounts: - mountPath: /etc/cloud name: vsphere-config-volume @@ -792,18 +1113,43 @@ data: - --v=4 - --timeout=300s - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s - --default-fstype=ext4 env: - name: ADDRESS value: /csi/csi.sock - image: quay.io/k8scsi/csi-provisioner:v2.0.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0 name: csi-provisioner resources: {} volumeMounts: - mountPath: /csi name: socket-dir + - args: + - --v=4 + - --kube-api-qps=100 + - --kube-api-burst=100 + - --timeout=300s + - --csi-address=$(ADDRESS) + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2 + name: csi-snapshotter + resources: {} + volumeMounts: + - mountPath: /csi + name: socket-dir dnsPolicy: Default + priorityClassName: system-cluster-critical serviceAccountName: vsphere-csi-controller tolerations: - effect: NoSchedule @@ -824,6 +1170,62 @@ metadata: namespace: '${NAMESPACE}' --- apiVersion: v1 +data: + data: | + apiVersion: v1 + data: + async-query-volume: "true" + block-volume-snapshot: "true" + cnsmgr-suspend-create-volume: "true" + csi-auth-check: "true" + csi-internal-generated-cluster-id: "true" + csi-migration: "true" + csi-windows-support: "true" + list-volumes: "true" + listview-tasks: "true" + max-pvscsi-targets-per-vm: "true" + multi-vcenter-csi-topology: "true" + online-volume-extend: "true" + pv-to-backingdiskobjectid-mapping: "false" + topology-preferential-datastores: "true" + trigger-csi-fullsync: "false" + kind: ConfigMap + metadata: + name: internal-feature-states.csi.vsphere.vmware.com + namespace: vmware-system-csi +kind: ConfigMap +metadata: + name: internal-feature-states + namespace: '${NAMESPACE}' +--- +apiVersion: v1 +data: + data: | + apiVersion: v1 + kind: Service + metadata: + labels: + app: vsphere-csi-controller + name: vsphere-csi-controller + namespace: vmware-system-csi + spec: + ports: + - name: ctlr + port: 2112 + protocol: TCP + targetPort: 2112 + - name: syncer + port: 2113 + protocol: TCP + targetPort: 2113 + selector: + app: vsphere-csi-controller +kind: ConfigMap +metadata: + name: vsphere-csi-controller-service + namespace: '${NAMESPACE}' +--- +apiVersion: v1 kind: Secret metadata: name: cloud-controller-manager diff --git a/templates/cluster-template-node-ipam.yaml b/templates/cluster-template-node-ipam.yaml index b0b4bdc97f..8128a0dccf 100644 --- a/templates/cluster-template-node-ipam.yaml +++ b/templates/cluster-template-node-ipam.yaml @@ -267,12 +267,24 @@ spec: matchLabels: cluster.x-k8s.io/cluster-name: '${CLUSTER_NAME}' resources: + - kind: ConfigMap + name: csi-namespace - kind: Secret name: vsphere-csi-controller - kind: ConfigMap - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role - kind: ConfigMap - name: vsphere-csi-controller-binding + name: vsphere-csi-controller-cluster-role-binding + - kind: Secret + name: vsphere-csi-node + - kind: ConfigMap + name: vsphere-csi-node-role + - kind: ConfigMap + name: vsphere-csi-node-role-binding + - kind: ConfigMap + name: vsphere-csi-node-cluster-role + - kind: ConfigMap + name: vsphere-csi-node-cluster-role-binding - kind: Secret name: csi-vsphere-config - kind: ConfigMap @@ -281,6 +293,10 @@ spec: name: vsphere-csi-node - kind: ConfigMap name: vsphere-csi-controller + - kind: ConfigMap + name: vsphere-csi-controller-service + - kind: ConfigMap + name: internal-feature-states - kind: Secret name: cloud-controller-manager - kind: Secret @@ -298,6 +314,18 @@ stringData: username: ${VSPHERE_USERNAME} --- apiVersion: v1 +data: + data: | + apiVersion: v1 + kind: Namespace + metadata: + name: vmware-system-csi +kind: ConfigMap +metadata: + name: csi-namespace + namespace: '${NAMESPACE}' +--- +apiVersion: v1 kind: Secret metadata: name: vsphere-csi-controller @@ -308,7 +336,7 @@ stringData: kind: ServiceAccount metadata: name: vsphere-csi-controller - namespace: kube-system + namespace: vmware-system-csi type: addons.cluster.x-k8s.io/resource-set --- apiVersion: v1 @@ -317,63 +345,75 @@ data: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role rules: - apiGroups: - - storage.k8s.io + - "" resources: - - csidrivers + - nodes + - pods verbs: - - create - - delete + - get + - list + - watch - apiGroups: - "" resources: - - nodes - - pods - - secrets - configmaps verbs: - get - list - watch + - create - apiGroups: - "" resources: - - persistentvolumes + - persistentvolumeclaims verbs: - get - list - watch - update - - create - - delete + - apiGroups: + - "" + resources: + - persistentvolumeclaims/status + verbs: - patch - apiGroups: - - storage.k8s.io + - "" resources: - - volumeattachments + - persistentvolumes verbs: - get - list - watch + - create - update + - delete - patch - apiGroups: - - storage.k8s.io + - "" resources: - - volumeattachments/status + - events verbs: + - get + - list + - watch + - create + - update - patch - apiGroups: - - "" + - coordination.k8s.io resources: - - persistentvolumeclaims + - leases verbs: - get - - list - watch + - list + - delete - update + - create - apiGroups: - storage.k8s.io resources: @@ -384,26 +424,69 @@ data: - list - watch - apiGroups: - - "" + - storage.k8s.io resources: - - events + - volumeattachments verbs: + - get - list - watch + - patch + - apiGroups: + - cns.vmware.com + resources: + - triggercsifullsyncs + verbs: - create + - get - update - - patch + - watch + - list - apiGroups: - - coordination.k8s.io + - cns.vmware.com resources: - - leases + - cnsvspherevolumemigrations verbs: + - create - get + - list - watch + - update + - delete + - apiGroups: + - cns.vmware.com + resources: + - cnsvolumeinfoes + verbs: + - create + - get - list + - watch - delete + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - create - update + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + - apiGroups: + - cns.vmware.com + resources: + - cnsvolumeoperationrequests + verbs: - create + - get + - list + - update + - delete - apiGroups: - snapshot.storage.k8s.io resources: @@ -411,16 +494,45 @@ data: verbs: - get - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - watch + - get + - list - apiGroups: - snapshot.storage.k8s.io resources: - volumesnapshotcontents verbs: + - create - get - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + - patch + - apiGroups: + - cns.vmware.com + resources: + - csinodetopologies + verbs: + - get + - update + - watch + - list kind: ConfigMap metadata: - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role namespace: '${NAMESPACE}' --- apiVersion: v1 @@ -429,18 +541,123 @@ data: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: vsphere-csi-controller-binding + name: vsphere-csi-controller-cluster-role-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role subjects: - kind: ServiceAccount name: vsphere-csi-controller - namespace: kube-system + namespace: vmware-system-csi +kind: ConfigMap +metadata: + name: vsphere-csi-controller-cluster-role-binding + namespace: '${NAMESPACE}' +--- +apiVersion: v1 +kind: Secret +metadata: + name: vsphere-csi-node + namespace: '${NAMESPACE}' +stringData: + data: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: vsphere-csi-node + namespace: vmware-system-csi +type: addons.cluster.x-k8s.io/resource-set +--- +apiVersion: v1 +data: + data: | + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: vsphere-csi-node-role + namespace: vmware-system-csi + rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch kind: ConfigMap metadata: - name: vsphere-csi-controller-binding + name: vsphere-csi-node-role + namespace: '${NAMESPACE}' +--- +apiVersion: v1 +data: + data: | + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: vsphere-csi-node-role-binding + namespace: vmware-system-csi + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: vsphere-csi-node-role + subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: vmware-system-csi +kind: ConfigMap +metadata: + name: vsphere-csi-node-role-binding + namespace: '${NAMESPACE}' +--- +apiVersion: v1 +data: + data: | + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: vsphere-csi-node-cluster-role + rules: + - apiGroups: + - cns.vmware.com + resources: + - csinodetopologies + verbs: + - create + - watch + - get + - patch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get +kind: ConfigMap +metadata: + name: vsphere-csi-node-cluster-role + namespace: '${NAMESPACE}' +--- +apiVersion: v1 +data: + data: | + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: vsphere-csi-node-cluster-role-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: vsphere-csi-node-cluster-role + subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: vmware-system-csi +kind: ConfigMap +metadata: + name: vsphere-csi-node-cluster-role-binding namespace: '${NAMESPACE}' --- apiVersion: v1 @@ -454,7 +671,7 @@ stringData: kind: Secret metadata: name: csi-vsphere-config - namespace: kube-system + namespace: vmware-system-csi stringData: csi-vsphere.conf: |+ [Global] @@ -493,7 +710,7 @@ data: kind: DaemonSet metadata: name: vsphere-csi-node - namespace: kube-system + namespace: vmware-system-csi spec: selector: matchLabels: @@ -514,41 +731,50 @@ data: value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock - image: quay.io/k8scsi/csi-node-driver-registrar:v2.0.1 - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - rm -rf /registration/csi.vsphere.vmware.com-reg.sock /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock + - --mode=kubelet-registration-probe + initialDelaySeconds: 3 name: node-driver-registrar resources: {} - securityContext: - privileged: true volumeMounts: - mountPath: /csi name: plugin-dir - mountPath: /registration name: registration-dir - - env: + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName - name: CSI_ENDPOINT value: unix:///csi/csi.sock + - name: MAX_VOLUMES_PER_NODE + value: "59" - name: X_CSI_MODE value: node - name: X_CSI_SPEC_REQ_VALIDATION value: "false" - - name: VSPHERE_CSI_CONFIG - value: /etc/cloud/csi-vsphere.conf + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" - name: LOGGER_LEVEL value: PRODUCTION - - name: X_CSI_LOG_LEVEL - value: INFO - - name: NODE_NAME + - name: GODEBUG + value: x509sha1=1 + - name: CSI_NAMESPACE valueFrom: fieldRef: - fieldPath: spec.nodeName - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v2.1.0 + fieldPath: metadata.namespace + - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES + value: "1" + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0 livenessProbe: failureThreshold: 3 httpGet: @@ -556,7 +782,7 @@ data: port: healthz initialDelaySeconds: 10 periodSeconds: 5 - timeoutSeconds: 3 + timeoutSeconds: 5 name: vsphere-csi-node ports: - containerPort: 9808 @@ -570,8 +796,6 @@ data: - SYS_ADMIN privileged: true volumeMounts: - - mountPath: /etc/cloud - name: vsphere-config-volume - mountPath: /csi name: plugin-dir - mountPath: /var/lib/kubelet @@ -579,30 +803,37 @@ data: name: pods-mount-dir - mountPath: /dev name: device-dir + - mountPath: /sys/block + name: blocks-dir + - mountPath: /sys/devices + name: sys-devices-dir - args: + - --v=4 - --csi-address=/csi/csi.sock - image: quay.io/k8scsi/livenessprobe:v2.1.0 + image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 name: liveness-probe resources: {} volumeMounts: - mountPath: /csi name: plugin-dir - dnsPolicy: Default + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + serviceAccountName: vsphere-csi-node tolerations: - effect: NoSchedule operator: Exists - effect: NoExecute operator: Exists volumes: - - name: vsphere-config-volume - secret: - secretName: csi-vsphere-config - hostPath: path: /var/lib/kubelet/plugins_registry type: Directory name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/ + path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com type: DirectoryOrCreate name: plugin-dir - hostPath: @@ -612,7 +843,17 @@ data: - hostPath: path: /dev name: device-dir + - hostPath: + path: /sys/block + type: Directory + name: blocks-dir + - hostPath: + path: /sys/devices + type: Directory + name: sys-devices-dir updateStrategy: + rollingUpdate: + maxUnavailable: 1 type: RollingUpdate kind: ConfigMap metadata: @@ -626,88 +867,168 @@ data: kind: Deployment metadata: name: vsphere-csi-controller - namespace: kube-system + namespace: vmware-system-csi spec: replicas: 1 selector: matchLabels: app: vsphere-csi-controller + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate template: metadata: labels: app: vsphere-csi-controller role: vsphere-csi spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - vsphere-csi-controller + topologyKey: kubernetes.io/hostname containers: - args: - --v=4 - --timeout=300s - --csi-address=$(ADDRESS) - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + - --kube-api-qps=100 + - --kube-api-burst=100 env: - name: ADDRESS value: /csi/csi.sock - image: quay.io/k8scsi/csi-attacher:v3.0.0 + image: registry.k8s.io/sig-storage/csi-attacher:v4.3.0 name: csi-attacher resources: {} volumeMounts: - mountPath: /csi name: socket-dir - - env: + - args: + - --v=4 + - --timeout=300s + - --handle-volume-inuse-error=false + - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-resizer:v1.8.0 + name: csi-resizer + resources: {} + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: - name: CSI_ENDPOINT - value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + value: unix:///csi/csi.sock - name: X_CSI_MODE value: controller + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" + - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT + value: 3m - name: VSPHERE_CSI_CONFIG value: /etc/cloud/csi-vsphere.conf - name: LOGGER_LEVEL value: PRODUCTION - - name: X_CSI_LOG_LEVEL - value: INFO - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v2.1.0 + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0 livenessProbe: failureThreshold: 3 httpGet: path: /healthz port: healthz - initialDelaySeconds: 10 - periodSeconds: 5 - timeoutSeconds: 3 + initialDelaySeconds: 30 + periodSeconds: 180 + timeoutSeconds: 10 name: vsphere-csi-controller ports: - containerPort: 9808 name: healthz protocol: TCP + - containerPort: 2112 + name: prometheus + protocol: TCP resources: {} + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 volumeMounts: - mountPath: /etc/cloud name: vsphere-config-volume readOnly: true - - mountPath: /var/lib/csi/sockets/pluginproxy/ + - mountPath: /csi name: socket-dir - args: - - --csi-address=$(ADDRESS) - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: quay.io/k8scsi/livenessprobe:v2.1.0 + - --v=4 + - --csi-address=/csi/csi.sock + image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 name: liveness-probe resources: {} volumeMounts: - - mountPath: /var/lib/csi/sockets/pluginproxy/ + - mountPath: /csi name: socket-dir - args: - --leader-election + - --leader-election-lease-duration=30s + - --leader-election-renew-deadline=20s + - --leader-election-retry-period=10s + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) env: - - name: X_CSI_FULL_SYNC_INTERVAL_MINUTES + - name: FULL_SYNC_INTERVAL_MINUTES value: "30" - - name: LOGGER_LEVEL - value: PRODUCTION - name: VSPHERE_CSI_CONFIG value: /etc/cloud/csi-vsphere.conf - image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v2.1.0 + - name: LOGGER_LEVEL + value: PRODUCTION + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: GODEBUG + value: x509sha1=1 + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v3.1.0 name: vsphere-syncer + ports: + - containerPort: 2113 + name: prometheus + protocol: TCP resources: {} + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 volumeMounts: - mountPath: /etc/cloud name: vsphere-config-volume @@ -716,18 +1037,43 @@ data: - --v=4 - --timeout=300s - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s - --default-fstype=ext4 env: - name: ADDRESS value: /csi/csi.sock - image: quay.io/k8scsi/csi-provisioner:v2.0.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0 name: csi-provisioner resources: {} volumeMounts: - mountPath: /csi name: socket-dir + - args: + - --v=4 + - --kube-api-qps=100 + - --kube-api-burst=100 + - --timeout=300s + - --csi-address=$(ADDRESS) + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2 + name: csi-snapshotter + resources: {} + volumeMounts: + - mountPath: /csi + name: socket-dir dnsPolicy: Default + priorityClassName: system-cluster-critical serviceAccountName: vsphere-csi-controller tolerations: - effect: NoSchedule @@ -748,6 +1094,62 @@ metadata: namespace: '${NAMESPACE}' --- apiVersion: v1 +data: + data: | + apiVersion: v1 + data: + async-query-volume: "true" + block-volume-snapshot: "true" + cnsmgr-suspend-create-volume: "true" + csi-auth-check: "true" + csi-internal-generated-cluster-id: "true" + csi-migration: "true" + csi-windows-support: "true" + list-volumes: "true" + listview-tasks: "true" + max-pvscsi-targets-per-vm: "true" + multi-vcenter-csi-topology: "true" + online-volume-extend: "true" + pv-to-backingdiskobjectid-mapping: "false" + topology-preferential-datastores: "true" + trigger-csi-fullsync: "false" + kind: ConfigMap + metadata: + name: internal-feature-states.csi.vsphere.vmware.com + namespace: vmware-system-csi +kind: ConfigMap +metadata: + name: internal-feature-states + namespace: '${NAMESPACE}' +--- +apiVersion: v1 +data: + data: | + apiVersion: v1 + kind: Service + metadata: + labels: + app: vsphere-csi-controller + name: vsphere-csi-controller + namespace: vmware-system-csi + spec: + ports: + - name: ctlr + port: 2112 + protocol: TCP + targetPort: 2112 + - name: syncer + port: 2113 + protocol: TCP + targetPort: 2113 + selector: + app: vsphere-csi-controller +kind: ConfigMap +metadata: + name: vsphere-csi-controller-service + namespace: '${NAMESPACE}' +--- +apiVersion: v1 kind: Secret metadata: name: cloud-controller-manager diff --git a/templates/cluster-template-topology.yaml b/templates/cluster-template-topology.yaml index f886449ee7..bcae7f2611 100644 --- a/templates/cluster-template-topology.yaml +++ b/templates/cluster-template-topology.yaml @@ -103,12 +103,24 @@ spec: matchLabels: cluster.x-k8s.io/cluster-name: '${CLUSTER_NAME}' resources: + - kind: ConfigMap + name: csi-namespace - kind: Secret name: vsphere-csi-controller - kind: ConfigMap - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role - kind: ConfigMap - name: vsphere-csi-controller-binding + name: vsphere-csi-controller-cluster-role-binding + - kind: Secret + name: vsphere-csi-node + - kind: ConfigMap + name: vsphere-csi-node-role + - kind: ConfigMap + name: vsphere-csi-node-role-binding + - kind: ConfigMap + name: vsphere-csi-node-cluster-role + - kind: ConfigMap + name: vsphere-csi-node-cluster-role-binding - kind: Secret name: csi-vsphere-config - kind: ConfigMap @@ -117,6 +129,10 @@ spec: name: vsphere-csi-node - kind: ConfigMap name: vsphere-csi-controller + - kind: ConfigMap + name: vsphere-csi-controller-service + - kind: ConfigMap + name: internal-feature-states - kind: Secret name: cloud-controller-manager - kind: Secret @@ -125,6 +141,18 @@ spec: name: cpi-manifests --- apiVersion: v1 +data: + data: | + apiVersion: v1 + kind: Namespace + metadata: + name: vmware-system-csi +kind: ConfigMap +metadata: + name: csi-namespace + namespace: '${NAMESPACE}' +--- +apiVersion: v1 kind: Secret metadata: name: vsphere-csi-controller @@ -135,7 +163,7 @@ stringData: kind: ServiceAccount metadata: name: vsphere-csi-controller - namespace: kube-system + namespace: vmware-system-csi type: addons.cluster.x-k8s.io/resource-set --- apiVersion: v1 @@ -144,63 +172,75 @@ data: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role rules: - apiGroups: - - storage.k8s.io + - "" resources: - - csidrivers + - nodes + - pods verbs: - - create - - delete + - get + - list + - watch - apiGroups: - "" resources: - - nodes - - pods - - secrets - configmaps verbs: - get - list - watch + - create - apiGroups: - "" resources: - - persistentvolumes + - persistentvolumeclaims verbs: - get - list - watch - update - - create - - delete + - apiGroups: + - "" + resources: + - persistentvolumeclaims/status + verbs: - patch - apiGroups: - - storage.k8s.io + - "" resources: - - volumeattachments + - persistentvolumes verbs: - get - list - watch + - create - update + - delete - patch - apiGroups: - - storage.k8s.io + - "" resources: - - volumeattachments/status + - events verbs: + - get + - list + - watch + - create + - update - patch - apiGroups: - - "" + - coordination.k8s.io resources: - - persistentvolumeclaims + - leases verbs: - get - - list - watch + - list + - delete - update + - create - apiGroups: - storage.k8s.io resources: @@ -211,26 +251,69 @@ data: - list - watch - apiGroups: - - "" + - storage.k8s.io resources: - - events + - volumeattachments verbs: + - get - list - watch + - patch + - apiGroups: + - cns.vmware.com + resources: + - triggercsifullsyncs + verbs: - create + - get - update - - patch + - watch + - list - apiGroups: - - coordination.k8s.io + - cns.vmware.com resources: - - leases + - cnsvspherevolumemigrations verbs: + - create - get + - list - watch + - update + - delete + - apiGroups: + - cns.vmware.com + resources: + - cnsvolumeinfoes + verbs: + - create + - get - list + - watch - delete + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - create - update + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + - apiGroups: + - cns.vmware.com + resources: + - cnsvolumeoperationrequests + verbs: - create + - get + - list + - update + - delete - apiGroups: - snapshot.storage.k8s.io resources: @@ -238,16 +321,45 @@ data: verbs: - get - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - watch + - get + - list - apiGroups: - snapshot.storage.k8s.io resources: - volumesnapshotcontents verbs: + - create - get - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + - patch + - apiGroups: + - cns.vmware.com + resources: + - csinodetopologies + verbs: + - get + - update + - watch + - list kind: ConfigMap metadata: - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role namespace: '${NAMESPACE}' --- apiVersion: v1 @@ -256,18 +368,123 @@ data: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: vsphere-csi-controller-binding + name: vsphere-csi-controller-cluster-role-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role subjects: - kind: ServiceAccount name: vsphere-csi-controller - namespace: kube-system + namespace: vmware-system-csi +kind: ConfigMap +metadata: + name: vsphere-csi-controller-cluster-role-binding + namespace: '${NAMESPACE}' +--- +apiVersion: v1 +kind: Secret +metadata: + name: vsphere-csi-node + namespace: '${NAMESPACE}' +stringData: + data: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: vsphere-csi-node + namespace: vmware-system-csi +type: addons.cluster.x-k8s.io/resource-set +--- +apiVersion: v1 +data: + data: | + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: vsphere-csi-node-role + namespace: vmware-system-csi + rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch kind: ConfigMap metadata: - name: vsphere-csi-controller-binding + name: vsphere-csi-node-role + namespace: '${NAMESPACE}' +--- +apiVersion: v1 +data: + data: | + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: vsphere-csi-node-role-binding + namespace: vmware-system-csi + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: vsphere-csi-node-role + subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: vmware-system-csi +kind: ConfigMap +metadata: + name: vsphere-csi-node-role-binding + namespace: '${NAMESPACE}' +--- +apiVersion: v1 +data: + data: | + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: vsphere-csi-node-cluster-role + rules: + - apiGroups: + - cns.vmware.com + resources: + - csinodetopologies + verbs: + - create + - watch + - get + - patch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get +kind: ConfigMap +metadata: + name: vsphere-csi-node-cluster-role + namespace: '${NAMESPACE}' +--- +apiVersion: v1 +data: + data: | + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: vsphere-csi-node-cluster-role-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: vsphere-csi-node-cluster-role + subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: vmware-system-csi +kind: ConfigMap +metadata: + name: vsphere-csi-node-cluster-role-binding namespace: '${NAMESPACE}' --- apiVersion: v1 @@ -281,7 +498,7 @@ stringData: kind: Secret metadata: name: csi-vsphere-config - namespace: kube-system + namespace: vmware-system-csi stringData: csi-vsphere.conf: |+ [Global] @@ -320,7 +537,7 @@ data: kind: DaemonSet metadata: name: vsphere-csi-node - namespace: kube-system + namespace: vmware-system-csi spec: selector: matchLabels: @@ -341,41 +558,50 @@ data: value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock - image: quay.io/k8scsi/csi-node-driver-registrar:v2.0.1 - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - rm -rf /registration/csi.vsphere.vmware.com-reg.sock /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock + - --mode=kubelet-registration-probe + initialDelaySeconds: 3 name: node-driver-registrar resources: {} - securityContext: - privileged: true volumeMounts: - mountPath: /csi name: plugin-dir - mountPath: /registration name: registration-dir - - env: + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName - name: CSI_ENDPOINT value: unix:///csi/csi.sock + - name: MAX_VOLUMES_PER_NODE + value: "59" - name: X_CSI_MODE value: node - name: X_CSI_SPEC_REQ_VALIDATION value: "false" - - name: VSPHERE_CSI_CONFIG - value: /etc/cloud/csi-vsphere.conf + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" - name: LOGGER_LEVEL value: PRODUCTION - - name: X_CSI_LOG_LEVEL - value: INFO - - name: NODE_NAME + - name: GODEBUG + value: x509sha1=1 + - name: CSI_NAMESPACE valueFrom: fieldRef: - fieldPath: spec.nodeName - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v2.1.0 + fieldPath: metadata.namespace + - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES + value: "1" + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0 livenessProbe: failureThreshold: 3 httpGet: @@ -383,7 +609,7 @@ data: port: healthz initialDelaySeconds: 10 periodSeconds: 5 - timeoutSeconds: 3 + timeoutSeconds: 5 name: vsphere-csi-node ports: - containerPort: 9808 @@ -397,8 +623,6 @@ data: - SYS_ADMIN privileged: true volumeMounts: - - mountPath: /etc/cloud - name: vsphere-config-volume - mountPath: /csi name: plugin-dir - mountPath: /var/lib/kubelet @@ -406,30 +630,37 @@ data: name: pods-mount-dir - mountPath: /dev name: device-dir + - mountPath: /sys/block + name: blocks-dir + - mountPath: /sys/devices + name: sys-devices-dir - args: + - --v=4 - --csi-address=/csi/csi.sock - image: quay.io/k8scsi/livenessprobe:v2.1.0 + image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 name: liveness-probe resources: {} volumeMounts: - mountPath: /csi name: plugin-dir - dnsPolicy: Default + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + serviceAccountName: vsphere-csi-node tolerations: - effect: NoSchedule operator: Exists - effect: NoExecute operator: Exists volumes: - - name: vsphere-config-volume - secret: - secretName: csi-vsphere-config - hostPath: path: /var/lib/kubelet/plugins_registry type: Directory name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/ + path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com type: DirectoryOrCreate name: plugin-dir - hostPath: @@ -439,7 +670,17 @@ data: - hostPath: path: /dev name: device-dir + - hostPath: + path: /sys/block + type: Directory + name: blocks-dir + - hostPath: + path: /sys/devices + type: Directory + name: sys-devices-dir updateStrategy: + rollingUpdate: + maxUnavailable: 1 type: RollingUpdate kind: ConfigMap metadata: @@ -453,88 +694,168 @@ data: kind: Deployment metadata: name: vsphere-csi-controller - namespace: kube-system + namespace: vmware-system-csi spec: replicas: 1 selector: matchLabels: app: vsphere-csi-controller + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate template: metadata: labels: app: vsphere-csi-controller role: vsphere-csi spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - vsphere-csi-controller + topologyKey: kubernetes.io/hostname containers: - args: - --v=4 - --timeout=300s - --csi-address=$(ADDRESS) - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + - --kube-api-qps=100 + - --kube-api-burst=100 env: - name: ADDRESS value: /csi/csi.sock - image: quay.io/k8scsi/csi-attacher:v3.0.0 + image: registry.k8s.io/sig-storage/csi-attacher:v4.3.0 name: csi-attacher resources: {} volumeMounts: - mountPath: /csi name: socket-dir - - env: + - args: + - --v=4 + - --timeout=300s + - --handle-volume-inuse-error=false + - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-resizer:v1.8.0 + name: csi-resizer + resources: {} + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: - name: CSI_ENDPOINT - value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + value: unix:///csi/csi.sock - name: X_CSI_MODE value: controller + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" + - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT + value: 3m - name: VSPHERE_CSI_CONFIG value: /etc/cloud/csi-vsphere.conf - name: LOGGER_LEVEL value: PRODUCTION - - name: X_CSI_LOG_LEVEL - value: INFO - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v2.1.0 + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0 livenessProbe: failureThreshold: 3 httpGet: path: /healthz port: healthz - initialDelaySeconds: 10 - periodSeconds: 5 - timeoutSeconds: 3 + initialDelaySeconds: 30 + periodSeconds: 180 + timeoutSeconds: 10 name: vsphere-csi-controller ports: - containerPort: 9808 name: healthz protocol: TCP + - containerPort: 2112 + name: prometheus + protocol: TCP resources: {} + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 volumeMounts: - mountPath: /etc/cloud name: vsphere-config-volume readOnly: true - - mountPath: /var/lib/csi/sockets/pluginproxy/ + - mountPath: /csi name: socket-dir - args: - - --csi-address=$(ADDRESS) - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: quay.io/k8scsi/livenessprobe:v2.1.0 + - --v=4 + - --csi-address=/csi/csi.sock + image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 name: liveness-probe resources: {} volumeMounts: - - mountPath: /var/lib/csi/sockets/pluginproxy/ + - mountPath: /csi name: socket-dir - args: - --leader-election + - --leader-election-lease-duration=30s + - --leader-election-renew-deadline=20s + - --leader-election-retry-period=10s + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) env: - - name: X_CSI_FULL_SYNC_INTERVAL_MINUTES + - name: FULL_SYNC_INTERVAL_MINUTES value: "30" - - name: LOGGER_LEVEL - value: PRODUCTION - name: VSPHERE_CSI_CONFIG value: /etc/cloud/csi-vsphere.conf - image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v2.1.0 + - name: LOGGER_LEVEL + value: PRODUCTION + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: GODEBUG + value: x509sha1=1 + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v3.1.0 name: vsphere-syncer + ports: + - containerPort: 2113 + name: prometheus + protocol: TCP resources: {} + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 volumeMounts: - mountPath: /etc/cloud name: vsphere-config-volume @@ -543,18 +864,43 @@ data: - --v=4 - --timeout=300s - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s - --default-fstype=ext4 env: - name: ADDRESS value: /csi/csi.sock - image: quay.io/k8scsi/csi-provisioner:v2.0.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0 name: csi-provisioner resources: {} volumeMounts: - mountPath: /csi name: socket-dir + - args: + - --v=4 + - --kube-api-qps=100 + - --kube-api-burst=100 + - --timeout=300s + - --csi-address=$(ADDRESS) + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2 + name: csi-snapshotter + resources: {} + volumeMounts: + - mountPath: /csi + name: socket-dir dnsPolicy: Default + priorityClassName: system-cluster-critical serviceAccountName: vsphere-csi-controller tolerations: - effect: NoSchedule @@ -575,6 +921,62 @@ metadata: namespace: '${NAMESPACE}' --- apiVersion: v1 +data: + data: | + apiVersion: v1 + data: + async-query-volume: "true" + block-volume-snapshot: "true" + cnsmgr-suspend-create-volume: "true" + csi-auth-check: "true" + csi-internal-generated-cluster-id: "true" + csi-migration: "true" + csi-windows-support: "true" + list-volumes: "true" + listview-tasks: "true" + max-pvscsi-targets-per-vm: "true" + multi-vcenter-csi-topology: "true" + online-volume-extend: "true" + pv-to-backingdiskobjectid-mapping: "false" + topology-preferential-datastores: "true" + trigger-csi-fullsync: "false" + kind: ConfigMap + metadata: + name: internal-feature-states.csi.vsphere.vmware.com + namespace: vmware-system-csi +kind: ConfigMap +metadata: + name: internal-feature-states + namespace: '${NAMESPACE}' +--- +apiVersion: v1 +data: + data: | + apiVersion: v1 + kind: Service + metadata: + labels: + app: vsphere-csi-controller + name: vsphere-csi-controller + namespace: vmware-system-csi + spec: + ports: + - name: ctlr + port: 2112 + protocol: TCP + targetPort: 2112 + - name: syncer + port: 2113 + protocol: TCP + targetPort: 2113 + selector: + app: vsphere-csi-controller +kind: ConfigMap +metadata: + name: vsphere-csi-controller-service + namespace: '${NAMESPACE}' +--- +apiVersion: v1 kind: Secret metadata: name: cloud-controller-manager diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index 2916bb99a4..385ec37924 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -257,12 +257,24 @@ spec: matchLabels: cluster.x-k8s.io/cluster-name: '${CLUSTER_NAME}' resources: + - kind: ConfigMap + name: csi-namespace - kind: Secret name: vsphere-csi-controller - kind: ConfigMap - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role - kind: ConfigMap - name: vsphere-csi-controller-binding + name: vsphere-csi-controller-cluster-role-binding + - kind: Secret + name: vsphere-csi-node + - kind: ConfigMap + name: vsphere-csi-node-role + - kind: ConfigMap + name: vsphere-csi-node-role-binding + - kind: ConfigMap + name: vsphere-csi-node-cluster-role + - kind: ConfigMap + name: vsphere-csi-node-cluster-role-binding - kind: Secret name: csi-vsphere-config - kind: ConfigMap @@ -271,6 +283,10 @@ spec: name: vsphere-csi-node - kind: ConfigMap name: vsphere-csi-controller + - kind: ConfigMap + name: vsphere-csi-controller-service + - kind: ConfigMap + name: internal-feature-states - kind: Secret name: cloud-controller-manager - kind: Secret @@ -288,6 +304,18 @@ stringData: username: ${VSPHERE_USERNAME} --- apiVersion: v1 +data: + data: | + apiVersion: v1 + kind: Namespace + metadata: + name: vmware-system-csi +kind: ConfigMap +metadata: + name: csi-namespace + namespace: '${NAMESPACE}' +--- +apiVersion: v1 kind: Secret metadata: name: vsphere-csi-controller @@ -298,7 +326,7 @@ stringData: kind: ServiceAccount metadata: name: vsphere-csi-controller - namespace: kube-system + namespace: vmware-system-csi type: addons.cluster.x-k8s.io/resource-set --- apiVersion: v1 @@ -307,63 +335,75 @@ data: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role rules: - apiGroups: - - storage.k8s.io + - "" resources: - - csidrivers + - nodes + - pods verbs: - - create - - delete + - get + - list + - watch - apiGroups: - "" resources: - - nodes - - pods - - secrets - configmaps verbs: - get - list - watch + - create - apiGroups: - "" resources: - - persistentvolumes + - persistentvolumeclaims verbs: - get - list - watch - update - - create - - delete + - apiGroups: + - "" + resources: + - persistentvolumeclaims/status + verbs: - patch - apiGroups: - - storage.k8s.io + - "" resources: - - volumeattachments + - persistentvolumes verbs: - get - list - watch + - create - update + - delete - patch - apiGroups: - - storage.k8s.io + - "" resources: - - volumeattachments/status + - events verbs: + - get + - list + - watch + - create + - update - patch - apiGroups: - - "" + - coordination.k8s.io resources: - - persistentvolumeclaims + - leases verbs: - get - - list - watch + - list + - delete - update + - create - apiGroups: - storage.k8s.io resources: @@ -374,26 +414,69 @@ data: - list - watch - apiGroups: - - "" + - storage.k8s.io resources: - - events + - volumeattachments verbs: + - get - list - watch + - patch + - apiGroups: + - cns.vmware.com + resources: + - triggercsifullsyncs + verbs: - create + - get - update - - patch + - watch + - list - apiGroups: - - coordination.k8s.io + - cns.vmware.com resources: - - leases + - cnsvspherevolumemigrations verbs: + - create - get + - list - watch + - update + - delete + - apiGroups: + - cns.vmware.com + resources: + - cnsvolumeinfoes + verbs: + - create + - get - list + - watch - delete + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - create - update + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + - apiGroups: + - cns.vmware.com + resources: + - cnsvolumeoperationrequests + verbs: - create + - get + - list + - update + - delete - apiGroups: - snapshot.storage.k8s.io resources: @@ -401,16 +484,45 @@ data: verbs: - get - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - watch + - get + - list - apiGroups: - snapshot.storage.k8s.io resources: - volumesnapshotcontents verbs: + - create - get - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + - patch + - apiGroups: + - cns.vmware.com + resources: + - csinodetopologies + verbs: + - get + - update + - watch + - list kind: ConfigMap metadata: - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role namespace: '${NAMESPACE}' --- apiVersion: v1 @@ -419,18 +531,123 @@ data: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: vsphere-csi-controller-binding + name: vsphere-csi-controller-cluster-role-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: vsphere-csi-controller-role + name: vsphere-csi-controller-cluster-role subjects: - kind: ServiceAccount name: vsphere-csi-controller - namespace: kube-system + namespace: vmware-system-csi +kind: ConfigMap +metadata: + name: vsphere-csi-controller-cluster-role-binding + namespace: '${NAMESPACE}' +--- +apiVersion: v1 +kind: Secret +metadata: + name: vsphere-csi-node + namespace: '${NAMESPACE}' +stringData: + data: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: vsphere-csi-node + namespace: vmware-system-csi +type: addons.cluster.x-k8s.io/resource-set +--- +apiVersion: v1 +data: + data: | + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: vsphere-csi-node-role + namespace: vmware-system-csi + rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch kind: ConfigMap metadata: - name: vsphere-csi-controller-binding + name: vsphere-csi-node-role + namespace: '${NAMESPACE}' +--- +apiVersion: v1 +data: + data: | + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: vsphere-csi-node-role-binding + namespace: vmware-system-csi + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: vsphere-csi-node-role + subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: vmware-system-csi +kind: ConfigMap +metadata: + name: vsphere-csi-node-role-binding + namespace: '${NAMESPACE}' +--- +apiVersion: v1 +data: + data: | + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: vsphere-csi-node-cluster-role + rules: + - apiGroups: + - cns.vmware.com + resources: + - csinodetopologies + verbs: + - create + - watch + - get + - patch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get +kind: ConfigMap +metadata: + name: vsphere-csi-node-cluster-role + namespace: '${NAMESPACE}' +--- +apiVersion: v1 +data: + data: | + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: vsphere-csi-node-cluster-role-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: vsphere-csi-node-cluster-role + subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: vmware-system-csi +kind: ConfigMap +metadata: + name: vsphere-csi-node-cluster-role-binding namespace: '${NAMESPACE}' --- apiVersion: v1 @@ -444,7 +661,7 @@ stringData: kind: Secret metadata: name: csi-vsphere-config - namespace: kube-system + namespace: vmware-system-csi stringData: csi-vsphere.conf: |+ [Global] @@ -483,7 +700,7 @@ data: kind: DaemonSet metadata: name: vsphere-csi-node - namespace: kube-system + namespace: vmware-system-csi spec: selector: matchLabels: @@ -504,41 +721,50 @@ data: value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock - image: quay.io/k8scsi/csi-node-driver-registrar:v2.0.1 - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - rm -rf /registration/csi.vsphere.vmware.com-reg.sock /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock + - --mode=kubelet-registration-probe + initialDelaySeconds: 3 name: node-driver-registrar resources: {} - securityContext: - privileged: true volumeMounts: - mountPath: /csi name: plugin-dir - mountPath: /registration name: registration-dir - - env: + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName - name: CSI_ENDPOINT value: unix:///csi/csi.sock + - name: MAX_VOLUMES_PER_NODE + value: "59" - name: X_CSI_MODE value: node - name: X_CSI_SPEC_REQ_VALIDATION value: "false" - - name: VSPHERE_CSI_CONFIG - value: /etc/cloud/csi-vsphere.conf + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" - name: LOGGER_LEVEL value: PRODUCTION - - name: X_CSI_LOG_LEVEL - value: INFO - - name: NODE_NAME + - name: GODEBUG + value: x509sha1=1 + - name: CSI_NAMESPACE valueFrom: fieldRef: - fieldPath: spec.nodeName - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v2.1.0 + fieldPath: metadata.namespace + - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES + value: "1" + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0 livenessProbe: failureThreshold: 3 httpGet: @@ -546,7 +772,7 @@ data: port: healthz initialDelaySeconds: 10 periodSeconds: 5 - timeoutSeconds: 3 + timeoutSeconds: 5 name: vsphere-csi-node ports: - containerPort: 9808 @@ -560,8 +786,6 @@ data: - SYS_ADMIN privileged: true volumeMounts: - - mountPath: /etc/cloud - name: vsphere-config-volume - mountPath: /csi name: plugin-dir - mountPath: /var/lib/kubelet @@ -569,30 +793,37 @@ data: name: pods-mount-dir - mountPath: /dev name: device-dir + - mountPath: /sys/block + name: blocks-dir + - mountPath: /sys/devices + name: sys-devices-dir - args: + - --v=4 - --csi-address=/csi/csi.sock - image: quay.io/k8scsi/livenessprobe:v2.1.0 + image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 name: liveness-probe resources: {} volumeMounts: - mountPath: /csi name: plugin-dir - dnsPolicy: Default + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + serviceAccountName: vsphere-csi-node tolerations: - effect: NoSchedule operator: Exists - effect: NoExecute operator: Exists volumes: - - name: vsphere-config-volume - secret: - secretName: csi-vsphere-config - hostPath: path: /var/lib/kubelet/plugins_registry type: Directory name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/ + path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com type: DirectoryOrCreate name: plugin-dir - hostPath: @@ -602,7 +833,17 @@ data: - hostPath: path: /dev name: device-dir + - hostPath: + path: /sys/block + type: Directory + name: blocks-dir + - hostPath: + path: /sys/devices + type: Directory + name: sys-devices-dir updateStrategy: + rollingUpdate: + maxUnavailable: 1 type: RollingUpdate kind: ConfigMap metadata: @@ -616,88 +857,168 @@ data: kind: Deployment metadata: name: vsphere-csi-controller - namespace: kube-system + namespace: vmware-system-csi spec: replicas: 1 selector: matchLabels: app: vsphere-csi-controller + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate template: metadata: labels: app: vsphere-csi-controller role: vsphere-csi spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - vsphere-csi-controller + topologyKey: kubernetes.io/hostname containers: - args: - --v=4 - --timeout=300s - --csi-address=$(ADDRESS) - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + - --kube-api-qps=100 + - --kube-api-burst=100 env: - name: ADDRESS value: /csi/csi.sock - image: quay.io/k8scsi/csi-attacher:v3.0.0 + image: registry.k8s.io/sig-storage/csi-attacher:v4.3.0 name: csi-attacher resources: {} volumeMounts: - mountPath: /csi name: socket-dir - - env: + - args: + - --v=4 + - --timeout=300s + - --handle-volume-inuse-error=false + - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-resizer:v1.8.0 + name: csi-resizer + resources: {} + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: - name: CSI_ENDPOINT - value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + value: unix:///csi/csi.sock - name: X_CSI_MODE value: controller + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" + - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT + value: 3m - name: VSPHERE_CSI_CONFIG value: /etc/cloud/csi-vsphere.conf - name: LOGGER_LEVEL value: PRODUCTION - - name: X_CSI_LOG_LEVEL - value: INFO - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v2.1.0 + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0 livenessProbe: failureThreshold: 3 httpGet: path: /healthz port: healthz - initialDelaySeconds: 10 - periodSeconds: 5 - timeoutSeconds: 3 + initialDelaySeconds: 30 + periodSeconds: 180 + timeoutSeconds: 10 name: vsphere-csi-controller ports: - containerPort: 9808 name: healthz protocol: TCP + - containerPort: 2112 + name: prometheus + protocol: TCP resources: {} + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 volumeMounts: - mountPath: /etc/cloud name: vsphere-config-volume readOnly: true - - mountPath: /var/lib/csi/sockets/pluginproxy/ + - mountPath: /csi name: socket-dir - args: - - --csi-address=$(ADDRESS) - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: quay.io/k8scsi/livenessprobe:v2.1.0 + - --v=4 + - --csi-address=/csi/csi.sock + image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 name: liveness-probe resources: {} volumeMounts: - - mountPath: /var/lib/csi/sockets/pluginproxy/ + - mountPath: /csi name: socket-dir - args: - --leader-election + - --leader-election-lease-duration=30s + - --leader-election-renew-deadline=20s + - --leader-election-retry-period=10s + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) env: - - name: X_CSI_FULL_SYNC_INTERVAL_MINUTES + - name: FULL_SYNC_INTERVAL_MINUTES value: "30" - - name: LOGGER_LEVEL - value: PRODUCTION - name: VSPHERE_CSI_CONFIG value: /etc/cloud/csi-vsphere.conf - image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v2.1.0 + - name: LOGGER_LEVEL + value: PRODUCTION + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: GODEBUG + value: x509sha1=1 + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v3.1.0 name: vsphere-syncer + ports: + - containerPort: 2113 + name: prometheus + protocol: TCP resources: {} + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 volumeMounts: - mountPath: /etc/cloud name: vsphere-config-volume @@ -706,18 +1027,43 @@ data: - --v=4 - --timeout=300s - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s - --default-fstype=ext4 env: - name: ADDRESS value: /csi/csi.sock - image: quay.io/k8scsi/csi-provisioner:v2.0.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0 name: csi-provisioner resources: {} volumeMounts: - mountPath: /csi name: socket-dir + - args: + - --v=4 + - --kube-api-qps=100 + - --kube-api-burst=100 + - --timeout=300s + - --csi-address=$(ADDRESS) + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2 + name: csi-snapshotter + resources: {} + volumeMounts: + - mountPath: /csi + name: socket-dir dnsPolicy: Default + priorityClassName: system-cluster-critical serviceAccountName: vsphere-csi-controller tolerations: - effect: NoSchedule @@ -738,6 +1084,62 @@ metadata: namespace: '${NAMESPACE}' --- apiVersion: v1 +data: + data: | + apiVersion: v1 + data: + async-query-volume: "true" + block-volume-snapshot: "true" + cnsmgr-suspend-create-volume: "true" + csi-auth-check: "true" + csi-internal-generated-cluster-id: "true" + csi-migration: "true" + csi-windows-support: "true" + list-volumes: "true" + listview-tasks: "true" + max-pvscsi-targets-per-vm: "true" + multi-vcenter-csi-topology: "true" + online-volume-extend: "true" + pv-to-backingdiskobjectid-mapping: "false" + topology-preferential-datastores: "true" + trigger-csi-fullsync: "false" + kind: ConfigMap + metadata: + name: internal-feature-states.csi.vsphere.vmware.com + namespace: vmware-system-csi +kind: ConfigMap +metadata: + name: internal-feature-states + namespace: '${NAMESPACE}' +--- +apiVersion: v1 +data: + data: | + apiVersion: v1 + kind: Service + metadata: + labels: + app: vsphere-csi-controller + name: vsphere-csi-controller + namespace: vmware-system-csi + spec: + ports: + - name: ctlr + port: 2112 + protocol: TCP + targetPort: 2112 + - name: syncer + port: 2113 + protocol: TCP + targetPort: 2113 + selector: + app: vsphere-csi-controller +kind: ConfigMap +metadata: + name: vsphere-csi-controller-service + namespace: '${NAMESPACE}' +--- +apiVersion: v1 kind: Secret metadata: name: cloud-controller-manager