From 2d2fcceb84c59842248295fd78d859f034ac6ee9 Mon Sep 17 00:00:00 2001 From: Alexandr Demicev Date: Tue, 7 Jan 2025 15:46:33 +0100 Subject: [PATCH] Update CPI/CSI part of the vsphere template Signed-off-by: Alexandr Demicev --- examples/vmware/cluster-template.yaml | 1378 ++++++++++++++++--------- 1 file changed, 897 insertions(+), 481 deletions(-) diff --git a/examples/vmware/cluster-template.yaml b/examples/vmware/cluster-template.yaml index 47f6dc00..3b8bd521 100644 --- a/examples/vmware/cluster-template.yaml +++ b/examples/vmware/cluster-template.yaml @@ -292,21 +292,9 @@ spec: cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} resources: - kind: Secret - name: vsphere-csi-controller + name: vsphere-config-secret - kind: ConfigMap - name: vsphere-csi-controller-role - - kind: ConfigMap - name: vsphere-csi-controller-binding - - kind: Secret - name: csi-vsphere-config - - kind: ConfigMap - name: csi.vsphere.vmware.com - - kind: ConfigMap - name: vsphere-csi-node - - kind: ConfigMap - name: vsphere-csi-controller - - kind: Secret - name: cloud-controller-manager + name: csi-manifests - kind: Secret name: cloud-provider-vsphere-credentials - kind: ConfigMap @@ -318,86 +306,130 @@ metadata: name: ${CLUSTER_NAME} namespace: ${NAMESPACE} stringData: - password: ${VSPHERE_PASSWORD} - username: ${VSPHERE_USERNAME} + password: "${VSPHERE_PASSWORD}" + username: "${VSPHERE_USERNAME}" --- apiVersion: v1 kind: Secret metadata: - name: vsphere-csi-controller + name: vsphere-config-secret namespace: ${NAMESPACE} stringData: - data: | + data: |- apiVersion: v1 - kind: ServiceAccount + kind: Secret metadata: - name: vsphere-csi-controller - namespace: kube-system + name: vsphere-config-secret + namespace: vmware-system-csi + stringData: + csi-vsphere.conf: |+ + [Global] + thumbprint = "${VSPHERE_TLS_THUMBPRINT}" + + [VirtualCenter "${VSPHERE_SERVER}"] + user = "${VSPHERE_USERNAME}" + password = "${VSPHERE_PASSWORD}" + datacenters = "${VSPHERE_DATACENTER}" + + [Network] + public-network = "${VSPHERE_NETWORK}" + + type: Opaque type: addons.cluster.x-k8s.io/resource-set --- apiVersion: v1 data: - data: | + data: |- + apiVersion: v1 + kind: Namespace + metadata: + name: vmware-system-csi + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + name: csi.vsphere.vmware.com + spec: + attachRequired: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: vsphere-csi-controller + namespace: vmware-system-csi + --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: vsphere-csi-controller-role rules: - apiGroups: - - storage.k8s.io + - "" resources: - - csidrivers + - nodes + - pods verbs: - - create - - delete + - get + - list + - watch - apiGroups: - "" resources: - - nodes - - pods - - secrets - configmaps verbs: - get - list - watch + - create - apiGroups: - "" resources: - - persistentvolumes + - persistentvolumeclaims verbs: - get - list - watch - update - - create - - delete + - apiGroups: + - "" + resources: + - persistentvolumeclaims/status + verbs: - patch - apiGroups: - - storage.k8s.io + - "" resources: - - volumeattachments + - persistentvolumes verbs: - get - list - watch + - create - update + - delete - patch - apiGroups: - - storage.k8s.io + - "" resources: - - volumeattachments/status + - events verbs: + - get + - list + - watch + - create + - update - patch - apiGroups: - - "" + - coordination.k8s.io resources: - - persistentvolumeclaims + - leases verbs: - get - - list - watch + - list + - delete - update + - create - apiGroups: - storage.k8s.io resources: @@ -408,26 +440,69 @@ data: - list - watch - apiGroups: - - "" + - storage.k8s.io resources: - - events + - volumeattachments verbs: + - get - list - watch + - patch + - apiGroups: + - cns.vmware.com + resources: + - triggercsifullsyncs + verbs: - create + - get - update - - patch + - watch + - list - apiGroups: - - coordination.k8s.io + - cns.vmware.com resources: - - leases + - cnsvspherevolumemigrations verbs: + - create - get + - list - watch + - update + - delete + - apiGroups: + - cns.vmware.com + resources: + - cnsvolumeinfoes + verbs: + - create + - get - list + - watch - delete + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - create - update + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + - apiGroups: + - cns.vmware.com + resources: + - cnsvolumeoperationrequests + verbs: - create + - get + - list + - update + - delete - apiGroups: - snapshot.storage.k8s.io resources: @@ -435,21 +510,43 @@ data: verbs: - get - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - watch + - get + - list - apiGroups: - snapshot.storage.k8s.io resources: - volumesnapshotcontents verbs: + - create - get - list -kind: ConfigMap -metadata: - name: vsphere-csi-controller-role - namespace: ${NAMESPACE} ---- -apiVersion: v1 -data: - data: | + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + - patch + - apiGroups: + - cns.vmware.com + resources: + - csinodetopologies + verbs: + - get + - update + - watch + - list + --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -461,278 +558,279 @@ data: subjects: - kind: ServiceAccount name: vsphere-csi-controller - namespace: kube-system -kind: ConfigMap -metadata: - name: vsphere-csi-controller-binding - namespace: ${NAMESPACE} ---- -apiVersion: v1 -kind: Secret -metadata: - name: csi-vsphere-config - namespace: ${NAMESPACE} -stringData: - data: | + namespace: vmware-system-csi + --- apiVersion: v1 - kind: Secret + kind: ServiceAccount metadata: - name: csi-vsphere-config - namespace: kube-system - stringData: - csi-vsphere.conf: |+ - [Global] - cluster-id = "default/${CLUSTER_NAME}" - - [VirtualCenter "${VSPHERE_SERVER}"] - user = "${VSPHERE_USERNAME}" - password = "${VSPHERE_PASSWORD}" - datacenters = "${VSPHERE_DATACENTER}" - insecure-flag = 1 - - - [Network] - public-network = "${VSPHERE_NETWORK}" - - type: Opaque -type: addons.cluster.x-k8s.io/resource-set ---- -apiVersion: v1 -data: - data: | - apiVersion: storage.k8s.io/v1 - kind: CSIDriver + name: vsphere-csi-node + namespace: vmware-system-csi + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole metadata: - name: csi.vsphere.vmware.com - spec: - attachRequired: true -kind: ConfigMap -metadata: - name: csi.vsphere.vmware.com - namespace: ${NAMESPACE} ---- -apiVersion: v1 -data: - data: | - apiVersion: apps/v1 - kind: DaemonSet + name: vsphere-csi-node-cluster-role + rules: + - apiGroups: + - cns.vmware.com + resources: + - csinodetopologies + verbs: + - create + - watch + - get + - patch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding metadata: + name: vsphere-csi-node-cluster-role-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: vsphere-csi-node-cluster-role + subjects: + - kind: ServiceAccount name: vsphere-csi-node - namespace: kube-system - spec: - selector: - matchLabels: - app: vsphere-csi-node - template: - metadata: - labels: - app: vsphere-csi-node - role: vsphere-csi - spec: - containers: - - args: - - --v=5 - - --csi-address=$(ADDRESS) - - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - env: - - name: ADDRESS - value: /csi/csi.sock - - name: DRIVER_REG_SOCK_PATH - value: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock - image: quay.io/k8scsi/csi-node-driver-registrar:v2.0.1 - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - rm -rf /registration/csi.vsphere.vmware.com-reg.sock /csi/csi.sock - name: node-driver-registrar - resources: {} - securityContext: - privileged: true - volumeMounts: - - mountPath: /csi - name: plugin-dir - - mountPath: /registration - name: registration-dir - - env: - - name: CSI_ENDPOINT - value: unix:///csi/csi.sock - - name: X_CSI_MODE - value: node - - name: X_CSI_SPEC_REQ_VALIDATION - value: "false" - - name: VSPHERE_CSI_CONFIG - value: /etc/cloud/csi-vsphere.conf - - name: LOGGER_LEVEL - value: PRODUCTION - - name: X_CSI_LOG_LEVEL - value: INFO - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v2.1.0 - livenessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - periodSeconds: 5 - timeoutSeconds: 3 - name: vsphere-csi-node - ports: - - containerPort: 9808 - name: healthz - protocol: TCP - resources: {} - securityContext: - allowPrivilegeEscalation: true - capabilities: - add: - - SYS_ADMIN - privileged: true - volumeMounts: - - mountPath: /etc/cloud - name: vsphere-config-volume - - mountPath: /csi - name: plugin-dir - - mountPath: /var/lib/kubelet - mountPropagation: Bidirectional - name: pods-mount-dir - - mountPath: /dev - name: device-dir - - args: - - --csi-address=/csi/csi.sock - image: quay.io/k8scsi/livenessprobe:v2.1.0 - name: liveness-probe - resources: {} - volumeMounts: - - mountPath: /csi - name: plugin-dir - dnsPolicy: Default - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - volumes: - - name: vsphere-config-volume - secret: - secretName: csi-vsphere-config - - hostPath: - path: /var/lib/kubelet/plugins_registry - type: Directory - name: registration-dir - - hostPath: - path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/ - type: DirectoryOrCreate - name: plugin-dir - - hostPath: - path: /var/lib/kubelet - type: Directory - name: pods-mount-dir - - hostPath: - path: /dev - name: device-dir - updateStrategy: - type: RollingUpdate -kind: ConfigMap -metadata: - name: vsphere-csi-node - namespace: ${NAMESPACE} ---- -apiVersion: v1 -data: - data: | + namespace: vmware-system-csi + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: vsphere-csi-node-role + namespace: vmware-system-csi + rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: vsphere-csi-node-binding + namespace: vmware-system-csi + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: vsphere-csi-node-role + subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: vmware-system-csi + --- + apiVersion: v1 + data: + pv-to-backingdiskobjectid-mapping: "false" + trigger-csi-fullsync: "false" + kind: ConfigMap + metadata: + name: internal-feature-states.csi.vsphere.vmware.com + namespace: vmware-system-csi + --- + apiVersion: v1 + kind: Service + metadata: + labels: + app: vsphere-csi-controller + name: vsphere-csi-controller + namespace: vmware-system-csi + spec: + ports: + - name: ctlr + port: 2112 + protocol: TCP + targetPort: 2112 + - name: syncer + port: 2113 + protocol: TCP + targetPort: 2113 + selector: + app: vsphere-csi-controller + --- apiVersion: apps/v1 kind: Deployment metadata: name: vsphere-csi-controller - namespace: kube-system + namespace: vmware-system-csi spec: replicas: 1 selector: matchLabels: app: vsphere-csi-controller + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate template: metadata: labels: app: vsphere-csi-controller role: vsphere-csi spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/controlplane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - vsphere-csi-controller + topologyKey: kubernetes.io/hostname containers: - args: - --v=4 - --timeout=300s - --csi-address=$(ADDRESS) - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + - --kube-api-qps=100 + - --kube-api-burst=100 env: - name: ADDRESS value: /csi/csi.sock - image: quay.io/k8scsi/csi-attacher:v3.0.0 + image: registry.k8s.io/sig-storage/csi-attacher:v4.5.1 name: csi-attacher - resources: {} volumeMounts: - mountPath: /csi name: socket-dir - - env: + - args: + - --v=4 + - --timeout=300s + - --handle-volume-inuse-error=false + - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-resizer:v1.10.1 + name: csi-resizer + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: - name: CSI_ENDPOINT - value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + value: unix:///csi/csi.sock - name: X_CSI_MODE value: controller + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" + - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT + value: 3m - name: VSPHERE_CSI_CONFIG value: /etc/cloud/csi-vsphere.conf - name: LOGGER_LEVEL value: PRODUCTION - - name: X_CSI_LOG_LEVEL - value: INFO - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v2.1.0 + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: registry.k8s.io/csi-vsphere/driver:v3.3.1 + imagePullPolicy: Always livenessProbe: failureThreshold: 3 httpGet: path: /healthz port: healthz - initialDelaySeconds: 10 - periodSeconds: 5 - timeoutSeconds: 3 + initialDelaySeconds: 30 + periodSeconds: 180 + timeoutSeconds: 10 name: vsphere-csi-controller ports: - containerPort: 9808 name: healthz protocol: TCP - resources: {} + - containerPort: 2112 + name: prometheus + protocol: TCP + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 volumeMounts: - mountPath: /etc/cloud name: vsphere-config-volume readOnly: true - - mountPath: /var/lib/csi/sockets/pluginproxy/ + - mountPath: /csi name: socket-dir - args: - - --csi-address=$(ADDRESS) - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: quay.io/k8scsi/livenessprobe:v2.1.0 + - --v=4 + - --csi-address=/csi/csi.sock + image: registry.k8s.io/sig-storage/livenessprobe:v2.12.0 name: liveness-probe - resources: {} volumeMounts: - - mountPath: /var/lib/csi/sockets/pluginproxy/ + - mountPath: /csi name: socket-dir - args: - --leader-election + - --leader-election-lease-duration=30s + - --leader-election-renew-deadline=20s + - --leader-election-retry-period=10s + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) env: - - name: X_CSI_FULL_SYNC_INTERVAL_MINUTES + - name: FULL_SYNC_INTERVAL_MINUTES value: "30" - - name: LOGGER_LEVEL - value: PRODUCTION - name: VSPHERE_CSI_CONFIG value: /etc/cloud/csi-vsphere.conf - image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v2.1.0 + - name: LOGGER_LEVEL + value: PRODUCTION + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: registry.k8s.io/csi-vsphere/syncer:v3.3.1 + imagePullPolicy: Always name: vsphere-syncer - resources: {} + ports: + - containerPort: 2113 + name: prometheus + protocol: TCP + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 volumeMounts: - mountPath: /etc/cloud name: vsphere-config-volume @@ -741,235 +839,522 @@ data: - --v=4 - --timeout=300s - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s - --default-fstype=ext4 env: - name: ADDRESS value: /csi/csi.sock - image: quay.io/k8scsi/csi-provisioner:v2.0.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v4.0.1 name: csi-provisioner - resources: {} + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --v=4 + - --kube-api-qps=100 + - --kube-api-burst=100 + - --timeout=300s + - --csi-address=$(ADDRESS) + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-snapshotter:v7.0.2 + name: csi-snapshotter volumeMounts: - mountPath: /csi name: socket-dir dnsPolicy: Default + priorityClassName: system-cluster-critical serviceAccountName: vsphere-csi-controller tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists volumes: - name: vsphere-config-volume secret: - secretName: csi-vsphere-config + secretName: vsphere-config-secret - emptyDir: {} name: socket-dir -kind: ConfigMap -metadata: - name: vsphere-csi-controller - namespace: ${NAMESPACE} ---- -apiVersion: v1 -kind: Secret -metadata: - name: cloud-controller-manager - namespace: ${NAMESPACE} -stringData: - data: | - apiVersion: v1 - kind: ServiceAccount - metadata: - name: cloud-controller-manager - namespace: kube-system -type: addons.cluster.x-k8s.io/resource-set ---- -apiVersion: v1 -kind: Secret -metadata: - name: cloud-provider-vsphere-credentials - namespace: ${NAMESPACE} -stringData: - data: | - apiVersion: v1 - kind: Secret - metadata: - name: cloud-provider-vsphere-credentials - namespace: kube-system - stringData: - ${VSPHERE_SERVER}.password: ${VSPHERE_PASSWORD} - ${VSPHERE_SERVER}.username: ${VSPHERE_USERNAME} - type: Opaque -type: addons.cluster.x-k8s.io/resource-set ---- -apiVersion: v1 -data: - data: | --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole + apiVersion: apps/v1 + kind: DaemonSet metadata: - labels: - component: cloud-controller-manager - vsphere-cpi-infra: role - name: system:cloud-controller-manager - rules: - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update - - apiGroups: - - "" - resources: - - nodes - verbs: - - '*' - - apiGroups: - - "" - resources: - - nodes/status - verbs: - - patch - - apiGroups: - - "" - resources: - - services - verbs: - - list - - patch - - update - - watch - - apiGroups: - - "" - resources: - - services/status - verbs: - - patch - - apiGroups: - - "" - resources: - - serviceaccounts - verbs: - - create - - get - - list - - watch - - update - - apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - get - - list - - watch - - update - - apiGroups: - - "" - resources: - - endpoints - verbs: - - create - - get - - list - - watch - - update - - apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - watch - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - watch - - list - - update - - create + name: vsphere-csi-node + namespace: vmware-system-csi + spec: + selector: + matchLabels: + app: vsphere-csi-node + template: + metadata: + labels: + app: vsphere-csi-node + role: vsphere-csi + spec: + containers: + - args: + - --v=5 + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.10.1 + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock + - --mode=kubelet-registration-probe + initialDelaySeconds: 3 + name: node-driver-registrar + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: MAX_VOLUMES_PER_NODE + value: "59" + - name: X_CSI_MODE + value: node + - name: X_CSI_SPEC_REQ_VALIDATION + value: "false" + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" + - name: LOGGER_LEVEL + value: PRODUCTION + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES + value: "1" + image: registry.k8s.io/csi-vsphere/driver:v3.3.1 + imagePullPolicy: Always + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 5 + name: vsphere-csi-node + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - SYS_ADMIN + privileged: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: pods-mount-dir + - mountPath: /dev + name: device-dir + - mountPath: /sys/block + name: blocks-dir + - mountPath: /sys/devices + name: sys-devices-dir + - args: + - --v=4 + - --csi-address=/csi/csi.sock + image: registry.k8s.io/sig-storage/livenessprobe:v2.12.0 + name: liveness-probe + volumeMounts: + - mountPath: /csi + name: plugin-dir + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + serviceAccountName: vsphere-csi-node + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory + name: registration-dir + - hostPath: + path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet + type: Directory + name: pods-mount-dir + - hostPath: + path: /dev + name: device-dir + - hostPath: + path: /sys/block + type: Directory + name: blocks-dir + - hostPath: + path: /sys/devices + type: Directory + name: sys-devices-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: vsphere-csi-node-windows + namespace: vmware-system-csi + spec: + selector: + matchLabels: + app: vsphere-csi-node-windows + template: + metadata: + labels: + app: vsphere-csi-node-windows + role: vsphere-csi-windows + spec: + containers: + - args: + - --v=5 + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + env: + - name: ADDRESS + value: unix://C:\\csi\\csi.sock + - name: DRIVER_REG_SOCK_PATH + value: C:\\var\\lib\\kubelet\\plugins\\csi.vsphere.vmware.com\\csi.sock + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.10.0 + livenessProbe: + exec: + command: + - /csi-node-driver-registrar.exe + - --kubelet-registration-path=C:\\var\\lib\\kubelet\\plugins\\csi.vsphere.vmware.com\\csi.sock + - --mode=kubelet-registration-probe + initialDelaySeconds: 3 + name: node-driver-registrar + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix://C:\\csi\\csi.sock + - name: MAX_VOLUMES_PER_NODE + value: "59" + - name: X_CSI_MODE + value: node + - name: X_CSI_SPEC_REQ_VALIDATION + value: "false" + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" + - name: LOGGER_LEVEL + value: PRODUCTION + - name: X_CSI_LOG_LEVEL + value: DEBUG + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES + value: "1" + image: registry.k8s.io/csi-vsphere/driver:v3.3.1 + imagePullPolicy: Always + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 5 + name: vsphere-csi-node + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + volumeMounts: + - mountPath: C:\csi + name: plugin-dir + - mountPath: C:\var\lib\kubelet + name: pods-mount-dir + - mountPath: \\.\pipe\csi-proxy-volume-v1 + name: csi-proxy-volume-v1 + - mountPath: \\.\pipe\csi-proxy-filesystem-v1 + name: csi-proxy-filesystem-v1 + - mountPath: \\.\pipe\csi-proxy-disk-v1 + name: csi-proxy-disk-v1 + - mountPath: \\.\pipe\csi-proxy-system-v1alpha1 + name: csi-proxy-system-v1alpha1 + - args: + - --v=4 + - --csi-address=/csi/csi.sock + image: registry.k8s.io/sig-storage/livenessprobe:v2.12.0 + name: liveness-probe + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: windows + priorityClassName: system-node-critical + serviceAccountName: vsphere-csi-node + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists + volumes: + - hostPath: + path: C:\var\lib\kubelet\plugins_registry\ + type: Directory + name: registration-dir + - hostPath: + path: C:\var\lib\kubelet\plugins\csi.vsphere.vmware.com\ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: \var\lib\kubelet + type: Directory + name: pods-mount-dir + - hostPath: + path: \\.\pipe\csi-proxy-disk-v1 + type: "" + name: csi-proxy-disk-v1 + - hostPath: + path: \\.\pipe\csi-proxy-volume-v1 + type: "" + name: csi-proxy-volume-v1 + - hostPath: + path: \\.\pipe\csi-proxy-filesystem-v1 + type: "" + name: csi-proxy-filesystem-v1 + - hostPath: + path: \\.\pipe\csi-proxy-system-v1alpha1 + type: "" + name: csi-proxy-system-v1alpha1 + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate +kind: ConfigMap +metadata: + name: csi-manifests + namespace: ${NAMESPACE} +--- +apiVersion: v1 +kind: Secret +metadata: + name: cloud-provider-vsphere-credentials + namespace: ${NAMESPACE} +stringData: + data: |- + apiVersion: v1 + kind: Secret metadata: labels: component: cloud-controller-manager - vsphere-cpi-infra: cluster-role-binding - name: system:cloud-controller-manager - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:cloud-controller-manager - subjects: - - kind: ServiceAccount - name: cloud-controller-manager + vsphere-cpi-infra: secret + name: cloud-provider-vsphere-credentials namespace: kube-system - - kind: User - name: cloud-controller-manager + stringData: + ${VSPHERE_SERVER}.password: "${VSPHERE_PASSWORD}" + ${VSPHERE_SERVER}.username: "${VSPHERE_USERNAME}" + type: Opaque +type: addons.cluster.x-k8s.io/resource-set +--- +apiVersion: v1 +data: + data: |- --- + # Source: vsphere-cpi/templates/service-account.yaml apiVersion: v1 - data: - vsphere.conf: | - global: - port: 443 - secretName: cloud-provider-vsphere-credentials - secretNamespace: kube-system - thumbprint: '${VSPHERE_TLS_THUMBPRINT}' - vcenter: - ${VSPHERE_SERVER}: - datacenters: - - '${VSPHERE_DATACENTER}' - server: '${VSPHERE_SERVER}' - kind: ConfigMap + kind: ServiceAccount metadata: - name: vsphere-cloud-config + name: cloud-controller-manager + labels: + app: vsphere-cpi + vsphere-cpi-infra: service-account + component: cloud-controller-manager namespace: kube-system --- + # Source: vsphere-cpi/templates/role.yaml apiVersion: rbac.authorization.k8s.io/v1 - kind: RoleBinding + kind: ClusterRole metadata: + name: cloud-controller-manager labels: + app: vsphere-cpi + vsphere-cpi-infra: role component: cloud-controller-manager - vsphere-cpi-infra: role-binding - name: servicecatalog.k8s.io:apiserver-authentication-reader - namespace: kube-system - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader - subjects: - - kind: ServiceAccount - name: cloud-controller-manager - namespace: kube-system - - kind: User - name: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - patch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - "coordination.k8s.io" + resources: + - leases + verbs: + - create + - get + - list + - watch + - update --- + # Source: vsphere-cpi/templates/daemonset.yaml apiVersion: apps/v1 kind: DaemonSet metadata: + name: vsphere-cpi labels: + app: vsphere-cpi + vsphere-cpi-infra: daemonset component: cloud-controller-manager tier: control-plane - name: vsphere-cloud-controller-manager namespace: kube-system + annotations: spec: selector: matchLabels: - name: vsphere-cloud-controller-manager + app: vsphere-cpi + updateStrategy: + type: RollingUpdate template: metadata: labels: + app: vsphere-cpi component: cloud-controller-manager - name: vsphere-cloud-controller-manager tier: control-plane + release: release-name + vsphere-cpi-infra: daemonset spec: + tolerations: + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + - effect: NoExecute + key: CriticalAddonsOnly + operator: Exists affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -977,58 +1362,89 @@ data: - matchExpressions: - key: node-role.kubernetes.io/control-plane operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists - containers: - - args: - - --v=2 - - --cloud-provider=vsphere - - --cloud-config=/etc/cloud/vsphere.conf - image: gcr.io/cloud-provider-vsphere/cpi/release/manager:${CPI_IMAGE_K8S_VERSION} - name: vsphere-cloud-controller-manager - resources: - requests: - cpu: 200m - volumeMounts: - - mountPath: /etc/cloud - name: vsphere-config-volume - readOnly: true - hostNetwork: true - priorityClassName: system-node-critical securityContext: + fsGroup: 1001 runAsUser: 1001 serviceAccountName: cloud-controller-manager - tolerations: - - effect: NoSchedule - key: node.cloudprovider.kubernetes.io/uninitialized - value: "true" - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Exists - - effect: NoSchedule - key: node.kubernetes.io/not-ready - operator: Exists + hostNetwork: true + dnsPolicy: ClusterFirst + priorityClassName: system-node-critical + containers: + - name: vsphere-cpi + image: registry.k8s.io/cloud-pv-vsphere/cloud-provider-vsphere:${CPI_IMAGE_K8S_VERSION} + imagePullPolicy: IfNotPresent + args: + - --cloud-provider=vsphere + - --v=2 + - --cloud-config=/etc/cloud/vsphere.conf + volumeMounts: + - mountPath: /etc/cloud + name: vsphere-config-volume + readOnly: true volumes: - - configMap: - name: vsphere-cloud-config - name: vsphere-config-volume - updateStrategy: - type: RollingUpdate + - name: vsphere-config-volume + configMap: + name: cloud-config + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + labels: + app: vsphere-cpi + component: cloud-controller-manager + vsphere-cpi-infra: role-binding + name: servicecatalog.k8s.io:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app: vsphere-cpi + component: cloud-controller-manager + vsphere-cpi-infra: cluster-role-binding + name: cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + data: + vsphere.conf: | + global: + port: 443 + secretName: cloud-provider-vsphere-credentials + secretNamespace: kube-system + thumbprint: '${VSPHERE_TLS_THUMBPRINT}' + vcenter: + ${VSPHERE_SERVER}: + datacenters: + - '${VSPHERE_DATACENTER}' + server: '${VSPHERE_SERVER}' + kind: ConfigMap + metadata: + name: cloud-config + namespace: kube-system kind: ConfigMap metadata: name: cpi-manifests namespace: ${NAMESPACE} ---- -apiVersion: v1 -kind: Secret -metadata: - name: ${CLUSTER_NAME} - namespace: ${NAMESPACE} -stringData: - password: ${VSPHERE_PASSWORD} - username: ${VSPHERE_USERNAME} ---- \ No newline at end of file