diff --git a/README.md b/README.md index 688b3955fc..0decac0493 100644 --- a/README.md +++ b/README.md @@ -7,13 +7,14 @@ This driver allows Kubernetes to use [azure file](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction) volume, csi plugin name: `file.csi.azure.com` ### Project Status -Status: Alpha +Status: Beta ### Container Images & CSI Compatibility: |Azure File CSI Driver Version | Image | v0.3.0| v1.0.0 | |-------------------------------|----------------------------------------------------|-------|--------| |v0.1.0-alpha |mcr.microsoft.com/k8s/csi/azurefile-csi:v0.1.0-alpha| yes | no | |v0.2.0-alpha |mcr.microsoft.com/k8s/csi/azurefile-csi:v0.2.0-alpha| no | yes | +|v0.3.0 |mcr.microsoft.com/k8s/csi/azurefile-csi:v0.3.0 | no | yes | |master branch |mcr.microsoft.com/k8s/csi/azurefile-csi:latest | no | yes | ### Kubernetes Compatibility @@ -21,6 +22,7 @@ Status: Alpha |------------------------------------------|------|-------| | v0.1.0-alpha | yes | yes | | v0.2.0-alpha | no | yes | +| v0.3.0 | no | yes | | master branch | no | yes | ### Driver parameters diff --git a/deploy/v0.3.0/crd-csi-driver-registry.yaml b/deploy/v0.3.0/crd-csi-driver-registry.yaml new file mode 100644 index 0000000000..56beef7317 --- /dev/null +++ b/deploy/v0.3.0/crd-csi-driver-registry.yaml @@ -0,0 +1,35 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + creationTimestamp: null + name: csidrivers.csi.storage.k8s.io +spec: + group: csi.storage.k8s.io + names: + kind: CSIDriver + plural: csidrivers + scope: Cluster + validation: + openAPIV3Schema: + properties: + spec: + description: Specification of the CSI Driver. + properties: + attachRequired: + description: + Indicates this CSI volume driver requires an attach operation, + and that Kubernetes should call attach and wait for any attach operation + to complete before proceeding to mount. + type: boolean + podInfoOnMountVersion: + description: + Indicates this CSI volume driver requires additional pod + information (like podName, podUID, etc.) during mount operations. + type: string + version: v1alpha1 +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/deploy/v0.3.0/crd-csi-node-info.yaml b/deploy/v0.3.0/crd-csi-node-info.yaml new file mode 100644 index 0000000000..103f8d5032 --- /dev/null +++ b/deploy/v0.3.0/crd-csi-node-info.yaml @@ -0,0 +1,37 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + creationTimestamp: null + name: csinodeinfos.csi.storage.k8s.io +spec: + group: csi.storage.k8s.io + names: + kind: CSINodeInfo + plural: csinodeinfos + scope: Cluster + validation: + openAPIV3Schema: + properties: + csiDrivers: + description: List of CSI drivers running on the node and their properties. + items: + properties: + driver: + description: The CSI driver that this object refers to. + type: string + nodeID: + description: The node from the driver point of view. + type: string + topologyKeys: + description: List of keys supported by the driver. + items: + type: string + type: array + type: array + version: v1alpha1 +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/deploy/v0.3.0/csi-azurefile-controller.yaml b/deploy/v0.3.0/csi-azurefile-controller.yaml new file mode 100644 index 0000000000..b6819563fc --- /dev/null +++ b/deploy/v0.3.0/csi-azurefile-controller.yaml @@ -0,0 +1,109 @@ +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: csi-azurefile-controller + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: csi-azurefile-controller + template: + metadata: + labels: + app: csi-azurefile-controller + spec: + serviceAccountName: csi-azurefile-controller-sa + nodeSelector: + beta.kubernetes.io/os: linux + containers: + - name: csi-provisioner + image: quay.io/k8scsi/csi-provisioner:v1.0.1 + args: + - "--provisioner=file.csi.azure.com" + - "--csi-address=$(ADDRESS)" + - "--connection-timeout=15s" + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: Always + volumeMounts: + - mountPath: /csi + name: socket-dir + - name: csi-attacher + image: quay.io/k8scsi/csi-attacher:v1.0.1 + args: + - --v=5 + - --csi-address=$(ADDRESS) + - --timeout=120s + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: Always + volumeMounts: + - mountPath: /csi + name: socket-dir + - name: cluster-driver-registrar + image: quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1 + args: + - --csi-address=$(ADDRESS) + - --driver-requires-attachment=true + - --v=5 + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: liveness-probe + image: quay.io/k8scsi/livenessprobe:v1.1.0 + args: + - --csi-address=/csi/csi.sock + - --connection-timeout=3s + - --health-port=9702 + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: azurefile + image: mcr.microsoft.com/k8s/csi/azurefile-csi:v0.3.0 + args: + - "--v=5" + - "--endpoint=$(CSI_ENDPOINT)" + - "--nodeid=$(KUBE_NODE_NAME)" + ports: + - containerPort: 9702 + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + env: + - name: AZURE_CREDENTIAL_FILE + value: "/etc/kubernetes/azure.json" + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + imagePullPolicy: Always + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /etc/kubernetes/ + name: azure-cred + - mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + name: msi + volumes: + - name: socket-dir + emptyDir: {} + - name: azure-cred + hostPath: + path: /etc/kubernetes/ + type: Directory + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings diff --git a/deploy/v0.3.0/csi-azurefile-node.yaml b/deploy/v0.3.0/csi-azurefile-node.yaml new file mode 100644 index 0000000000..2d5428abf7 --- /dev/null +++ b/deploy/v0.3.0/csi-azurefile-node.yaml @@ -0,0 +1,110 @@ +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-azurefile-node + namespace: kube-system +spec: + selector: + matchLabels: + app: csi-azurefile-node + template: + metadata: + labels: + app: csi-azurefile-node + spec: + hostNetwork: true + nodeSelector: + beta.kubernetes.io/os: linux + containers: + - name: liveness-probe + imagePullPolicy: Always + volumeMounts: + - mountPath: /csi + name: socket-dir + image: quay.io/k8scsi/livenessprobe:v1.0.2 + args: + - --csi-address=/csi/csi.sock + - --connection-timeout=3s + - --health-port=9702 + - name: node-driver-registrar + image: quay.io/k8scsi/csi-node-driver-registrar:v1.1.0 + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=5 + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/file.csi.azure.com-reg.sock /csi/csi.sock"] + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/file.csi.azure.com/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: azurefile + image: mcr.microsoft.com/k8s/csi/azurefile-csi:v0.3.0 + args: + - "--v=5" + - "--endpoint=$(CSI_ENDPOINT)" + - "--nodeid=$(KUBE_NODE_NAME)" + ports: + - containerPort: 9702 + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + env: + - name: AZURE_CREDENTIAL_FILE + value: "/etc/kubernetes/azure.json" + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + imagePullPolicy: Always + securityContext: + privileged: true + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /var/lib/kubelet/ + mountPropagation: Bidirectional + name: mountpoint-dir + - mountPath: /etc/kubernetes/ + name: azure-cred + - mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + name: msi + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/file.csi.azure.com + type: DirectoryOrCreate + name: socket-dir + - hostPath: + path: /var/lib/kubelet/ + type: DirectoryOrCreate + name: mountpoint-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + name: registration-dir + - hostPath: + path: /etc/kubernetes/ + type: Directory + name: azure-cred + - hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings + name: msi diff --git a/deploy/v0.3.0/rbac-csi-azurefile-controller.yaml b/deploy/v0.3.0/rbac-csi-azurefile-controller.yaml new file mode 100644 index 0000000000..324f2ea014 --- /dev/null +++ b/deploy/v0.3.0/rbac-csi-azurefile-controller.yaml @@ -0,0 +1,160 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-azurefile-controller-sa + namespace: kube-system + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azurefile-external-provisioner-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azurefile-csi-provisioner-binding +subjects: + - kind: ServiceAccount + name: csi-azurefile-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azurefile-external-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azurefile-external-attacher-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azurefile-csi-attacher-binding +subjects: + - kind: ServiceAccount + name: csi-azurefile-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azurefile-external-attacher-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azurefile-cluster-driver-registrar-role +rules: + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csidrivers"] + verbs: ["create", "delete"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azurefile-csi-driver-registrar-binding +subjects: + - kind: ServiceAccount + name: csi-azurefile-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azurefile-cluster-driver-registrar-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azurefile-external-snapshotter-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azurefile-csi-snapshotter-binding +subjects: + - kind: ServiceAccount + name: csi-azurefile-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azurefile-external-snapshotter-role + apiGroup: rbac.authorization.k8s.io diff --git a/docs/install-azurefile-csi-driver.md b/docs/install-azurefile-csi-driver.md index f7f2fefab6..d080828483 100644 --- a/docs/install-azurefile-csi-driver.md +++ b/docs/install-azurefile-csi-driver.md @@ -1,4 +1,5 @@ ## Install azurefile CSI driver on a Kubernetes cluster - [install v0.2.0 CSI driver](./install-csi-driver-v0.2.0.md) + - [install v0.3.0 CSI driver](./install-csi-driver-v0.3.0.md) - [install CSI driver development version](./install-csi-driver-master.md) diff --git a/docs/install-csi-driver-v0.3.0.md b/docs/install-csi-driver-v0.3.0.md new file mode 100644 index 0000000000..13b533e6a0 --- /dev/null +++ b/docs/install-csi-driver-v0.3.0.md @@ -0,0 +1,36 @@ +# Install azurefile CSI driveri v0.3.0 on a kubernetes cluster + +## Installation with kubectl + +``` +kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/azurefile-csi-driver/master/deploy/v0.3.0/crd-csi-driver-registry.yaml +kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/azurefile-csi-driver/master/deploy/v0.3.0/crd-csi-node-info.yaml +kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/azurefile-csi-driver/master/deploy/v0.3.0/rbac-csi-azurefile-controller.yaml +kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/azurefile-csi-driver/master/deploy/v0.3.0/csi-azurefile-controller.yaml +kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/azurefile-csi-driver/master/deploy/v0.3.0/csi-azurefile-node.yaml +``` + +- check pods status: + +``` +watch kubectl get po -o wide -n kube-system | grep csi-azurefile +``` + +example output: + +``` +NAME READY STATUS RESTARTS AGE IP NODE +csi-azurefile-controller-56bfddd689-dh5tk 6/6 Running 0 35s 10.240.0.19 k8s-agentpool-22533604-0 +csi-azurefile-node-cvgbs 3/3 Running 0 7m4s 10.240.0.35 k8s-agentpool-22533604-1 +csi-azurefile-node-dr4s4 3/3 Running 0 7m4s 10.240.0.4 k8s-agentpool-22533604-0 +``` + +- clean up azure file CSI driver + +``` +kubectl delete -f https://raw.githubusercontent.com/kubernetes-sigs/azurefile-csi-driver/master/deploy/v0.3.0/csi-azurefile-controller.yaml +kubectl delete -f https://raw.githubusercontent.com/kubernetes-sigs/azurefile-csi-driver/master/deploy/v0.3.0/csi-azurefile-node.yaml +kubectl delete -f https://raw.githubusercontent.com/kubernetes-sigs/azurefile-csi-driver/master/deploy/v0.3.0/crd-csi-driver-registry.yaml +kubectl delete -f https://raw.githubusercontent.com/kubernetes-sigs/azurefile-csi-driver/master/deploy/v0.3.0/crd-csi-node-info.yaml +kubectl delete -f https://raw.githubusercontent.com/kubernetes-sigs/azurefile-csi-driver/master/deploy/v0.3.0/rbac-csi-azurefile-controller.yaml +``` diff --git a/test/e2e/README.md b/test/e2e/README.md index 38fa9c7f8e..861139b94f 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -2,7 +2,9 @@ ## Run E2E tests Locally ### Prerequisite -Make sure a kubernetes cluster is set up and kubeconfig is under $HOME/.kube/config + - Make sure a kubernetes cluster(with version >= 1.13) is set up and kubeconfig is under `$HOME/.kube/config` + - Copy out `/etc/kubernetes/azure.json` under one agent node to local machine + > For AKS cluster, need to modify `resourceGroup` to the node resource group name inside `/etc/kubernetes/azure.json` ### Run test ```