feat: disk speed limit

Proxmox has a disk speed limit. We can pass these parameters during the mount process. This will help us understand how the application will perform in well-known cloud environments. Signed-off-by: Serge Logvinov <[email protected]>
sergelogvinov · Oct 15, 2023 · fcf1150 · fcf1150
1 parent a243ffb
commit fcf1150
Show file tree

Hide file tree

Showing 11 changed files with 78 additions and 25 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,7 +1,7 @@
 # syntax = docker/dockerfile:1.4
 ########################################
 
-FROM golang:1.21-buster AS develop
+FROM golang:1.21-bookworm AS develop
 
 WORKDIR /src
 COPY ["go.mod", "go.sum", "/src"]

diff --git a/README.md b/README.md
@@ -310,10 +310,11 @@ For more detailed options and a comprehensive understanding, refer to the follow
 * [Volume Expansion](https://kubernetes-csi.github.io/docs/volume-expansion.html): Volumes can be expanded by editing `PersistentVolumeClaim` objects.
 * [Storage capacity](https://kubernetes.io/docs/concepts/storage/storage-capacity/): Controller expose the Proxmox storade capacity.
 * [Encrypted volumes](https://kubernetes-csi.github.io/docs/secrets-and-credentials-storage-class.html): Encryption with LUKS.
+* [Volume bandwidth](https://pve.proxmox.com/wiki/Manual:_qm.conf): Maximum read/write limits.
 
 ### Planned features
 
-* [Volume bandwidth](https://pve.proxmox.com/wiki/Manual:_qm.conf): Maximum read/write limits.
+* [Volume Snapshot](https://kubernetes-csi.github.io/docs/snapshot-restore-feature.html): Create snapshots of volumes.
 
 ## Resources
 

diff --git a/charts/proxmox-csi-plugin/Chart.yaml b/charts/proxmox-csi-plugin/Chart.yaml
@@ -17,7 +17,7 @@ maintainers:
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.6
+version: 0.1.7
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/charts/proxmox-csi-plugin/templates/csidriver.yaml b/charts/proxmox-csi-plugin/templates/csidriver.yaml
@@ -8,3 +8,4 @@ spec:
   storageCapacity: true
   volumeLifecycleModes:
   - Persistent
+  - Ephemeral
diff --git a/charts/proxmox-csi-plugin/values.yaml b/charts/proxmox-csi-plugin/values.yaml
@@ -84,7 +84,7 @@ controller:
     image:
       repository: registry.k8s.io/sig-storage/csi-attacher
       pullPolicy: IfNotPresent
-      tag: v4.3.0
+      tag: v4.4.0
     # -- Attacher resource requests and limits.
     # ref: https://kubernetes.io/docs/user-guide/compute-resources/
     resources:
@@ -96,7 +96,7 @@ controller:
     image:
       repository: registry.k8s.io/sig-storage/csi-provisioner
       pullPolicy: IfNotPresent
-      tag: v3.5.0
+      tag: v3.6.0
     # -- Provisioner resource requests and limits.
     # ref: https://kubernetes.io/docs/user-guide/compute-resources/
     resources:
@@ -108,7 +108,7 @@ controller:
     image:
       repository: registry.k8s.io/sig-storage/csi-resizer
       pullPolicy: IfNotPresent
-      tag: v1.8.0
+      tag: v1.9.0
     # -- Resizer resource requests and limits.
     # ref: https://kubernetes.io/docs/user-guide/compute-resources/
     resources:
@@ -132,7 +132,7 @@ node:
     image:
       repository: registry.k8s.io/sig-storage/csi-node-driver-registrar
       pullPolicy: IfNotPresent
-      tag: v2.8.0
+      tag: v2.9.0
     # -- Node registrar resource requests and limits.
     # ref: https://kubernetes.io/docs/user-guide/compute-resources/
     resources:
@@ -159,7 +159,7 @@ livenessprobe:
   image:
     repository: registry.k8s.io/sig-storage/livenessprobe
     pullPolicy: IfNotPresent
-    tag: v2.10.0
+    tag: v2.11.0
   # -- Failure threshold for livenessProbe
   failureThreshold: 5
   # -- Initial delay seconds for livenessProbe

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -24,7 +24,7 @@ services:
         source: ./
         target: /src
   csi-attacher:
-    image: registry.k8s.io/sig-storage/csi-attacher:v4.3.0
+    image: registry.k8s.io/sig-storage/csi-attacher:v4.4.0
     network_mode: "service:base"
     command:
       - "--v=5"
@@ -40,7 +40,7 @@ services:
         source: ./hack
         target: /etc/kubernetes
   csi-resizer:
-    image: registry.k8s.io/sig-storage/csi-resizer:v1.8.0
+    image: registry.k8s.io/sig-storage/csi-resizer:v1.9.0
     network_mode: "service:base"
     command:
       - "--v=5"
@@ -56,7 +56,7 @@ services:
         source: ./hack
         target: /etc/kubernetes
   csi-provisioner:
-    image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0
+    image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.0
     network_mode: "service:base"
     command:
       - "--v=5"

diff --git a/docs/options.md b/docs/options.md
@@ -8,27 +8,48 @@ Deploy examples you can find [here](deploy/).
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
-  name: proxmox-data-xfs
+  name: proxmox-storage-class-name
 
 parameters:
   # Pre defined options
-  csi.storage.k8s.io/fstype: xfs|ext4
-  ## If you want to encrypt the disk
+  ## File system format (default: ext4)
+  csi.storage.k8s.io/fstype: ext4|xfs
+
+  ## Optional: If you want to encrypt the disk
   csi.storage.k8s.io/node-stage-secret-name: "proxmox-csi-secret"
   csi.storage.k8s.io/node-stage-secret-namespace: "kube-system"
+  ## Have to be the same as node-stage-secret-* if you want to expand the volume
   csi.storage.k8s.io/node-expand-secret-name: "proxmox-csi-secret"
   csi.storage.k8s.io/node-expand-secret-namespace: "kube-system"
 
   # Proxmox csi options
+  ## Proxmox storage ID
   storage: data
+
+  ## Optional: Proxmox csi options
   cache: directsync|none|writeback|writethrough
   ssd: "true|false"
 
-# This field allows you to specify additional mount options to be applied when the volume is mounted on the node
+  ## Optional: Proxmox disk speed limit
+  diskIOPS: "4000"
+  diskMBps: "1000"
+
+# Optional: This field allows you to specify additional mount options to be applied when the volume is mounted on the node
 mountOptions:
   # Common for ssd
   - noatime
 
+# Optional: Allowed topologies restricts what nodes this StorageClass can be used on
+allowedTopologies:
+- matchLabelExpressions:
+  - key: topology.kubernetes.io/region
+    values:
+    - Region-1
+  - key: topology.kubernetes.io/zone
+    values:
+    - pve-1
+    - pve-3
+
 provisioner: csi.proxmox.sinextra.dev
 allowVolumeExpansion: true
 reclaimPolicy: Delete|Retain
@@ -51,7 +72,10 @@ metadata:
 
 * `storage` - proxmox storage ID
 * `cache` - qemu cache param: `directsync`, `none`, `writeback`, `writethrough` [Official documentation](https://pve.proxmox.com/wiki/Performance_Tweaks)
-* `ssd` - true if SSD/NVME disk
+* `ssd` - set true if SSD/NVME disk
+
+* `diskIOPS` - maximum r/w I/O in operations per second
+* `diskMBps` - maximum r/w throughput in megabytes per second
 
 ## AllowVolumeExpansion
 

diff --git a/go.mod b/go.mod
@@ -11,7 +11,7 @@ require (
 	github.com/sergelogvinov/proxmox-cloud-controller-manager v0.2.0
 	github.com/siderolabs/go-blockdevice v0.4.7-0.20230911102818-8a2102a6ec14
 	github.com/stretchr/testify v1.8.4
-	google.golang.org/grpc v1.56.2
+	google.golang.org/grpc v1.58.3
 	k8s.io/api v0.28.2
 	k8s.io/apimachinery v0.28.2
 	k8s.io/client-go v0.28.2
@@ -35,7 +35,7 @@ require (
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
-	github.com/imdario/mergo v0.3.15 // indirect
+	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
@@ -56,7 +56,7 @@ require (
 	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

diff --git a/go.sum b/go.sum
@@ -41,8 +41,8 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJY
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM=
-github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
+github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
+github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
 github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww=
 github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -149,10 +149,10 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
-google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI=
-google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM=
+google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ=
+google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=

diff --git a/pkg/csi/controller.go b/pkg/csi/controller.go
@@ -309,6 +309,28 @@ func (d *ControllerService) ControllerPublishVolume(_ context.Context, request *
 		options["cache"] = volCtx[StorageCacheKey]
 	}
 
+	if volCtx[StorageDiskIOPS] != "" {
+		iops, err := strconv.Atoi(volCtx[StorageDiskIOPS]) //nolint:govet
+		if err != nil {
+			klog.Errorf("failed %s must be a number: %v", StorageDiskIOPS, err)
+
+			return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("failed %s must be a number: %v", StorageDiskIOPS, err))
+		}
+
+		options["iops"] = strconv.Itoa(iops)
+	}
+
+	if volCtx[StorageDiskMBps] != "" {
+		mbps, err := strconv.Atoi(volCtx[StorageDiskMBps]) //nolint:govet
+		if err != nil {
+			klog.Errorf("failed %s must be a number: %v", StorageDiskMBps, err)
+
+			return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("failed %s must be a number: %v", StorageDiskMBps, err))
+		}
+
+		options["mbps"] = strconv.Itoa(mbps)
+	}
+
 	d.volumeLocks.Lock()
 	defer d.volumeLocks.Unlock()
 

diff --git a/pkg/csi/driver.go b/pkg/csi/driver.go
@@ -30,6 +30,11 @@ const (
 	// StorageSSDKey is it ssd disk
 	StorageSSDKey = "ssd"
 
+	// StorageDiskIOPS is maximum r/w I/O in operations per second
+	StorageDiskIOPS = "diskIOPS"
+	// StorageDiskMBps is maximum r/w throughput in MB/s
+	StorageDiskMBps = "diskMBps"
+
 	// MaxVolumesPerNode is the maximum number of volumes that can be attached to a node
 	MaxVolumesPerNode = 16
 	// MinVolumeSize is the minimum size of a volume