From fcf1150a69cbdb5985080552f4ae10557c857125 Mon Sep 17 00:00:00 2001 From: Serge Logvinov Date: Sun, 15 Oct 2023 04:50:38 +0300 Subject: [PATCH] feat: disk speed limit Proxmox has a disk speed limit. We can pass these parameters during the mount process. This will help us understand how the application will perform in well-known cloud environments. Signed-off-by: Serge Logvinov --- Dockerfile | 2 +- README.md | 3 +- charts/proxmox-csi-plugin/Chart.yaml | 2 +- .../templates/csidriver.yaml | 1 + charts/proxmox-csi-plugin/values.yaml | 10 +++--- docker-compose.yml | 6 ++-- docs/options.md | 34 ++++++++++++++++--- go.mod | 6 ++-- go.sum | 12 +++---- pkg/csi/controller.go | 22 ++++++++++++ pkg/csi/driver.go | 5 +++ 11 files changed, 78 insertions(+), 25 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2ba4d27..45b48ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # syntax = docker/dockerfile:1.4 ######################################## -FROM golang:1.21-buster AS develop +FROM golang:1.21-bookworm AS develop WORKDIR /src COPY ["go.mod", "go.sum", "/src"] diff --git a/README.md b/README.md index 1335274..46000af 100644 --- a/README.md +++ b/README.md @@ -310,10 +310,11 @@ For more detailed options and a comprehensive understanding, refer to the follow * [Volume Expansion](https://kubernetes-csi.github.io/docs/volume-expansion.html): Volumes can be expanded by editing `PersistentVolumeClaim` objects. * [Storage capacity](https://kubernetes.io/docs/concepts/storage/storage-capacity/): Controller expose the Proxmox storade capacity. * [Encrypted volumes](https://kubernetes-csi.github.io/docs/secrets-and-credentials-storage-class.html): Encryption with LUKS. +* [Volume bandwidth](https://pve.proxmox.com/wiki/Manual:_qm.conf): Maximum read/write limits. ### Planned features -* [Volume bandwidth](https://pve.proxmox.com/wiki/Manual:_qm.conf): Maximum read/write limits. +* [Volume Snapshot](https://kubernetes-csi.github.io/docs/snapshot-restore-feature.html): Create snapshots of volumes. ## Resources diff --git a/charts/proxmox-csi-plugin/Chart.yaml b/charts/proxmox-csi-plugin/Chart.yaml index fd34e38..7dc3658 100644 --- a/charts/proxmox-csi-plugin/Chart.yaml +++ b/charts/proxmox-csi-plugin/Chart.yaml @@ -17,7 +17,7 @@ maintainers: # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.6 +version: 0.1.7 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/proxmox-csi-plugin/templates/csidriver.yaml b/charts/proxmox-csi-plugin/templates/csidriver.yaml index 5d0652e..7de851f 100644 --- a/charts/proxmox-csi-plugin/templates/csidriver.yaml +++ b/charts/proxmox-csi-plugin/templates/csidriver.yaml @@ -8,3 +8,4 @@ spec: storageCapacity: true volumeLifecycleModes: - Persistent + - Ephemeral diff --git a/charts/proxmox-csi-plugin/values.yaml b/charts/proxmox-csi-plugin/values.yaml index 1637be8..3dd9c97 100644 --- a/charts/proxmox-csi-plugin/values.yaml +++ b/charts/proxmox-csi-plugin/values.yaml @@ -84,7 +84,7 @@ controller: image: repository: registry.k8s.io/sig-storage/csi-attacher pullPolicy: IfNotPresent - tag: v4.3.0 + tag: v4.4.0 # -- Attacher resource requests and limits. # ref: https://kubernetes.io/docs/user-guide/compute-resources/ resources: @@ -96,7 +96,7 @@ controller: image: repository: registry.k8s.io/sig-storage/csi-provisioner pullPolicy: IfNotPresent - tag: v3.5.0 + tag: v3.6.0 # -- Provisioner resource requests and limits. # ref: https://kubernetes.io/docs/user-guide/compute-resources/ resources: @@ -108,7 +108,7 @@ controller: image: repository: registry.k8s.io/sig-storage/csi-resizer pullPolicy: IfNotPresent - tag: v1.8.0 + tag: v1.9.0 # -- Resizer resource requests and limits. # ref: https://kubernetes.io/docs/user-guide/compute-resources/ resources: @@ -132,7 +132,7 @@ node: image: repository: registry.k8s.io/sig-storage/csi-node-driver-registrar pullPolicy: IfNotPresent - tag: v2.8.0 + tag: v2.9.0 # -- Node registrar resource requests and limits. # ref: https://kubernetes.io/docs/user-guide/compute-resources/ resources: @@ -159,7 +159,7 @@ livenessprobe: image: repository: registry.k8s.io/sig-storage/livenessprobe pullPolicy: IfNotPresent - tag: v2.10.0 + tag: v2.11.0 # -- Failure threshold for livenessProbe failureThreshold: 5 # -- Initial delay seconds for livenessProbe diff --git a/docker-compose.yml b/docker-compose.yml index 1423695..a7b8223 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,7 +24,7 @@ services: source: ./ target: /src csi-attacher: - image: registry.k8s.io/sig-storage/csi-attacher:v4.3.0 + image: registry.k8s.io/sig-storage/csi-attacher:v4.4.0 network_mode: "service:base" command: - "--v=5" @@ -40,7 +40,7 @@ services: source: ./hack target: /etc/kubernetes csi-resizer: - image: registry.k8s.io/sig-storage/csi-resizer:v1.8.0 + image: registry.k8s.io/sig-storage/csi-resizer:v1.9.0 network_mode: "service:base" command: - "--v=5" @@ -56,7 +56,7 @@ services: source: ./hack target: /etc/kubernetes csi-provisioner: - image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.0 network_mode: "service:base" command: - "--v=5" diff --git a/docs/options.md b/docs/options.md index b6fd1c0..86ed5b8 100644 --- a/docs/options.md +++ b/docs/options.md @@ -8,27 +8,48 @@ Deploy examples you can find [here](deploy/). apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: - name: proxmox-data-xfs + name: proxmox-storage-class-name parameters: # Pre defined options - csi.storage.k8s.io/fstype: xfs|ext4 - ## If you want to encrypt the disk + ## File system format (default: ext4) + csi.storage.k8s.io/fstype: ext4|xfs + + ## Optional: If you want to encrypt the disk csi.storage.k8s.io/node-stage-secret-name: "proxmox-csi-secret" csi.storage.k8s.io/node-stage-secret-namespace: "kube-system" + ## Have to be the same as node-stage-secret-* if you want to expand the volume csi.storage.k8s.io/node-expand-secret-name: "proxmox-csi-secret" csi.storage.k8s.io/node-expand-secret-namespace: "kube-system" # Proxmox csi options + ## Proxmox storage ID storage: data + + ## Optional: Proxmox csi options cache: directsync|none|writeback|writethrough ssd: "true|false" -# This field allows you to specify additional mount options to be applied when the volume is mounted on the node + ## Optional: Proxmox disk speed limit + diskIOPS: "4000" + diskMBps: "1000" + +# Optional: This field allows you to specify additional mount options to be applied when the volume is mounted on the node mountOptions: # Common for ssd - noatime +# Optional: Allowed topologies restricts what nodes this StorageClass can be used on +allowedTopologies: +- matchLabelExpressions: + - key: topology.kubernetes.io/region + values: + - Region-1 + - key: topology.kubernetes.io/zone + values: + - pve-1 + - pve-3 + provisioner: csi.proxmox.sinextra.dev allowVolumeExpansion: true reclaimPolicy: Delete|Retain @@ -51,7 +72,10 @@ metadata: * `storage` - proxmox storage ID * `cache` - qemu cache param: `directsync`, `none`, `writeback`, `writethrough` [Official documentation](https://pve.proxmox.com/wiki/Performance_Tweaks) -* `ssd` - true if SSD/NVME disk +* `ssd` - set true if SSD/NVME disk + +* `diskIOPS` - maximum r/w I/O in operations per second +* `diskMBps` - maximum r/w throughput in megabytes per second ## AllowVolumeExpansion diff --git a/go.mod b/go.mod index 127aea3..9e464e7 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/sergelogvinov/proxmox-cloud-controller-manager v0.2.0 github.com/siderolabs/go-blockdevice v0.4.7-0.20230911102818-8a2102a6ec14 github.com/stretchr/testify v1.8.4 - google.golang.org/grpc v1.56.2 + google.golang.org/grpc v1.58.3 k8s.io/api v0.28.2 k8s.io/apimachinery v0.28.2 k8s.io/client-go v0.28.2 @@ -35,7 +35,7 @@ require ( github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.3.0 // indirect - github.com/imdario/mergo v0.3.15 // indirect + github.com/imdario/mergo v0.3.16 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -56,7 +56,7 @@ require ( golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 068ab45..e9073d1 100644 --- a/go.sum +++ b/go.sum @@ -41,8 +41,8 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJY github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= -github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= +github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww= github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -149,10 +149,10 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI= -google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= +google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= +google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= diff --git a/pkg/csi/controller.go b/pkg/csi/controller.go index 6b9a28f..a95d111 100644 --- a/pkg/csi/controller.go +++ b/pkg/csi/controller.go @@ -309,6 +309,28 @@ func (d *ControllerService) ControllerPublishVolume(_ context.Context, request * options["cache"] = volCtx[StorageCacheKey] } + if volCtx[StorageDiskIOPS] != "" { + iops, err := strconv.Atoi(volCtx[StorageDiskIOPS]) //nolint:govet + if err != nil { + klog.Errorf("failed %s must be a number: %v", StorageDiskIOPS, err) + + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("failed %s must be a number: %v", StorageDiskIOPS, err)) + } + + options["iops"] = strconv.Itoa(iops) + } + + if volCtx[StorageDiskMBps] != "" { + mbps, err := strconv.Atoi(volCtx[StorageDiskMBps]) //nolint:govet + if err != nil { + klog.Errorf("failed %s must be a number: %v", StorageDiskMBps, err) + + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("failed %s must be a number: %v", StorageDiskMBps, err)) + } + + options["mbps"] = strconv.Itoa(mbps) + } + d.volumeLocks.Lock() defer d.volumeLocks.Unlock() diff --git a/pkg/csi/driver.go b/pkg/csi/driver.go index d86021e..3deec2b 100644 --- a/pkg/csi/driver.go +++ b/pkg/csi/driver.go @@ -30,6 +30,11 @@ const ( // StorageSSDKey is it ssd disk StorageSSDKey = "ssd" + // StorageDiskIOPS is maximum r/w I/O in operations per second + StorageDiskIOPS = "diskIOPS" + // StorageDiskMBps is maximum r/w throughput in MB/s + StorageDiskMBps = "diskMBps" + // MaxVolumesPerNode is the maximum number of volumes that can be attached to a node MaxVolumesPerNode = 16 // MinVolumeSize is the minimum size of a volume