-
Notifications
You must be signed in to change notification settings - Fork 554
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
rbd: support QoS based on capacity for rbd volume
1. QoS provides settings for rbd volume read/write iops and read/write bandwidth. 2. All QoS parameters are placed in the SC, send QoS parameters from SC to Cephcsi through PVC create request. 3. We need provide QoS parameters in the SC as below: - BaseReadIops - BaseWriteIops - BaseReadBytesPerSecond - BaseWriteBytesPerSecond - ReadIopsPerGB - WriteIopsPerGB - ReadBpsPerGB - WriteBpsPerGB - BaseVolSizeBytes There are 4 base qos parameters among them, when users apply for a volume capacity equal to or less than BaseVolSizebytes, use base qos limit. For the portion of capacity exceeding BaseVolSizebytes, QoS will be increased in steps set per GB. If the step size parameter per GB is not provided, only base QoS limit will be used and not associated with capacity size. 4. If PVC has resize request, adjust the QoS limit according to the QoS parameters after resizing. Signed-off-by: Yite Gu <[email protected]>
- Loading branch information
Showing
4 changed files
with
355 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,299 @@ | ||
| /* | ||
| Copyright 2024 The Ceph-CSI Authors. | ||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package rbd | ||
|
|
||
| import ( | ||
| "context" | ||
| "errors" | ||
| "strconv" | ||
|
|
||
| "github.com/ceph/ceph-csi/internal/util/log" | ||
|
|
||
| librbd "github.com/ceph/go-ceph/rbd" | ||
| ) | ||
|
|
||
| const ( | ||
| // Qos parameters name of StorageClass. | ||
| baseReadIops = "BaseReadIops" | ||
| baseWriteIops = "BaseWriteIops" | ||
| baseReadBytesPerSecond = "BaseReadBytesPerSecond" | ||
| baseWriteBytesPerSecond = "BaseWriteBytesPerSecond" | ||
| readIopsPerGB = "ReadIopsPerGB" | ||
| writeIopsPerGB = "WriteIopsPerGB" | ||
| readBpsPerGB = "ReadBpsPerGB" | ||
| writeBpsPerGB = "WriteBpsPerGB" | ||
| baseVolSizeBytes = "BaseVolSizeBytes" | ||
|
|
||
| // Qos type name of rbd image. | ||
| readIopsLimit = "rbd_qos_read_iops_limit" | ||
| writeIopsLimit = "rbd_qos_write_iops_limit" | ||
| readBpsLimit = "rbd_qos_read_bps_limit" | ||
| writeBpsLimit = "rbd_qos_write_bps_limit" | ||
| metadataConfPrefix = "conf_" | ||
|
|
||
| // The params use to calc qos based on capacity. | ||
| baseQosReadIopsLimit = "rbd_base_qos_read_iops_limit" | ||
| baseQosWriteIopsLimit = "rbd_base_qos_write_iops_limit" | ||
| baseQosReadBpsLimit = "rbd_base_qos_read_bps_limit" | ||
| baseQosWriteBpsLimit = "rbd_base_qos_write_bps_limit" | ||
| readIopsPerGBLimit = "rbd_read_iops_per_gb_limit" | ||
| writeIopsPerGBLimit = "rbd_write_iops_per_gb_limit" | ||
| readBpsPerGBLimit = "rbd_read_bps_per_gb_limit" | ||
| writeBpsPerGBLimit = "rbd_write_bps_per_gb_limit" | ||
| baseQosVolSize = "rbd_base_qos_vol_size" | ||
| ) | ||
|
|
||
| type qosSpec struct { | ||
| baseLimitType string | ||
| baseLimit string | ||
| perGBLimitType string | ||
| perGBLimit string | ||
| provide bool | ||
| } | ||
|
|
||
| func parseQosParams( | ||
| params map[string]string, | ||
| ) map[string]*qosSpec { | ||
| rbdQosParameters := map[string]*qosSpec{ | ||
| baseReadIops: {readIopsLimit, "", readIopsPerGB, "", false}, | ||
| baseWriteIops: {writeIopsLimit, "", writeIopsPerGB, "", false}, | ||
| baseReadBytesPerSecond: {readBpsLimit, "", readBpsPerGB, "", false}, | ||
| baseWriteBytesPerSecond: {writeBpsLimit, "", writeBpsPerGB, "", false}, | ||
| } | ||
| for k, v := range params { | ||
| if qos, ok := rbdQosParameters[k]; ok && v != "" { | ||
| qos.baseLimit = v | ||
| qos.provide = true | ||
| for p, q := range params { | ||
| if p == qos.perGBLimitType { | ||
| if q != "" { | ||
| qos.perGBLimit = q | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return rbdQosParameters | ||
| } | ||
|
|
||
| func (rv *rbdVolume) SetQOS( | ||
| ctx context.Context, | ||
| params map[string]string, | ||
| ) error { | ||
| baseVolSize := "" | ||
| if v, ok := params[baseVolSizeBytes]; ok && v != "" { | ||
| baseVolSize = v | ||
| } | ||
|
|
||
| rbdQosParameters := parseQosParams(params) | ||
| for _, qos := range rbdQosParameters { | ||
| if qos.provide { | ||
| err := calcQosBasedOnCapacity(ctx, rv, *qos, baseVolSize) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func (rv *rbdVolume) ApplyQOS( | ||
| ctx context.Context, | ||
| ) error { | ||
| if len(rv.QosParameters) != 0 { | ||
| err := setRbdImageQos(ctx, rv) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func calcQosBasedOnCapacity( | ||
| ctx context.Context, | ||
| rbdVol *rbdVolume, | ||
| qos qosSpec, | ||
| baseVolSize string, | ||
| ) error { | ||
| if rbdVol.QosParameters == nil { | ||
| rbdVol.QosParameters = make(map[string]string) | ||
| } | ||
|
|
||
| // Don't set qos if base qos limit empty. | ||
| if qos.baseLimit == "" { | ||
| return nil | ||
| } | ||
| baseLimit, err := strconv.ParseInt(qos.baseLimit, 10, 64) | ||
| if err != nil { | ||
| log.ErrorLog(ctx, "%v", err) | ||
|
|
||
| return err | ||
| } | ||
|
|
||
| // if provide qosPerGB and baseVolSize, we will set qos based on capacity, | ||
| // otherwise, we only set base qos limit. | ||
| if qos.perGBLimit != "" && baseVolSize != "" { | ||
| perGBLimit, err := strconv.ParseInt(qos.perGBLimit, 10, 64) | ||
| if err != nil { | ||
| log.ErrorLog(ctx, "%v", err) | ||
|
|
||
| return err | ||
| } | ||
|
|
||
| baseVolSizeInt, err := strconv.ParseInt(baseVolSize, 10, 64) | ||
| if err != nil { | ||
| log.ErrorLog(ctx, "%v", err) | ||
|
|
||
| return err | ||
| } | ||
|
|
||
| if rbdVol.VolSize <= baseVolSizeInt { | ||
| rbdVol.QosParameters[qos.baseLimitType] = qos.baseLimit | ||
| } else { | ||
| capacityQos := (rbdVol.VolSize - baseVolSizeInt) / int64(oneGB) * perGBLimit | ||
| finalQosLimit := baseLimit + capacityQos | ||
| rbdVol.QosParameters[qos.baseLimitType] = strconv.FormatInt(finalQosLimit, 10) | ||
| } | ||
| } else { | ||
| rbdVol.QosParameters[qos.baseLimitType] = qos.baseLimit | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func setRbdImageQos( | ||
| ctx context.Context, | ||
| rbdVol *rbdVolume, | ||
| ) error { | ||
| for k, v := range rbdVol.QosParameters { | ||
| err := rbdVol.SetMetadata(metadataConfPrefix+k, v) | ||
| if err != nil { | ||
| log.ErrorLog(ctx, "failed to set rbd qos, %s: %s, %v", k, v, err) | ||
|
|
||
| return err | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func (rv *rbdVolume) SaveQOS( | ||
| ctx context.Context, | ||
| params map[string]string, | ||
| ) error { | ||
| needSaveQosParameters := map[string]string{ | ||
| baseReadIops: baseQosReadIopsLimit, | ||
| baseWriteIops: baseQosWriteIopsLimit, | ||
| baseReadBytesPerSecond: baseQosReadBpsLimit, | ||
| baseWriteBytesPerSecond: baseQosWriteBpsLimit, | ||
| readIopsPerGB: readIopsPerGBLimit, | ||
| writeIopsPerGB: writeIopsPerGBLimit, | ||
| readBpsPerGB: readBpsPerGBLimit, | ||
| writeBpsPerGB: writeBpsPerGBLimit, | ||
| baseVolSizeBytes: baseQosVolSize, | ||
| } | ||
| for k, v := range params { | ||
| if param, ok := needSaveQosParameters[k]; ok { | ||
| if v != "" { | ||
| err := rv.SetMetadata(param, v) | ||
| if err != nil { | ||
| log.ErrorLog(ctx, "failed to save metadata, %s: %s, %v", k, v, err) | ||
|
|
||
| return err | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func getRbdImageQos( | ||
| ctx context.Context, | ||
| rbdVol *rbdVolume, | ||
| ) (map[string]qosSpec, string, error) { | ||
| QosParams := map[string]struct { | ||
| rbdQosType string | ||
| rbdQosPerGBType string | ||
| }{ | ||
| baseQosReadIopsLimit: {readIopsLimit, readIopsPerGBLimit}, | ||
| baseQosWriteIopsLimit: {writeIopsLimit, writeIopsPerGBLimit}, | ||
| baseQosReadBpsLimit: {readBpsLimit, readBpsPerGBLimit}, | ||
| baseQosWriteBpsLimit: {writeBpsLimit, writeBpsPerGBLimit}, | ||
| } | ||
| rbdQosParameters := make(map[string]qosSpec) | ||
| for k, param := range QosParams { | ||
| baseLimit, err := rbdVol.GetMetadata(k) | ||
| if errors.Is(err, librbd.ErrNotFound) { | ||
| // if base qos dose not exist, skipping. | ||
| continue | ||
| } else if err != nil { | ||
| log.ErrorLog(ctx, "failed to get metadata: %v", err) | ||
|
|
||
| return nil, "", err | ||
| } | ||
| perGBLimit, err := rbdVol.GetMetadata(param.rbdQosPerGBType) | ||
| if errors.Is(err, librbd.ErrNotFound) { | ||
| // rbdQosPerGBType does not exist, set it empty. | ||
| perGBLimit = "" | ||
| } else if err != nil { | ||
| log.ErrorLog(ctx, "failed to get metadata: %v", err) | ||
|
|
||
| return nil, "", err | ||
| } | ||
| rbdQosParameters[k] = qosSpec{param.rbdQosType, baseLimit, param.rbdQosPerGBType, perGBLimit, true} | ||
| } | ||
| baseVolSize, err := rbdVol.GetMetadata(baseQosVolSize) | ||
| if errors.Is(err, librbd.ErrNotFound) { | ||
| // rbdBaseQosVolSize does not exist, set it empty. | ||
| baseVolSize = "" | ||
| } else if err != nil { | ||
| log.ErrorLog(ctx, "failed to get metadata: %v", err) | ||
|
|
||
| return nil, "", err | ||
| } | ||
|
|
||
| return rbdQosParameters, baseVolSize, nil | ||
| } | ||
|
|
||
| func (rv *rbdVolume) AdjustQOS( | ||
| ctx context.Context, | ||
| ) error { | ||
| rbdQosParameters, baseQosVolSize, err := getRbdImageQos(ctx, rv) | ||
| if err != nil { | ||
| log.ErrorLog(ctx, "get image metadata failed: %v", err) | ||
|
|
||
| return err | ||
| } | ||
| for _, param := range rbdQosParameters { | ||
| err = calcQosBasedOnCapacity(ctx, rv, param, baseQosVolSize) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
| err = setRbdImageQos(ctx, rv) | ||
| if err != nil { | ||
| log.ErrorLog(ctx, "set image metadata failed: %v", err) | ||
|
|
||
| return err | ||
| } | ||
|
|
||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters