diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..019fa48 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,36 @@ +name: CI Create Release + +on: + push: + # Sequence of patterns matched against refs/tags + tags: + - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 + +jobs: + release: + runs-on: ubuntu-latest + steps: + - name: Checkout Source + uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.23.1' + - name: Get Version + id: branch-names + uses: tj-actions/branch-names@v8 + with: + strip_tag_prefix: v + - name: Build + run: | + chmod +x ./build.sh + ./build.sh ${{ steps.branch-names.outputs.tag }} X86 + ./build.sh ${{ steps.branch-names.outputs.tag }} ARM + - name: Create Release and Upload Release Asset + uses: softprops/action-gh-release@v2 + if: startsWith(github.ref, 'refs/tags/') + with: + body: TODO New Release. + draft: true + files: | + eSDK_Huawei_Storage_CSM_V${{ steps.branch-names.outputs.tag }}_X86_64.zip + eSDK_Huawei_Storage_CSM_V${{ steps.branch-names.outputs.tag }}_ARM_64.zip \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..306c984 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +# Created by .ignore support plugin (hsz.mobi) +### Example user template template +### Example user template + +# IntelliJ project files +.idea +*.iml +out +gen +go.sum diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8bb4938 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,38 @@ +# eg: docker build --target xxx --platform linux/amd64 --build-arg VER=${VER} -f Dockerfile -t xxx:${VER} . +ARG VER + +FROM busybox:stable-glibc as csm-prometheus-collector +LABEL version="${VER}" +LABEL maintainers="Huawei CSM development team" +LABEL description="Kubernetes CSM(prometheus) for Huawei Storage" + +ARG binary=./csm-prometheus-collector +COPY ${binary} csm-prometheus-collector +ENTRYPOINT ["/csm-prometheus-collector"] + +FROM busybox:stable-glibc as csm-cmi +LABEL version="${VER}" +LABEL maintainers="Huawei CSM development team" +LABEL description="Kubernetes CSM(cmi) for Huawei Storage" + +ARG binary=./csm-cmi +COPY ${binary} csm-cmi +ENTRYPOINT ["/csm-cmi"] + +FROM busybox:stable-glibc as csm-topo-service +LABEL version="${VER}" +LABEL maintainers="Huawei CSM development team" +LABEL description="Kubernetes CSM(topo) for Huawei Storage" + +ARG binary=./csm-topo-service +COPY ${binary} csm-topo-service +ENTRYPOINT ["/csm-topo-service"] + +FROM busybox:stable-glibc as csm-liveness-probe +LABEL version="${VER}" +LABEL maintainers="Huawei CSM development team" +LABEL description="Kubernetes CSM(livenessprobe) for Huawei Storage" + +ARG binary=./csm-liveness-probe +COPY ${binary} csm-liveness-probe +ENTRYPOINT ["/csm-liveness-probe"] \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..48d2a2e --- /dev/null +++ b/Makefile @@ -0,0 +1,33 @@ +# usage: make -f Makefile VER={VER} PLATFORM={PLATFORM} RELEASE_VER=${RELEASE_VER} + +# (required) [x.y.x] +VER=VER +# (required) [X86 ARM] +PLATFORM=PLATFORM + +export GO111MODULE=on + +Build_Version = github.com/huawei/csm/v2/utils/version.buildVersion +Build_Arch = github.com/huawei/csm/v2/utils/version.buildArch +flag = -ldflags '-w -s -bindnow -X "${Build_Version}=${VER}" -X "${Build_Arch}=${PLATFORM}"' -buildmode=pie + +# Platform [X86, ARM] +ifeq (${PLATFORM}, X86) +env = CGO_ENABLED=0 GOOS=linux GOARCH=amd64 +else +env = CGO_ENABLED=0 GOOS=linux GOARCH=arm64 +endif + +all:PREPARE BUILD + +PREPARE: + rm -rf ./${TMP_DIR_PATH} + mkdir -p ./${TMP_DIR_PATH} + +BUILD: + go mod tidy +# usage: [env] go build [-o output] [flags] packages + ${env} go build -o ${TMP_DIR_PATH}/csm-prometheus-collector ${flag} -buildmode=pie ./cmd/third-party-monitor-server/prometheus-collector + ${env} go build -o ${TMP_DIR_PATH}/csm-cmi ${flag} -buildmode=pie ./cmd/container-monitor-interface/cmi + ${env} go build -o ${TMP_DIR_PATH}/csm-topo-service ${flag} -buildmode=pie ./cmd/storage-monitor-server/topo-service + ${env} go build -o ${TMP_DIR_PATH}/csm-liveness-probe ${flag} -buildmode=pie ./cmd/livenessprobe diff --git a/README.md b/README.md index eef5991..f1c8d43 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,15 @@ -# csm -Huawei Container Storage Monitor +# CSM for Huawei Storage + +![GitHub](https://img.shields.io/github/license/Huawei/csm) +[![Go Report Card](https://goreportcard.com/badge/github.com/huawei/csm)](https://goreportcard.com/report/github.com/huawei/csm) +![GitHub go.mod Go version (subdirectory of monorepo)](https://img.shields.io/github/go-mod/go-version/Huawei/csm) +![GitHub Release Date](https://img.shields.io/github/release-date/Huawei/csm) +![GitHub release (latest by date)](https://img.shields.io/github/downloads/Huawei/csm/latest/total) + +## Description +Container Storage Monitor (CSM) is a tool used for visual display of Huawei storage resources and Kubernetes resources in Kubernetes container scenarios. This tool can notify storage of the relationship between a PV/Pod and a LUN/filesystem so that the relationship can be displayed on the storage for storage administrators to view. It can also upload the performance, capacity, IOPS, and other data of a LUN/file system to a third-party network management system for application administrators to view. In this way, O&M availability in container scenarios can be improved. + +## Documentation +You can click [Release](https://github.com/Huawei/csm/releases) to obtain the released Huawei csm package. + +For details, see the user guide in the docs directory. \ No newline at end of file diff --git a/build.sh b/build.sh new file mode 100644 index 0000000..506600b --- /dev/null +++ b/build.sh @@ -0,0 +1,113 @@ +#!/bin/bash +# +# Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# usage: bash build.sh {VER} {PLATFORM} + +# [x.y.z] +VER=$1 +# [X86 ARM] +PLATFORM=$2 + +set -e +workdir=$(cd $(dirname $0); pwd) + +# tmp dir is used to build binary files and images +export TMP_DIR_PATH="${workdir}/eSDK_CSM_V${VER}_${PLATFORM}_64" +# release dir is used to assemble the release package +release_dir_path="${workdir}/release" + +# init tmp dir and release dir +rm -rf "${TMP_DIR_PATH}" +mkdir -p "${TMP_DIR_PATH}" +rm -rf "${release_dir_path}/image" +mkdir -p "${release_dir_path}/image" + +echo "Start to make with Makefile" +make -f Makefile VER=$1 PLATFORM=$2 + +echo "Platform confirmation" +if [[ "${PLATFORM}" == "ARM" ]];then + PULL_FLAG="--platform=arm64" + BUILD_FLAG="--platform linux/arm64" +elif [[ "${PLATFORM}" == "X86" ]];then + PULL_FLAG="--platform=amd64" + BUILD_FLAG="--platform linux/amd64" +else + echo "Wrong PLATFORM, support [X86, ARM]" + exit +fi + +echo "Start to pull busybox image with architecture" +docker pull ${PULL_FLAG} busybox:stable-glibc + +# build the image +function build_image() { + cp -rf Dockerfile "${TMP_DIR_PATH}"/Dockerfile + + # cd to tmp dir to build image + cd "${TMP_DIR_PATH}" + local images=("csm-prometheus-collector" "csm-topo-service" "csm-cmi" "csm-liveness-probe") + # shellcheck disable=SC2068 + for img in ${images[@]}; do + echo "build the ${img} image" + chmod +x "${img}" + docker build ${BUILD_FLAG} -f Dockerfile -t ${img}:${VER} --target ${img} --build-arg VER=${VER} . + docker save ${img}:${VER} -o ${img}-${VER}.tar + mv ${img}-${VER}.tar ${release_dir_path}/image + done +} +build_image + +# pack the package +echo "pack deploy files" +cp -rf "${workdir}"/helm "${release_dir_path}" + +echo "pack example files" +cp -rf "${workdir}"/example "${release_dir_path}" + +echo "pack manual files" +cp -rf "${workdir}"/manual "${release_dir_path}" +cp -rf "${workdir}"/helm/huawei-csm/crds "${release_dir_path}"/manual/huawei-csm + +# cd to release dir to pack the package +cd "${release_dir_path}" + +# set version in values.yaml and upload-image.sh +sed -i "s/{{version}}/${VER}/g" helm/huawei-csm/values.yaml +sed -i "s/{{version}}/${VER}/g" helm/huawei-csm/upload-image.sh + +# set version in manual templates +sed -i "s/{{version}}/${VER}/g" manual/huawei-csm/templates/csm-prometheus.yaml +sed -i "s/{{version}}/${VER}/g" manual/huawei-csm/templates/csm-storage.yaml + +# parse ${VAR} to Semantic Version style +# Charts https://helm.sh/docs/topics/charts/ +# Semantic Version https://semver.org/lang/zh-CN/` +# example: +# 2.0.0.B070 -> 2.0.0-B070 +# 2.0.0 -> 2.0.0 +chart_version=$(echo ${VER} | sed -e 's/\([0-9]\+\.[0-9]\+\.[0-9]\+\)\./\1-/') +sed -i "s/{{version}}/${chart_version}/g" helm/huawei-csm/Chart.yaml + +# zip the release package and move it to workdir +zip -rq -o eSDK_Huawei_Storage_CSM_V"${VER}"_"${PLATFORM}"_64.zip ./* +mv eSDK_Huawei_Storage_CSM_V"${VER}"_"${PLATFORM}"_64.zip "${workdir}" + +# cd to workdir to remove tmp files +cd "${workdir}" +rm -rf "${TMP_DIR_PATH}" +rm -rf "${release_dir_path}" diff --git a/build/Dockerfile b/build/Dockerfile new file mode 100644 index 0000000..1e2e416 --- /dev/null +++ b/build/Dockerfile @@ -0,0 +1,42 @@ +# eg: docker build --target xxx --platform linux/amd64 --build-arg VER=${VER} -f Dockerfile -t xxx:${VER} . +ARG VER + +FROM busybox:stable-glibc as csm-prometheus-collector +LABEL version="${VER}" +LABEL maintainers="Huawei CSM development team" +LABEL description="Kubernetes CSM(prometheus) for Huawei Storage" + +ARG binary=./csm-prometheus-collector +COPY ${binary} csm-prometheus-collector +ENTRYPOINT ["/csm-prometheus-collector"] + + +# Use distroless as minimal base image to package the manager binary +# Refer to https://github.com/GoogleContainerTools/distroless for more details +FROM busybox:stable-glibc as csm-cmi +LABEL version="${VER}" +LABEL maintainers="Huawei CSM development team" +LABEL description="Kubernetes CSM(cmi) for Huawei Storage" + +ARG binary=./csm-cmi +COPY ${binary} csm-cmi +ENTRYPOINT ["/csm-cmi"] + + +FROM busybox:stable-glibc as csm-topo-service +LABEL version="${VER}" +LABEL maintainers="Huawei CSM development team" +LABEL description="Kubernetes CSM(topo) for Huawei Storage" + +ARG binary=./csm-topo-service +COPY ${binary} csm-topo-service +ENTRYPOINT ["/csm-topo-service"] + +FROM busybox:stable-glibc as csm-liveness-probe +LABEL version="${VER}" +LABEL maintainers="Huawei CSM development team" +LABEL description="Kubernetes CSM(livenessprobe) for Huawei Storage" + +ARG binary=./csm-liveness-probe +COPY ${binary} csm-liveness-probe +ENTRYPOINT ["/csm-liveness-probe"] \ No newline at end of file diff --git a/build/Makefile b/build/Makefile new file mode 100644 index 0000000..904a94a --- /dev/null +++ b/build/Makefile @@ -0,0 +1,36 @@ +# usage: make -f Makefile VER={VER} PLATFORM={PLATFORM} RELEASE_VER=${RELEASE_VER} + +# (required) [x.y.x] +VER=VER +# (required) [X86 ARM] +PLATFORM=PLATFORM +# (Optional) [2.5.RC1 2.5.RC2 ...] eSDK Version +RELEASE_VER=RELEASE_VER + +export GO111MODULE=on +export GOPATH:=$(GOPATH):$(shell pwd) + +Build_Version = github.com/huawei/csm/v2/utils/version.buildVersion +Build_Arch = github.com/huawei/csm/v2/utils/version.buildArch +flag = -ldflags '-w -s -linkmode "external" -extldflags "-Wl,-z,now" -X "${Build_Version}=${VER}" -X "${Build_Arch}=${PLATFORM}"' -buildmode=pie + +# Platform [X86, ARM] +ifeq (${PLATFORM}, X86) +env = CGO_CFLAGS="-fstack-protector-strong -D_FORTIFY_SOURCE=2 -O2" GOOS=linux GOARCH=amd64 +else +env = CGO_CFLAGS="-fstack-protector-strong -D_FORTIFY_SOURCE=2 -O2" GOOS=linux GOARCH=arm64 +endif + +all:PREPARE BUILD + +PREPARE: + rm -rf ./${PACKAGE_NAME} + mkdir -p ./${PACKAGE_NAME} + +BUILD: + go mod tidy +# usage: [env] go build [-o output] [flags] packages + ${env} go build -o ./${PACKAGE_NAME}/csm-prometheus-collector ${flag} -buildmode=pie ../cmd/third-party-monitor-server/prometheus-collector + ${env} go build -o ./${PACKAGE_NAME}/csm-cmi ${flag} -buildmode=pie ../cmd/container-monitor-interface/cmi + ${env} go build -o ./${PACKAGE_NAME}/csm-topo-service ${flag} -buildmode=pie ../cmd/storage-monitor-server/topo-service + ${env} go build -o ./${PACKAGE_NAME}/csm-liveness-probe ${flag} -buildmode=pie ../cmd/livenessprobe diff --git a/build/build.sh b/build/build.sh new file mode 100644 index 0000000..b70669b --- /dev/null +++ b/build/build.sh @@ -0,0 +1,105 @@ +#!/bin/bash +# +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -e +workdir=$(cd $(dirname $0); pwd) + +export PACKAGE_NAME="eSDK_${RELEASE_VER}_CSM_V${VER}_${PLATFORM}_64" +export GOPROXY=http://mirrors.tools.huawei.com/goproxy/ +export GOSUMDB=off +# shellcheck disable=SC2164 +# shellcheck disable=SC2154 +if [ "${isRelease}" == "true" ]; then + echo "buildVersion=${ENV_RELEASE_VERSION}" > buildInfo.properties +else + echo "buildVersion=${RELEASE_VER}.$(date "+%Y%m%d%H%M%S")" > buildInfo.properties +fi + +### step 1: build the binary +# shellcheck disable=SC2164 +cd ${workdir} +make -f Makefile VER="${VER}" PLATFORM="${PLATFORM}" RELEASE_VER="${RELEASE_VER}" + +### step 2: load the image +if [ "${PLATFORM}" == "ARM" ]; then + wget http://10.29.160.97/busybox-arm.tar + docker load -i busybox-arm.tar + docker tag busybox:1.36.1 busybox-arm:stable-glibc + sed -i 's/busybox:stable-glibc/busybox-arm:stable-glibc/g' Dockerfile +else + wget http://10.29.160.97/busybox-x86.tar + docker load -i busybox-x86.tar + docker tag busybox:1.36.1 busybox-x86:stable-glibc + sed -i 's/busybox:stable-glibc/busybox-x86:stable-glibc/g' Dockerfile +fi + +### step 3: build the image +function build_image() { + # shellcheck disable=SC2164 + cp -rf Dockerfile ./"${PACKAGE_NAME}"/Dockerfile + # shellcheck disable=SC2164 + cd ./"${PACKAGE_NAME}"/ + echo "create image dir" + mkdir -p ../release/image/ + # shellcheck disable=SC2054 + local images=("csm-prometheus-collector" "csm-topo-service" "csm-cmi" "csm-liveness-probe") + # shellcheck disable=SC2068 + for img in ${images[@]}; do + echo "build the ${img} image" + chmod +x "${img}" + # shellcheck disable=SC2086 + docker build -f Dockerfile -t "${img}":${VER} --target "${img}" --build-arg VER=${VER} . + docker save "${img}":"${VER}" -o "${img}"-"${VER}".tar + mv "${img}"-"${VER}".tar ../release/image/ + done +} +build_image + +### step 4: pack the package +echo "pack deploy files" +cp -rf ../../helm ../release + +echo "pack example files" +cp -rf ../../example ../release + +echo "pack manual files" +cp -rf ../../manual ../release +cp -rf ../../helm/huawei-csm/crds ../release/manual/huawei-csm + +# shellcheck disable=SC2164 +cd ../release + +# set version in values.yaml and Chart.yaml +sed -i "s/{{version}}/${VER}/g" helm/huawei-csm/values.yaml +sed -i "s/{{version}}/${VER}/g" helm/huawei-csm/upload-image.sh + +# set version in manual templates +sed -i "s/{{version}}/${VER}/g" manual/huawei-csm/templates/csm-prometheus.yaml +sed -i "s/{{version}}/${VER}/g" manual/huawei-csm/templates/csm-storage.yaml + +# parse ${VAR} to Semantic Version style +# Charts https://helm.sh/docs/topics/charts/ +# Semantic Version https://semver.org/lang/zh-CN/` +# example: +# 2.0.0.B070 -> 2.0.0-B070 +# 2.0.0 -> 2.0.0 +chart_version=$(echo ${VER} | sed -e 's/\([0-9]\+\.[0-9]\+\.[0-9]\+\)\./\1-/') +sed -i "s/{{version}}/${chart_version}/g" helm/huawei-csm/Chart.yaml + +# shellcheck disable=SC2035 +zip -rq -o eSDK_Enterprise_Storage_"${RELEASE_VER}"_CSM_V"${VER}"_"${PLATFORM}"_64.zip * +mkdir ${workdir}/../../output +cp eSDK_Enterprise_Storage_"${RELEASE_VER}"_CSM_V"${VER}"_"${PLATFORM}"_64.zip ${workdir}/../../output diff --git a/client/apis/xuanwu/v1/doc.go b/client/apis/xuanwu/v1/doc.go new file mode 100644 index 0000000..e0f4991 --- /dev/null +++ b/client/apis/xuanwu/v1/doc.go @@ -0,0 +1,17 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +// +k8s:deepcopy-gen=package +// +groupName=xuanwu.huawei.io +// Package v1 is v1 version of the API +package v1 diff --git a/client/apis/xuanwu/v1/register.go b/client/apis/xuanwu/v1/register.go new file mode 100644 index 0000000..deb9022 --- /dev/null +++ b/client/apis/xuanwu/v1/register.go @@ -0,0 +1,55 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package v1 contains API Schema definitions for the xuanwu v1 API group +// +kubebuilder:object:generate=true +// +groupName=xuanwu.huawei.io +package v1 + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// GroupName is the group name use in this package. +const GroupName = "xuanwu.huawei.io" + +var ( + // schemeBuilder is the new scheme builder + schemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = schemeBuilder.AddToScheme + // SchemeGroupVersion is group version used to register these objects. + SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1"} +) + +// Resource takes an unqualified resource and returns a Group qualified GroupResource +func Resource(resource string) schema.GroupResource { + return SchemeGroupVersion.WithResource(resource).GroupResource() +} + +func init() { + schemeBuilder.Register(addKnownTypes) +} + +// addKnownTypes adds the set of types defined in this package to the supplied scheme. +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &ResourceTopology{}, + &ResourceTopologyList{}, + ) + v1.AddToGroupVersion(scheme, SchemeGroupVersion) + return nil +} diff --git a/client/apis/xuanwu/v1/resourcetopology.go b/client/apis/xuanwu/v1/resourcetopology.go new file mode 100644 index 0000000..c6037d7 --- /dev/null +++ b/client/apis/xuanwu/v1/resourcetopology.go @@ -0,0 +1,100 @@ +/* +Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + +// ResourceTopologyStatusPhase defines the ResourceTopologyStatusPhase type +type ResourceTopologyStatusPhase string + +const ( + // ResourceTopologyStatusNormal indicates that the resource is normal + ResourceTopologyStatusNormal ResourceTopologyStatusPhase = "Normal" + // ResourceTopologyStatusPending indicates that the resource is pending + ResourceTopologyStatusPending ResourceTopologyStatusPhase = "Pending" + // ResourceTopologyStatusDeleting indicates that the resource is deleting + ResourceTopologyStatusDeleting ResourceTopologyStatusPhase = "Deleting" + // ResourceTopologyStatusCrash indicates that the resource is deleting + ResourceTopologyStatusCrash ResourceTopologyStatusPhase = "Crash" +) + +// ResourceTopologySpec defines the fields in Spec +type ResourceTopologySpec struct { + // Provisioner is the volume provisioner name + // +kubebuilder:validation:Required + Provisioner string `json:"provisioner" protobuf:"bytes,2,name=provisioner"` + + // VolumeHandle is the backend name and identity of the volume, format as . + // +kubebuilder:validation:Required + VolumeHandle string `json:"volumeHandle" protobuf:"bytes,2,name=volumeHandle"` + + // Tags defines pv and other relationships and ownership + // +kubebuilder:validation:Required + Tags []Tag `json:"tags" protobuf:"bytes,2,name=tags"` +} + +// ResourceTopologyStatus status of resource topology +type ResourceTopologyStatus struct { + // Status is the status of the ResourceTopology + Status ResourceTopologyStatusPhase `json:"status,omitempty" protobuf:"bytes,2,opt,name=status"` + + // Tags defines pv and other relationships and ownership + Tags []Tag `json:"tags,omitempty" protobuf:"bytes,3,opt,name=tags"` +} + +// Tag defines pv and other relationships and ownership +type Tag struct { + ResourceInfo `json:",inline"` + + // Owner defines who does the resource belongs to + // +kubebuilder:validation:Optional + Owner ResourceInfo `json:"owner" protobuf:"bytes,2,name=owner"` +} + +// ResourceInfo define resource information +type ResourceInfo struct { + metaV1.TypeMeta `json:",inline"` + // NameSpace is the namespace of the resource + Namespace string `json:"namespace,omitempty" protobuf:"bytes,2,opt,name=target"` + // Name is the name of the resource + Name string `json:"name,omitempty" protobuf:"bytes,2,opt,name=name"` +} + +// ResourceTopology is the Schema for the ResourceTopologys API +// +genclient +// +genclient:nonNamespaced +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster,shortName="rt" +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Provisioner",type=string,JSONPath=`.spec.provisioner` +// +kubebuilder:printcolumn:name="VolumeHandle",type=string,JSONPath=`.spec.volumeHandle` +// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.status` +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` +type ResourceTopology struct { + metaV1.TypeMeta `json:",inline"` + metaV1.ObjectMeta `json:"metadata,omitempty"` + Spec ResourceTopologySpec `json:"spec,omitempty"` + Status ResourceTopologyStatus `json:"status,omitempty"` +} + +// ResourceTopologyList contains a list of ResourceTopology +// +kubebuilder:object:root=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type ResourceTopologyList struct { + metaV1.TypeMeta `json:",inline"` + metaV1.ListMeta `json:"metadata,omitempty"` + Items []ResourceTopology `json:"items"` +} diff --git a/client/apis/xuanwu/v1/zz_generated.deepcopy.go b/client/apis/xuanwu/v1/zz_generated.deepcopy.go new file mode 100644 index 0000000..76c69d6 --- /dev/null +++ b/client/apis/xuanwu/v1/zz_generated.deepcopy.go @@ -0,0 +1,161 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceInfo) DeepCopyInto(out *ResourceInfo) { + *out = *in + out.TypeMeta = in.TypeMeta + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceInfo. +func (in *ResourceInfo) DeepCopy() *ResourceInfo { + if in == nil { + return nil + } + out := new(ResourceInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceTopology) DeepCopyInto(out *ResourceTopology) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceTopology. +func (in *ResourceTopology) DeepCopy() *ResourceTopology { + if in == nil { + return nil + } + out := new(ResourceTopology) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ResourceTopology) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceTopologyList) DeepCopyInto(out *ResourceTopologyList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ResourceTopology, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceTopologyList. +func (in *ResourceTopologyList) DeepCopy() *ResourceTopologyList { + if in == nil { + return nil + } + out := new(ResourceTopologyList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ResourceTopologyList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceTopologySpec) DeepCopyInto(out *ResourceTopologySpec) { + *out = *in + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make([]Tag, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceTopologySpec. +func (in *ResourceTopologySpec) DeepCopy() *ResourceTopologySpec { + if in == nil { + return nil + } + out := new(ResourceTopologySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceTopologyStatus) DeepCopyInto(out *ResourceTopologyStatus) { + *out = *in + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make([]Tag, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceTopologyStatus. +func (in *ResourceTopologyStatus) DeepCopy() *ResourceTopologyStatus { + if in == nil { + return nil + } + out := new(ResourceTopologyStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Tag) DeepCopyInto(out *Tag) { + *out = *in + out.ResourceInfo = in.ResourceInfo + out.Owner = in.Owner + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Tag. +func (in *Tag) DeepCopy() *Tag { + if in == nil { + return nil + } + out := new(Tag) + in.DeepCopyInto(out) + return out +} diff --git a/client/hack/README.md b/client/hack/README.md new file mode 100644 index 0000000..e69de29 diff --git a/client/hack/boilerplate.go.txt b/client/hack/boilerplate.go.txt new file mode 100644 index 0000000..d8de972 --- /dev/null +++ b/client/hack/boilerplate.go.txt @@ -0,0 +1,13 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ \ No newline at end of file diff --git a/client/hack/tools.go b/client/hack/tools.go new file mode 100644 index 0000000..28a84ca --- /dev/null +++ b/client/hack/tools.go @@ -0,0 +1,21 @@ +//go:build tools +// +build tools + +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package hack is used for code generation +package hack + +import _ "k8s.io/code-generator" diff --git a/client/hack/update-codegen.sh b/client/hack/update-codegen.sh new file mode 100644 index 0000000..197df80 --- /dev/null +++ b/client/hack/update-codegen.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +# Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY GROUP, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +# generate-groups.sh parameter introduction +# the generators comma separated to run (deepcopy,defaulter,client,lister,informer) or "all". +# the output package name (e.g. github.com/example/project/pkg/generated). +# the external types dir (e.g. github.com/example/apis or github.com/example/project/pkg/apis). +# the groups and their versions in the format "groupA:v1,v2 groupB:v1 groupC:v2", relative +# to . + +# parameters that need to be modified +# go mod name +MODULE=huawei-csm +# crd group name +GROUP=xuanwu +# code folder location, modify according to your project location +OUTPUT_BASE=/root/xuanwu/huawei-csm + +# clear the old file before executing script +if [ -d "${OUTPUT_BASE}/${MODULE}" ]; + then + rm -rf ${OUTPUT_BASE}/${MODULE} + echo "Delete the old generated file ${OUTPUT_BASE}/${MODULE}" + else + echo "Generate file ${OUTPUT_BASE}/${MODULE} by code-generator" +fi + +# execute script +# make sure the directory structure is correct +bash ./vendor/k8s.io/code-generator/generate-groups.sh \ + "deepcopy,client,informer,lister" \ + ${MODULE}/pkg/client \ + ${MODULE}/client/apis \ + ${GROUP}:v1 \ + --go-header-file ./client/hack/boilerplate.go.txt \ + --output-base ${OUTPUT_BASE} + +# copy the newly generated zz_generated.deepcopy.go to the source directory, +# or replace the old zz_generated.deepcopy.go if the old one exists +cp ${OUTPUT_BASE}/${MODULE}/client/apis/${GROUP}/v1/zz_generated.deepcopy.go ${OUTPUT_BASE}/client/apis/${GROUP}/v1 +echo "Copy the newly generated zz_generated.deepcopy.go from ./${MODULE}/client/apis/${GROUP}/v1 to the ./client/apis/${GROUP}/v1" + +# delete the temporary directory of the newly generated /client/apis/ file +rm -rf ${OUTPUT_BASE}/${MODULE}/client/apis/ + +# delete the old listers,informers and clientset before copy new ons to the ./client/ +files=("clientset" "informers" "listers") +for file in ${files[@]} +do + if [ -d "${OUTPUT_BASE}/pkg/client/${file}" ]; + then + rm -rf ${OUTPUT_BASE}/pkg/client/${file} + echo "Delete old file ${OUTPUT_BASE}/client/${file}" + fi +done + +# copy the generated lister informer clientset +cp -rf ${OUTPUT_BASE}/${MODULE}/pkg/client/* ${OUTPUT_BASE}/pkg/client/ +echo "Copy the newly generated listers,informers and clientset from ./${MODULE}/pkg/client/ to ./pkg/client/" + +# delete the newly generated file +rm -rf ${OUTPUT_BASE}/github.com/ +echo "Delete the newly generated directory ${OUTPUT_BASE}/github.com/" diff --git a/client/hack/update-crd.sh b/client/hack/update-crd.sh new file mode 100644 index 0000000..f5ada1b --- /dev/null +++ b/client/hack/update-crd.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/.. && pwd) + +# find or download controller-gen +CONTROLLER_GEN=$(which controller-gen) + +if [ "$CONTROLLER_GEN" = "" ] +then + TMP_DIR=$(mktemp -d) + cd $TMP_DIR + go mod init tmp + go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.8.0 + CONTROLLER_GEN=$(which controller-gen) +fi + +if [ "$CONTROLLER_GEN" = "" ] +then + echo "ERROR: failed to get controller-gen"; + exit 1; +fi + +MODULE=huawei-csm +$CONTROLLER_GEN paths=${MODULE}/client/apis/xuanwu/v1 crd:crdVersions=v1 output:crd:artifacts:config=deploy/crd diff --git a/cmd/container-monitor-interface/cmi/main.go b/cmd/container-monitor-interface/cmi/main.go new file mode 100644 index 0000000..678785d --- /dev/null +++ b/cmd/container-monitor-interface/cmi/main.go @@ -0,0 +1,139 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package main is the process entry +package main + +import ( + "fmt" + "net" + "os" + "syscall" + + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "google.golang.org/grpc" + + "github.com/huawei/csm/v2/config" + cmiConfig "github.com/huawei/csm/v2/config/cmi" + logConfig "github.com/huawei/csm/v2/config/log" + "github.com/huawei/csm/v2/grpc/lib/go/cmi" + "github.com/huawei/csm/v2/provider/collect" + grpchelper "github.com/huawei/csm/v2/provider/grpc/helper" + "github.com/huawei/csm/v2/provider/grpc/server" + "github.com/huawei/csm/v2/provider/utils" + "github.com/huawei/csm/v2/utils/log" + "github.com/huawei/csm/v2/utils/version" +) + +const ( + containerName = "cmi-controller" + namespaceEnv = "NAMESPACE" + defaultNamespace = "huawei-csm" + versionCmName = "huawei-csm-version" +) + +var cmiService = &cobra.Command{ + Use: "cmi", + Long: `container monitor interface`, +} + +func main() { + manager := config.NewOptionManager(cmiService.Flags(), logConfig.Option, cmiConfig.Option) + manager.AddFlags() + + cmiService.Run = func(cmd *cobra.Command, args []string) { + err := manager.ValidateConfig() + if err != nil { + log.Errorf("validate config failed, error: %v", err) + return + } + + err = log.InitLogging(logConfig.GetLogFile()) + if err != nil { + logrus.Errorf("init log config err: %v", err) + return + } + + err = version.InitVersionConfigMapWithName(containerName, + version.ContainerMonitorInterfaceVersion, namespaceEnv, defaultNamespace, versionCmName) + if err != nil { + log.Errorf("init version file error: [%v]", err) + return + } + + err = grpchelper.InitClientSet() + if err != nil { + log.Errorf("init client set failed, error: %v", err) + return + } + + stopCh := make(chan struct{}) + defer close(stopCh) + startBackendWatcher(stopCh) + err = StartGrpcServer(cmiConfig.GetCmiAddress()) + if err != nil { + log.Errorf("start grpc server failed, error: %v", err) + return + } + } + + if err := cmiService.Execute(); err != nil { + log.Errorf("Start cmi server failed, error: %v", err) + return + } +} + +// StartGrpcServer start grpc server +func StartGrpcServer(address string) error { + log.Infoln("Starting cmi server") + opts := []grpc.ServerOption{ + grpc.UnaryInterceptor(log.EnsureGRPCContext), + } + grpcServer := grpc.NewServer(opts...) + + cmi.RegisterIdentityServer(grpcServer, &server.Identity{}) + cmi.RegisterLabelServiceServer(grpcServer, &server.Label{}) + cmi.RegisterCollectorServer(grpcServer, &server.Collector{}) + + if err := utils.CleanupSocketFile(address); err != nil { + return fmt.Errorf("cleanup unix socket failed, error: %v", err) + } + + lis, err := net.Listen("unix", address) + if err != nil { + return fmt.Errorf("listen unix socket failed, error: %v", err) + } + + signalChan := make(chan os.Signal, 1) + go func() { + if err = grpcServer.Serve(lis); err != nil { + log.Errorf("cmi server stopped serving, error: %v", err) + signalChan <- syscall.SIGINT + } + }() + + defer func() { + grpcServer.GracefulStop() + }() + + // terminate grpc server gracefully before leaving main function + <-signalChan + + return nil +} + +func startBackendWatcher(stopCh chan struct{}) { + go collect.RunBackendInformer(stopCh) +} diff --git a/cmd/livenessprobe/main.go b/cmd/livenessprobe/main.go new file mode 100644 index 0000000..8fca017 --- /dev/null +++ b/cmd/livenessprobe/main.go @@ -0,0 +1,139 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package main is the process entry +package main + +import ( + "context" + "flag" + "net" + "net/http" + "time" + + "github.com/spf13/cobra" + + "github.com/huawei/csm/v2/grpc/lib/go/cmi" + "github.com/huawei/csm/v2/utils/log" + "github.com/huawei/csm/v2/utils/version" +) + +const ( + containerName = "liveness-probe" + namespaceEnv = "NAMESPACE" + defaultNamespace = "huawei-csm" + defaultIpAddress = "0.0.0.0" + defaultHealthzPort = "9808" + defaultProbeTimeout = 10 * time.Second + defaultCmiAddress = "/cmi/cmi.sock" + defaultLogFile = "liveness-probe" + versionCmName = "huawei-csm-version" + + healthz = "/healthz" +) + +// Command line flags +var ( + probeTimeout time.Duration + cmiAddress string + ipAddress string + healthzPort string + logFile string + + livenessprobe = &cobra.Command{ + Use: "livenessprobe", + Long: `liveness probe for CSM services`, + } +) + +type healthProbe struct { + client *cmi.ClientSet +} + +func main() { + parseFlags() + + livenessprobe.Run = func(cmd *cobra.Command, args []string) { + // Init the logging + err := log.InitLogging(logFile) + if err != nil { + log.Errorf("init log error: [%v]", err) + return + } + + err = version.InitVersionConfigMapWithName(containerName, + version.CsmLivenessProbeVersion, namespaceEnv, defaultNamespace, versionCmName) + if err != nil { + log.Errorf("init version file error: [%v]", err) + return + } + + clientSet, err := cmi.GetClientSet(cmiAddress) + if err != nil { + log.Errorf("get cmi client set failed: [%v]", err) + return + } + defer clientSet.Conn.Close() + + mux := http.NewServeMux() + + hp := healthProbe{client: clientSet} + mux.HandleFunc(healthz, hp.probe) + + addr := net.JoinHostPort(ipAddress, healthzPort) + log.Infof("serveMux listening at [%s]", addr) + err = http.ListenAndServe(addr, mux) + if err != nil { + log.Errorf("failed to start http server with error: [%v]", err) + return + } + } + + if err := livenessprobe.Execute(); err != nil { + log.Errorf("start liveness probe server failed, error: [%v]", err) + return + } +} + +func (hp *healthProbe) probe(w http.ResponseWriter, req *http.Request) { + ctx, cancel := context.WithTimeout(req.Context(), probeTimeout) + defer cancel() + log.AddContext(ctx).Infoln("start to probe cmi service") + + _, err := hp.client.IdentityClient.Probe(ctx, &cmi.ProbeRequest{}) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + log.Errorf("probe cmi service failed: [%v]", err) + return + } + + w.WriteHeader(http.StatusOK) + log.AddContext(ctx).Infoln("probe cmi service succeeded") +} + +func parseFlags() { + livenessprobe.Flags().AddGoFlagSet(flag.CommandLine) + livenessprobe.Flags().DurationVar(&probeTimeout, "probe-timeout", defaultProbeTimeout, + "Probe timeout in seconds.") + livenessprobe.Flags().StringVar(&cmiAddress, "cmi-address", defaultCmiAddress, + "Address of the CMI driver socket.") + livenessprobe.Flags().StringVar(&ipAddress, "ip-address", defaultIpAddress, + "The listening ip address in the container.") + livenessprobe.Flags().StringVar(&healthzPort, "healthz-port", defaultHealthzPort, + "TCP ports for listening healthz requests.") + livenessprobe.Flags().StringVar(&logFile, "log-file", defaultLogFile, + "The log file name of the liveness probe") +} diff --git a/cmd/storage-monitor-server/topo-service/main.go b/cmd/storage-monitor-server/topo-service/main.go new file mode 100644 index 0000000..20b1e6e --- /dev/null +++ b/cmd/storage-monitor-server/topo-service/main.go @@ -0,0 +1,166 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package main is the process entry +package main + +import ( + "context" + "os" + "syscall" + + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + k8sInformers "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/scheme" + + "github.com/huawei/csm/v2/config" + clientConfig "github.com/huawei/csm/v2/config/client" + leaderElectionConfig "github.com/huawei/csm/v2/config/leaderelection" + logConfig "github.com/huawei/csm/v2/config/log" + controllerConfig "github.com/huawei/csm/v2/config/topology" + leaderElection "github.com/huawei/csm/v2/controller/leaderelection" + "github.com/huawei/csm/v2/controller/resourcetopology" + "github.com/huawei/csm/v2/controller/utils" + csmScheme "github.com/huawei/csm/v2/pkg/client/clientset/versioned/scheme" + informers "github.com/huawei/csm/v2/pkg/client/informers/externalversions" + "github.com/huawei/csm/v2/utils/log" + "github.com/huawei/csm/v2/utils/version" +) + +const ( + defaultNamespace = "huawei-csm" + containerName = "topo-service" + leaderLockObjectName = "resource-topology" + namespaceEnv = "NAMESPACE" + versionCmName = "huawei-csm-version" +) + +var topoService = &cobra.Command{ + Use: "controller", + Long: `resource topology controller`, +} + +func main() { + manager := config.NewOptionManager(topoService.Flags(), + logConfig.Option, controllerConfig.Option, leaderElectionConfig.Option, clientConfig.Option) + manager.AddFlags() + + topoService.Run = func(cmd *cobra.Command, args []string) { + err := manager.ValidateConfig() + if err != nil { + logrus.Errorf("validate config err: [%v]", err) + return + } + + err = log.InitLogging(logConfig.GetLogFile()) + if err != nil { + logrus.Errorf("init log config err: [%v]", err) + return + } + + err = version.InitVersionConfigMapWithName(containerName, + version.CsmTopoServiceVersion, namespaceEnv, defaultNamespace, versionCmName) + if err != nil { + log.Errorf("init version file error: [%v]", err) + return + } + + clientsSet, err := utils.NewClientsSet(clientConfig.GetKubeConfig(), controllerConfig.GetCmiAddress()) + if err != nil { + log.Errorf("new client set error: [%v]", err) + return + } + + ctx := context.WithValue(context.Background(), "controller", "resourceTopologyController") + + signalChan := make(chan os.Signal, 1) + defer close(signalChan) + + startController(ctx, clientsSet, signalChan) + + err = waitTopoServiceStop(ctx, signalChan) + if err != nil { + log.Errorf("wait topo service stop error: [%v]", err) + return + } + } + + if err := topoService.Execute(); err != nil { + log.Errorf("server meet err: [%v], exit", err) + return + } +} + +func startController(ctx context.Context, clientSets *utils.ClientsSet, ch chan os.Signal) { + if leaderElectionConfig.EnableLeaderElection() { + go leaderElection.Run(ctx, clientSets, newLeaderElectionParams(), runController, ch) + } else { + log.AddContext(ctx).Infoln("start resourceTopology controller without leader election") + go runController(ctx, clientSets, ch) + } +} + +func waitTopoServiceStop(ctx context.Context, signalChan chan os.Signal) error { + // Stop the main when stop signals are received + utils.WaitSignal(ctx, signalChan) + return nil +} + +func newLeaderElectionParams() *leaderElection.Params { + return leaderElection.NewParams(). + SetDefaultNamespace(leaderElectionConfig.GetLeaderLockNamespace()). + SetLockName(leaderLockObjectName). + SetLeaderLeaseDuration(leaderElectionConfig.GetLeaderLeaseDuration()). + SetLeaderRenewDeadline(leaderElectionConfig.GetLeaderRenewDeadline()). + SetLeaderRetryPeriod(leaderElectionConfig.GetLeaderRetryPeriod()) +} + +func runController(ctx context.Context, clients *utils.ClientsSet, ch chan os.Signal) { + factory := informers.NewSharedInformerFactory(clients.XuanwuClient, controllerConfig.GetResyncPeriod()) + k8sFactory := k8sInformers.NewSharedInformerFactory(clients.KubeClient, 0) + // Add ResourceTopology types to the default Kubernetes so events can be logged for them + if err := csmScheme.AddToScheme(scheme.Scheme); err != nil { + log.AddContext(ctx).Errorf("add to scheme error: %v", err) + ch <- syscall.SIGINT + return + } + + ctrl := resourcetopology.NewController(resourcetopology.ControllerRequest{ + KubeClient: clients.KubeClient, + XuanwuClient: clients.XuanwuClient, + TopologyInformer: factory.Xuanwu().V1().ResourceTopologies(), + VolumeInformer: k8sFactory.Core().V1().PersistentVolumes(), + ClaimInformer: k8sFactory.Core().V1().PersistentVolumeClaims(), + PodInformer: k8sFactory.Core().V1().Pods(), + ReSyncPeriod: controllerConfig.GetResyncPeriod(), + EventRecorder: clients.EventRecorder, + CmiClient: clients.CmiClient, + }) + + run := func(ctx context.Context) { + // Run the controller process + stopCh := make(chan struct{}) + factory.Start(stopCh) + k8sFactory.Start(stopCh) + go ctrl.Run(ctx, controllerConfig.GetControllerWorkers(), stopCh) + + // Stop the controller until get signal + utils.WaitExitSignal(ctx, "controller") + + close(stopCh) + } + + run(ctx) +} diff --git a/cmd/third-party-monitor-server/prometheus-collector/main.go b/cmd/third-party-monitor-server/prometheus-collector/main.go new file mode 100644 index 0000000..1931de5 --- /dev/null +++ b/cmd/third-party-monitor-server/prometheus-collector/main.go @@ -0,0 +1,129 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package main is the process entry +package main + +import ( + "crypto/tls" + "fmt" + "net/http" + + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "github.com/huawei/csm/v2/config" + exporterConfig "github.com/huawei/csm/v2/config/exporter" + logConfig "github.com/huawei/csm/v2/config/log" + clientSet "github.com/huawei/csm/v2/server/prometheus-exporter/clientset" + exporterHandler "github.com/huawei/csm/v2/server/prometheus-exporter/exporterhandler" + "github.com/huawei/csm/v2/utils/log" + "github.com/huawei/csm/v2/utils/version" +) + +const ( + containerName = "prometheus-collector" + namespaceEnv = "NAMESPACE" + defaultNamespace = "huawei-csm" + versionCmName = "huawei-csm-version" + + healthz = "/healthz" + + httpsCert = "/etc/secret-volume/tls.crt" + httpsKey = "/etc/secret-volume/tls.key" +) + +var prometheusExporter = &cobra.Command{ + Use: "exporter", + Long: `prometheus exporter`, +} + +func main() { + manager := config.NewOptionManager(prometheusExporter.Flags(), logConfig.Option, exporterConfig.Option) + manager.AddFlags() + + prometheusExporter.Run = func(cmd *cobra.Command, args []string) { + err := manager.ValidateConfig() + if err != nil { + // ValidateConfig error the log ValidateConfig will print + logrus.Errorf("validate config err: [%v]", err) + return + } + + err = verifyStartInfo() + if err != nil { + // error log in verifyStartInfo + logrus.Errorf("verify start info err: [%v]", err) + return + } + + err = initListener() + if err != nil { + // error log in initListener + log.Errorf("init listener err: [%v]", err) + return + } + } + + if err := prometheusExporter.Execute(); err != nil { + log.Errorf("server meet err: [%v], exit", err) + return + } +} + +func verifyStartInfo() error { + err := log.InitLogging(logConfig.GetLogFile()) + if err != nil { + logrus.Errorf("init log config err: [%v]", err) + return err + } + + err = version.InitVersionConfigMapWithName(containerName, + version.CsmPrometheusCollectorVersion, namespaceEnv, defaultNamespace, versionCmName) + if err != nil { + log.Errorf("init version file error: [%v]", err) + return err + } + return nil +} + +func initListener() error { + client := clientSet.InitExporterClientSet(exporterConfig.GetStorageGRPCSock()) + if client.InitError != nil { + clientSet.DeleteExporterClientSet() + return fmt.Errorf("init exporter client set err: [%v]", client.InitError) + } + + var err error + http.HandleFunc("/", exporterHandler.MetricsHandler) + http.HandleFunc(healthz, exporterHandler.HealthHandler) + if exporterConfig.GetUseHttps() { + server := &http.Server{ + Addr: exporterConfig.GetIpAddress() + ":" + exporterConfig.GetExporterPort(), + Handler: nil, + TLSConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + }, + } + err = server.ListenAndServeTLS(httpsCert, httpsKey) + } else { + err = http.ListenAndServe(exporterConfig.GetIpAddress()+":"+exporterConfig.GetExporterPort(), nil) + } + + if err != nil { + clientSet.DeleteExporterClientSet() + return fmt.Errorf("start service error: %v", err) + } + return nil +} diff --git a/config/client/client_config.go b/config/client/client_config.go new file mode 100644 index 0000000..5209ab5 --- /dev/null +++ b/config/client/client_config.go @@ -0,0 +1,62 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package client is used to init client configurations and flags +package client + +import ( + "errors" + "path/filepath" + + "github.com/spf13/pflag" + + "github.com/huawei/csm/v2/config/consts" +) + +const ( + clientOptionName = "ClientOption" +) + +// Option is a client option instance for manager init +var Option = &option{} + +type option struct { + kubeConfig string +} + +// GetName return name string of client option +func (o *option) GetName() string { + return clientOptionName +} + +// AddFlags is to add flags for client configurations +func (o *option) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&o.kubeConfig, consts.KubeConfig, "", "The absolute path to the kubeConfig file.") +} + +// ValidateConfig is to validate input client configurations +func (o *option) ValidateConfig() error { + if o.kubeConfig == "" { + return nil + } + if !filepath.IsAbs(o.kubeConfig) { + return errors.New("kubeConfig file path is not absolute") + } + return nil +} + +// GetKubeConfig returns the kube config file path +func GetKubeConfig() string { + return Option.kubeConfig +} diff --git a/config/client/client_config_test.go b/config/client/client_config_test.go new file mode 100644 index 0000000..ea41dd1 --- /dev/null +++ b/config/client/client_config_test.go @@ -0,0 +1,82 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package client +package client + +import ( + "errors" + "path/filepath" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" +) + +func Test_option_ValidateConfig_Success(t *testing.T) { + // arrange + o := &option{ + kubeConfig: "fakeConfig", + } + + // mock + mock := gomonkey.NewPatches() + + // expect + mock.ApplyFunc(filepath.IsAbs, func(path string) bool { + return true + }) + + // act + err := o.ValidateConfig() + + // assert + if err != nil { + t.Errorf("Test_option_ValidateConfig_Success failed: [%v]", err) + } + + // clean + t.Cleanup(func() { + mock.Reset() + }) +} + +func Test_option_ValidateConfig_Fail(t *testing.T) { + // arrange + o := &option{ + kubeConfig: "fakeConfig", + } + want := errors.New("kubeConfig file path is not absolute") + + // mock + mock := gomonkey.NewPatches() + + // expect + mock.ApplyFunc(filepath.IsAbs, func(path string) bool { + return false + }) + + // act + got := o.ValidateConfig() + + // assert + if !reflect.DeepEqual(got, want) { + t.Errorf("Test_option_ValidateConfig_Fail: want [%v], got [%v]", want, got) + } + + // clean + t.Cleanup(func() { + mock.Reset() + }) +} diff --git a/config/cmi/provider_config.go b/config/cmi/provider_config.go new file mode 100644 index 0000000..6fb8740 --- /dev/null +++ b/config/cmi/provider_config.go @@ -0,0 +1,95 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package cmi defines config of cmi service +package cmi + +import ( + "github.com/spf13/pflag" +) + +const ( + defaultQueryPageSize = 100 + defaultClientMaxThreads = 20 + defaultProviderName = "cmi.huawei.com" + defaultProviderOptionName = "providerOptionName" + defaultCmiAddress = "/cmi/cmi.sock" + defaultNamespace = "huawei-csi" +) + +// Option contains provider option args +var Option = NewProviderOption() + +type providerOption struct { + queryStoragePageSize int + clientMaxThreads int + providerName string + cmiAddress string + backendNamespace string +} + +// GetName return option name +func (p *providerOption) GetName() string { + return defaultProviderOptionName +} + +// AddFlags add flags +func (p *providerOption) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&p.providerName, "cmi-name", defaultProviderName, "Name of provider") + fs.StringVar(&p.cmiAddress, "cmi-address", defaultCmiAddress, "Path to cmi socket") + fs.IntVar(&p.queryStoragePageSize, "page-size", defaultQueryPageSize, "Max size of query storage") + fs.StringVar(&p.backendNamespace, "backend-namespace", defaultNamespace, "Namespace of backend") + fs.IntVar(&p.clientMaxThreads, "client-max-threads", defaultClientMaxThreads, "Max client threads") +} + +// ValidateConfig validate config +func (p *providerOption) ValidateConfig() error { + return nil +} + +// NewProviderOption init an instance of ProviderOption +func NewProviderOption() *providerOption { + return &providerOption{ + queryStoragePageSize: defaultQueryPageSize, + providerName: defaultProviderName, + cmiAddress: defaultCmiAddress, + } +} + +// GetProviderName get provider name +func GetProviderName() string { + return Option.providerName +} + +// GetCmiAddress get cmi address +func GetCmiAddress() string { + return Option.cmiAddress +} + +// GetNamespace get namespace +func GetNamespace() string { + return Option.backendNamespace +} + +// GetQueryStoragePageSize get query storage page size +func GetQueryStoragePageSize() int { + return Option.queryStoragePageSize +} + +// GetClientMaxThreads get client max threads +func GetClientMaxThreads() int { + return Option.clientMaxThreads +} diff --git a/config/consts/config_key.go b/config/consts/config_key.go new file mode 100644 index 0000000..e524f38 --- /dev/null +++ b/config/consts/config_key.go @@ -0,0 +1,83 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package consts contains all the keys of configuration +package consts + +const ( + // LogFile key name of log file config + LogFile = "log-file" + // LoggingModule key name of log mod config + LoggingModule = "logging-module" + // LogLevel key name of log level config + LogLevel = "logging-level" + // LogFileDir key name of log file dir config + LogFileDir = "log-file-dir" +) + +const ( + // SupportResources key name of support resources config + SupportResources = "support-resources" + // ControllerWorkers key name of controller works count config + ControllerWorkers = "controller-workers" + // RtRetryBaseDelay key name of base rt controller retry delay config + RtRetryBaseDelay = "rt-retry-base-delay" + // PvRetryBaseDelay key name of base pv controller retry delay config + PvRetryBaseDelay = "pv-retry-base-delay" + // PodRetryBaseDelay key name of base pod controller retry delay config + PodRetryBaseDelay = "pod-retry-base-delay" + // RtRetryMaxDelay key name of rt controller max retry delay config + RtRetryMaxDelay = "rt-retry-max-delay" + // PvRetryMaxDelay key name of pv controller max retry delay config + PvRetryMaxDelay = "pv-retry-max-delay" + // PodRetryMaxDelay key name of pod controller max retry delay config + PodRetryMaxDelay = "pod-retry-max-delay" + // ResyncPeriod key name of reSync interval of the controller + ResyncPeriod = "resync-period" + // CmiAddress key name of cmi endpoint address + CmiAddress = "cmi-address" +) + +const ( + // KubeConfig key name of kube config file path config + KubeConfig = "kube-config" +) + +const ( + // IpAddress the listening ip address of the prometheus + IpAddress = "ip-address" + // ExporterPort prometheus exporter key name of kube config file path config + ExporterPort = "exporter-port" + // StorageGRPCSock the path of the grpc sock file + StorageGRPCSock = "cmi-address" + // StorageBackendNamespace the namespace of the sbc + StorageBackendNamespace = "storage-backend-namespace" + // CSIDriverName the name of the csi driver + CSIDriverName = "csi-driver-name" + // UseHttps the option to use https or not + UseHttps = "use-https" +) + +const ( + // EnableLeaderElection key name of controller leader election switch config + EnableLeaderElection = "enable-leader-election" + // LeaderLockNamespace key name of controller leader election lock ns config + LeaderLockNamespace = "leader-lock-namespace" + // LeaderLeaseDuration key name of controller leader election lease duration config + LeaderLeaseDuration = "leader-lease-duration" + // LeaderRenewDeadline key name of controller leader election renew deadline config + LeaderRenewDeadline = "leader-renew-deadline" + // LeaderRetryPeriod key name of controller leader election retry period config + LeaderRetryPeriod = "leader-retry-period" +) diff --git a/config/exporter/exporter_config.go b/config/exporter/exporter_config.go new file mode 100644 index 0000000..82a8707 --- /dev/null +++ b/config/exporter/exporter_config.go @@ -0,0 +1,104 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package exporter is to init configuration and flags for resource prometheus exporter +package exporter + +import ( + "github.com/spf13/pflag" + + confConsts "github.com/huawei/csm/v2/config/consts" +) + +const ( + defaultIpAddress = "0.0.0.0" + defaultExporterPort = "8887" + controllerOptionName = "PrometheusExporterOption" + defaultStorageGRPCSock = "/var/cmi/cmi.sock" + defaultStorageBackendNamespace = "huawei-csi" + defaultCSIDriverName = "csi.huawei.com" + defaultUseHttps = true +) + +var ( + // Option is a prometheus exporter option instance fot manager init + Option = &option{} +) + +type option struct { + ipAddress string + exporterPort string + storageGRPCSock string + storageBackendNamespace string + csiDriverName string + useHttps bool +} + +// GetName return the name string of the ControllerOption +func (o *option) GetName() string { + return controllerOptionName +} + +// AddFlags is to add flags for the resource topology controller configurations +func (o *option) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&o.ipAddress, confConsts.IpAddress, defaultIpAddress, + "The listening ip address.") + fs.StringVar(&o.exporterPort, confConsts.ExporterPort, defaultExporterPort, + "The exporter port in the container.") + fs.StringVar(&o.storageGRPCSock, confConsts.StorageGRPCSock, defaultStorageGRPCSock, + "The storage grpc client sock file name.") + fs.StringVar(&o.storageBackendNamespace, confConsts.StorageBackendNamespace, defaultStorageBackendNamespace, + "The storage backend namespace name.") + fs.StringVar(&o.csiDriverName, confConsts.CSIDriverName, defaultCSIDriverName, + "The CSI driver name.") + fs.BoolVar(&o.useHttps, confConsts.UseHttps, defaultUseHttps, + "Use https or not.") +} + +// ValidateConfig is to validate input resource topology controller configurations +func (o *option) ValidateConfig() error { + return nil +} + +// GetIpAddress returns the listening ip address +func GetIpAddress() string { + return Option.ipAddress +} + +// GetExporterPort returns the exporter port +func GetExporterPort() string { + return Option.exporterPort +} + +// GetStorageGRPCSock returns the storage GRPC sock +func GetStorageGRPCSock() string { + return Option.storageGRPCSock +} + +// GetStorageBackendNamespace returns the storage backend namespace +func GetStorageBackendNamespace() string { + return Option.storageBackendNamespace +} + +// GetCSIDriverName returns the storage backend namespace +func GetCSIDriverName() string { + return Option.csiDriverName +} + +// GetUseHttps returns use https or not +func GetUseHttps() bool { + return Option.useHttps +} diff --git a/config/leaderelection/leader_election_config.go b/config/leaderelection/leader_election_config.go new file mode 100644 index 0000000..d8f0fcc --- /dev/null +++ b/config/leaderelection/leader_election_config.go @@ -0,0 +1,93 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package leaderelection +package leaderelection + +import ( + "time" + + "github.com/spf13/pflag" + + confConsts "github.com/huawei/csm/v2/config/consts" +) + +const ( + leaderElectionOptionName = "LeaderElectionOption" + defaultLeaderLockNamespace = "default" + defaultLeaderLeaseDuration = 8 * time.Second + defaultLeaderRenewDeadline = 6 * time.Second + defaultLeaderRetryPeriod = 2 * time.Second + enableLeaderElection = false +) + +// Option is a client option instance for manager init +var Option = &option{} + +type option struct { + leaderLeaseDuration time.Duration + leaderRenewDeadline time.Duration + leaderRetryPeriod time.Duration + leaderLockNamespace string + enableLeaderElection bool +} + +// GetName return name string of client option +func (o *option) GetName() string { + return leaderElectionOptionName +} + +// AddFlags is to add flags for client configurations +func (o *option) AddFlags(fs *pflag.FlagSet) { + fs.BoolVar(&o.enableLeaderElection, confConsts.EnableLeaderElection, enableLeaderElection, + "Start a leader election client and gain leadership for controller") + fs.StringVar(&o.leaderLockNamespace, confConsts.LeaderLockNamespace, defaultLeaderLockNamespace, + "Configure leader election lock namespace") + fs.DurationVar(&o.leaderLeaseDuration, confConsts.LeaderLeaseDuration, defaultLeaderLeaseDuration, + "Configure leader election lease duration") + fs.DurationVar(&o.leaderRenewDeadline, confConsts.LeaderRenewDeadline, defaultLeaderRenewDeadline, + "Configure leader election lease renew deadline") + fs.DurationVar(&o.leaderRetryPeriod, confConsts.LeaderRetryPeriod, defaultLeaderRetryPeriod, + "Configure leader election lease retry period") +} + +// ValidateConfig is to validate input client configurations +func (o *option) ValidateConfig() error { + return nil +} + +// GetLeaderLeaseDuration returns the duration of leader lease +func GetLeaderLeaseDuration() time.Duration { + return Option.leaderLeaseDuration +} + +// GetLeaderRenewDeadline returns the deadline of renew a leader +func GetLeaderRenewDeadline() time.Duration { + return Option.leaderRenewDeadline +} + +// GetLeaderRetryPeriod returns leader retry period +func GetLeaderRetryPeriod() time.Duration { + return Option.leaderRetryPeriod +} + +// EnableLeaderElection returns leader election is on +func EnableLeaderElection() bool { + return Option.enableLeaderElection +} + +// GetLeaderLockNamespace returns the leader lock namespace +func GetLeaderLockNamespace() string { + return Option.leaderLockNamespace +} diff --git a/config/log/log_config.go b/config/log/log_config.go new file mode 100644 index 0000000..9f8798c --- /dev/null +++ b/config/log/log_config.go @@ -0,0 +1,58 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package log is used to init log configurations and flags +package log + +import ( + "flag" + + "github.com/spf13/pflag" + + "github.com/huawei/csm/v2/config/consts" +) + +const ( + logOptionName = "LogOption" + defaultLogFileName = "topo-service" +) + +// Option is a log option instance for manager init +var Option = &option{} + +type option struct { + logFile string +} + +// GetName return name string of log option +func (o *option) GetName() string { + return logOptionName +} + +// AddFlags is to add flags for log configurations +func (o *option) AddFlags(fs *pflag.FlagSet) { + fs.AddGoFlagSet(flag.CommandLine) + fs.StringVar(&o.logFile, consts.LogFile, defaultLogFileName, + "The log file name of the resource topology service.") +} + +// ValidateConfig is to validate input log configurations +func (o *option) ValidateConfig() error { + return nil +} + +// GetLogFile returns the log file name +func GetLogFile() string { + return Option.logFile +} diff --git a/config/manager.go b/config/manager.go new file mode 100644 index 0000000..8636c8a --- /dev/null +++ b/config/manager.go @@ -0,0 +1,68 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package config contains all configuration and flags parts for different services +package config + +import ( + "fmt" + + "github.com/sirupsen/logrus" + "github.com/spf13/pflag" +) + +// Option is to help load configuration +type Option interface { + // GetName returns the name of the option + GetName() string + // AddFlags adds the flags to the option + AddFlags(*pflag.FlagSet) + // ValidateConfig validates the input configuration + ValidateConfig() error +} + +// Manager helps to manage the configuration of server +type Manager struct { + options []Option + flagSet *pflag.FlagSet +} + +// NewOptionManager creates a new option manager of specified options using the specified flag set +func NewOptionManager(fs *pflag.FlagSet, options ...Option) *Manager { + return &Manager{ + flagSet: fs, + options: options, + } +} + +// AddFlags adds the topology service config needed flags to set +func (m *Manager) AddFlags() { + for _, o := range m.options { + logrus.Infof("loading config [%s]", o.GetName()) + o.AddFlags(m.flagSet) + } +} + +// ValidateConfig validate input config +func (m *Manager) ValidateConfig() error { + for _, o := range m.options { + logrus.Infof("validating config [%s]", o.GetName()) + err := o.ValidateConfig() + if err != nil { + return fmt.Errorf("validate config [%s] failed: [%v]", o.GetName(), err) + } + } + + return nil +} diff --git a/config/topology/controller_config.go b/config/topology/controller_config.go new file mode 100644 index 0000000..6054d9b --- /dev/null +++ b/config/topology/controller_config.go @@ -0,0 +1,185 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package topology is to init configuration and flags for resource topology controller +package topology + +import ( + "errors" + "fmt" + "time" + + "github.com/spf13/pflag" + + confConsts "github.com/huawei/csm/v2/config/consts" +) + +const ( + defaultControllerWorkers = 4 + controllerOptionName = "ControllerOption" + defaultRtRetryBaseDelay = 5 * time.Second + defaultPvRetryBaseDelay = 5 * time.Second + defaultPodRetryBaseDelay = 5 * time.Second + defaultRtRetryMaxDelay = 5 * time.Minute + defaultPvRetryMaxDelay = 1 * time.Minute + defaultPodRetryMaxDelay = 1 * time.Minute + defaultResyncPeriod = 15 * time.Minute + defaultCmiAddress = "/cmi/cmi.sock" + minSupportResourceNum = 2 + defaultCSIDriverName = "csi.huawei.com" + minResyncPeriod = 5 * time.Minute +) + +var ( + defaultSupportResources = []string{"Pod", "PersistentVolume"} + // Option is a controller option instance fot manager init + Option = &option{} +) + +type option struct { + supportResources []string + rtRetryBaseDelay time.Duration + pvRetryBaseDelay time.Duration + podRetryBaseDelay time.Duration + rtRetryMaxDelay time.Duration + pvRetryMaxDelay time.Duration + podRetryMaxDelay time.Duration + resyncPeriod time.Duration + cmiAddress string + controllerWorkers int + csiDriverName string +} + +// GetName return the name string of the ControllerOption +func (o *option) GetName() string { + return controllerOptionName +} + +// AddFlags is to add flags for the resource topology controller configurations +func (o *option) AddFlags(fs *pflag.FlagSet) { + fs.IntVar(&o.controllerWorkers, confConsts.ControllerWorkers, defaultControllerWorkers, + "Number of worker for controller") + fs.StringArrayVar(&o.supportResources, confConsts.SupportResources, defaultSupportResources, + "Define which resources can be added to tags. Example: --supportedResources=Pod,PersistentVolume") + fs.DurationVar(&o.rtRetryBaseDelay, confConsts.RtRetryBaseDelay, defaultRtRetryBaseDelay, + "Base retry delay of failed resourceTopology creation or deletion. "+ + "It doubles with each failure, up to rt-retry-interval-max.") + fs.DurationVar(&o.pvRetryBaseDelay, confConsts.PvRetryBaseDelay, defaultPvRetryBaseDelay, + "Base retry delay of failed pv work task. "+ + "It doubles with each failure, up to pv-retry-interval-max.") + fs.DurationVar(&o.podRetryBaseDelay, confConsts.PodRetryBaseDelay, defaultPodRetryBaseDelay, + "Base retry delay of failed pod work task. "+ + "It doubles with each failure, up to pod-retry-interval-max.") + fs.DurationVar(&o.rtRetryMaxDelay, confConsts.RtRetryMaxDelay, defaultRtRetryMaxDelay, + "Maximum retry delay of failed resourceTopology creation or deletion.") + fs.DurationVar(&o.pvRetryMaxDelay, confConsts.PvRetryMaxDelay, defaultPvRetryMaxDelay, + "Maximum retry delay of failed pv work task.") + fs.DurationVar(&o.podRetryMaxDelay, confConsts.PodRetryMaxDelay, defaultPodRetryMaxDelay, + "Maximum retry delay of failed pod work task.") + fs.DurationVar(&o.resyncPeriod, confConsts.ResyncPeriod, defaultResyncPeriod, + "The reSync interval of the controller.") + fs.StringVar(&o.cmiAddress, confConsts.CmiAddress, defaultCmiAddress, + "The socket address of container monitoring interface.") + fs.StringVar(&o.csiDriverName, confConsts.CSIDriverName, defaultCSIDriverName, + "The CSI driver name.") +} + +// ValidateConfig is to validate input resource topology controller configurations +func (o *option) ValidateConfig() error { + workers := o.controllerWorkers + if workers < 1 { + return fmt.Errorf("invalid controller workers count [%d]", workers) + } + + if len(o.supportResources) < minSupportResourceNum { + return errors.New("supported resources should be at least 2") + } + + if o.rtRetryMaxDelay < o.rtRetryBaseDelay { + return fmt.Errorf("rt retry max delay [%s] is less than rt retry base delay [%s]", + o.rtRetryMaxDelay, o.rtRetryBaseDelay) + } + + if o.pvRetryMaxDelay < o.pvRetryBaseDelay { + return fmt.Errorf("pv retry max delay [%s] is less than pv retry base delay [%s]", + o.pvRetryMaxDelay, o.pvRetryBaseDelay) + } + + if o.podRetryMaxDelay < o.podRetryBaseDelay { + return fmt.Errorf("pod retry max delay [%s] is less than pod retry base delay [%s]", + o.podRetryMaxDelay, o.podRetryBaseDelay) + } + + if o.resyncPeriod <= minResyncPeriod { + return fmt.Errorf("resync period [%s] is less than min resync period [%s]", + o.resyncPeriod, minResyncPeriod) + } + + return nil +} + +// GetControllerWorkers returns the number of controller workers +func GetControllerWorkers() int { + return Option.controllerWorkers +} + +// GetSupportResources returns the supported resource name list +func GetSupportResources() []string { + return Option.supportResources +} + +// GetRtRetryBaseDelay returns the base retry delay of rt controller +func GetRtRetryBaseDelay() time.Duration { + return Option.rtRetryBaseDelay +} + +// GetRtRetryMaxDelay returns the max retry delay of rt controller +func GetRtRetryMaxDelay() time.Duration { + return Option.rtRetryMaxDelay +} + +// GetPvRetryBaseDelay returns the base retry delay of pv controller +func GetPvRetryBaseDelay() time.Duration { + return Option.pvRetryBaseDelay +} + +// GetPvRetryMaxDelay returns the max retry delay of pv controller +func GetPvRetryMaxDelay() time.Duration { + return Option.pvRetryMaxDelay +} + +// GetPodRetryBaseDelay returns the base retry delay of pod controller +func GetPodRetryBaseDelay() time.Duration { + return Option.podRetryBaseDelay +} + +// GetPodRetryMaxDelay returns the max retry delay of pod controller +func GetPodRetryMaxDelay() time.Duration { + return Option.podRetryMaxDelay +} + +// GetResyncPeriod returns the resync interval +func GetResyncPeriod() time.Duration { + return Option.resyncPeriod +} + +// GetCmiAddress returns container monitoring interface address +func GetCmiAddress() string { + return Option.cmiAddress +} + +// GetCSIDriverName returns the storage backend namespace +func GetCSIDriverName() string { + return Option.csiDriverName +} diff --git a/config/topology/controller_config_test.go b/config/topology/controller_config_test.go new file mode 100644 index 0000000..8ca572c --- /dev/null +++ b/config/topology/controller_config_test.go @@ -0,0 +1,75 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package topology +package topology + +import ( + "errors" + "fmt" + "reflect" + "testing" +) + +func Test_option_ValidateConfig_Success(t *testing.T) { + // arrange + o := &option{ + controllerWorkers: 4, + supportResources: []string{"Pod", "PersistentVolume"}, + resyncPeriod: defaultResyncPeriod, + } + + // act + err := o.ValidateConfig() + + // assert + if err != nil { + t.Errorf("Test_option_ValidateConfig_Success failed: [%v]", err) + } +} + +func Test_option_ValidateConfig_ControllerWorkersLessThanOne_Failed(t *testing.T) { + // arrange + o := &option{ + controllerWorkers: 0, + supportResources: []string{"Pod", "PersistentVolume"}, + } + want := fmt.Errorf("invalid controller workers count [%d]", o.controllerWorkers) + + // act + got := o.ValidateConfig() + + // assert + if !reflect.DeepEqual(got, want) { + t.Errorf("Test_option_ValidateConfig_ControllerWorkersLessThanOne_Failed: want [%v], got [%v]", want, got) + } +} + +func Test_option_ValidateConfig_SupportResourcesLengthLessThanTwo_Failed(t *testing.T) { + // arrange + o := &option{ + controllerWorkers: 1, + supportResources: []string{}, + } + want := errors.New("supported resources should be at least 2") + + // act + got := o.ValidateConfig() + + // assert + if !reflect.DeepEqual(got, want) { + t.Errorf("Test_option_ValidateConfig_SupportResourcesLengthLessThanTwo_Failed: "+ + "want [%v], got [%v]", want, got) + } +} diff --git a/controller/leaderelection/leader_election.go b/controller/leaderelection/leader_election.go new file mode 100644 index 0000000..4d02d44 --- /dev/null +++ b/controller/leaderelection/leader_election.go @@ -0,0 +1,172 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package leaderelection offers leader election starter +package leaderelection + +import ( + "context" + "errors" + "fmt" + "os" + "syscall" + + "k8s.io/client-go/tools/leaderelection" + "k8s.io/client-go/tools/leaderelection/resourcelock" + "k8s.io/client-go/tools/record" + + "github.com/huawei/csm/v2/controller/utils" + "github.com/huawei/csm/v2/utils/log" +) + +type leaderElectionRunner struct { + params *Params + + leaderElector *leaderelection.LeaderElector + hostname string + resLockConfig resourcelock.ResourceLockConfig + resLock resourcelock.Interface + leaderElectionConfig leaderelection.LeaderElectionConfig + + err error +} + +// Run will run the func with leader election +func Run(ctx context.Context, clientSet *utils.ClientsSet, params *Params, + runFunc func(context.Context, *utils.ClientsSet, chan os.Signal), ch chan os.Signal) { + runner := &leaderElectionRunner{params: params} + runner.channelCheck(ctx, ch). + setHostname(ctx). + setResLockConfig(clientSet.EventRecorder). + setResLock(ctx, clientSet). + setLeaderElectionConfig(ctx, runFunc, clientSet, ch). + setLeaderElector(ctx). + run(ctx, ch) +} + +func (runner *leaderElectionRunner) channelCheck(ctx context.Context, ch chan os.Signal) *leaderElectionRunner { + if ch == nil { + errMsg := "the channel should not be nil" + log.AddContext(ctx).Errorln(errMsg) + runner.err = errors.New(errMsg) + } + return runner +} + +func (runner *leaderElectionRunner) setHostname(ctx context.Context) *leaderElectionRunner { + if runner.err != nil { + return runner + } + + hostname, err := os.Hostname() + if err != nil { + errMsg := fmt.Sprintf("error getting hostname: [%v]", err) + log.AddContext(ctx).Errorln(errMsg) + runner.err = errors.New(errMsg) + return runner + } + + runner.hostname = hostname + return runner +} + +func (runner *leaderElectionRunner) setResLockConfig(recorder record.EventRecorder) *leaderElectionRunner { + if runner.err != nil { + return runner + } + + runner.resLockConfig = resourcelock.ResourceLockConfig{ + Identity: runner.hostname, + EventRecorder: recorder, + } + + return runner +} + +func (runner *leaderElectionRunner) setResLock(ctx context.Context, clientSet *utils.ClientsSet) *leaderElectionRunner { + if runner.err != nil { + return runner + } + + lock, err := resourcelock.New( + resourcelock.LeasesResourceLock, + utils.GetNameSpaceFromEnv(runner.params.namespaceEnv, runner.params.defaultNamespace), + runner.params.lockName, + clientSet.KubeClient.CoreV1(), + clientSet.KubeClient.CoordinationV1(), + runner.resLockConfig) + if err != nil { + errMsg := fmt.Sprintf("error creating resource lock: [%v]", err) + log.AddContext(ctx).Errorln(errMsg) + runner.err = errors.New(errMsg) + return runner + } + + runner.resLock = lock + return runner +} + +func (runner *leaderElectionRunner) setLeaderElectionConfig(ctx context.Context, + runFunc func(context.Context, *utils.ClientsSet, chan os.Signal), + clientSet *utils.ClientsSet, ch chan os.Signal) *leaderElectionRunner { + if runner.err != nil { + return runner + } + + runner.leaderElectionConfig = leaderelection.LeaderElectionConfig{ + Lock: runner.resLock, + LeaseDuration: runner.params.leaderLeaseDuration, + RenewDeadline: runner.params.leaderRenewDeadline, + RetryPeriod: runner.params.leaderRetryPeriod, + Callbacks: leaderelection.LeaderCallbacks{ + OnStartedLeading: func(ctx context.Context) { + runFunc(ctx, clientSet, ch) + }, + OnStoppedLeading: func() { + log.AddContext(ctx).Errorln("controller manager lost master") + ch <- syscall.SIGINT + }, + OnNewLeader: func(identity string) { + log.AddContext(ctx).Infof("new leader elected, current leader [%s]", identity) + }, + }, + } + return runner +} + +func (runner *leaderElectionRunner) setLeaderElector(ctx context.Context) *leaderElectionRunner { + if runner.err != nil { + return runner + } + + leaderElector, err := leaderelection.NewLeaderElector(runner.leaderElectionConfig) + if err != nil { + errMsg := fmt.Sprintf("error creating leader elector: [%v]", err) + log.AddContext(ctx).Errorln(errMsg) + runner.err = errors.New(errMsg) + return runner + } + + runner.leaderElector = leaderElector + return runner +} + +func (runner *leaderElectionRunner) run(ctx context.Context, ch chan os.Signal) { + if runner.err != nil { + ch <- syscall.SIGINT + return + } + + runner.leaderElector.Run(ctx) +} diff --git a/controller/leaderelection/params.go b/controller/leaderelection/params.go new file mode 100644 index 0000000..f18ba93 --- /dev/null +++ b/controller/leaderelection/params.go @@ -0,0 +1,69 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package leaderelection offers leader election starter +package leaderelection + +import "time" + +const ( + defaultNamespaceEnv = "NAMESPACE" +) + +// Params leader election parameters +type Params struct { + leaderLeaseDuration time.Duration + leaderRenewDeadline time.Duration + leaderRetryPeriod time.Duration + lockName string + namespaceEnv string + defaultNamespace string +} + +// NewParams returns an init leader election parameters struct +func NewParams() *Params { + return &Params{ + namespaceEnv: defaultNamespaceEnv, + } +} + +// SetLeaderLeaseDuration sets the leader lease duration +func (p *Params) SetLeaderLeaseDuration(leaderLeaseDuration time.Duration) *Params { + p.leaderLeaseDuration = leaderLeaseDuration + return p +} + +// SetLeaderRenewDeadline sets the leader renew deadline +func (p *Params) SetLeaderRenewDeadline(leaderRenewDeadline time.Duration) *Params { + p.leaderRenewDeadline = leaderRenewDeadline + return p +} + +// SetLeaderRetryPeriod sets the leader retry period +func (p *Params) SetLeaderRetryPeriod(leaderRetryPeriod time.Duration) *Params { + p.leaderRetryPeriod = leaderRetryPeriod + return p +} + +// SetLockName sets the leader lock name +func (p *Params) SetLockName(lockName string) *Params { + p.lockName = lockName + return p +} + +// SetDefaultNamespace sets default namespace +func (p *Params) SetDefaultNamespace(defaultNamespace string) *Params { + p.defaultNamespace = defaultNamespace + return p +} diff --git a/controller/resource/inner_tag.go b/controller/resource/inner_tag.go new file mode 100644 index 0000000..c1f82e7 --- /dev/null +++ b/controller/resource/inner_tag.go @@ -0,0 +1,60 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package resource defines some support resource interface for topology +package resource + +import ( + "fmt" + + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + apiXuanwuV1 "github.com/huawei/csm/v2/client/apis/xuanwu/v1" +) + +var factoryMap map[metaV1.TypeMeta]func() InnerTag + +func init() { + factoryMap = make(map[metaV1.TypeMeta]func() InnerTag) + factoryMap[metaV1.TypeMeta{Kind: podV1Kind, APIVersion: podV1ApiVersion}] = func() InnerTag { return &PodV1Tag{} } + factoryMap[metaV1.TypeMeta{Kind: persistentVolumeV1Kind, APIVersion: persistentVolumeV1ApiVersion}] = + func() InnerTag { return &PersistentVolumeV1Tag{} } +} + +// InnerTag defines some support resource interface for topology tags change +type InnerTag interface { + // InitFromResourceInfo init InnerTag by resource info + InitFromResourceInfo(apiXuanwuV1.ResourceInfo) + // HasOwner check if resource has owner + HasOwner() bool + // GetOwner returns owner resource inner tags of resource + GetOwner() (InnerTag, error) + // Exists checks if resource exists + Exists() bool + // Ready checks if resource is ready + Ready() bool + // IsDeleting check if resource is deleting + IsDeleting() bool + // ToResourceInfo convert inner tag to resource info + ToResourceInfo() apiXuanwuV1.ResourceInfo +} + +// NewInnerTag returns an empty InnerTag interface implementation object +func NewInnerTag(meta metaV1.TypeMeta) (InnerTag, error) { + factory, ok := factoryMap[meta] + if !ok { + return nil, fmt.Errorf("unsupported tag type [%v]", meta) + } + return factory(), nil +} diff --git a/controller/resource/inner_tag_test.go b/controller/resource/inner_tag_test.go new file mode 100644 index 0000000..3551af5 --- /dev/null +++ b/controller/resource/inner_tag_test.go @@ -0,0 +1,72 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package resource +package resource + +import ( + "fmt" + "reflect" + "testing" + + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestNewInnerTag_PvV1(t *testing.T) { + // arrange + meta := metaV1.TypeMeta{Kind: persistentVolumeV1Kind, APIVersion: persistentVolumeV1ApiVersion} + want := &PersistentVolumeV1Tag{} + + // act + got, err := NewInnerTag(meta) + + // assert + if err != nil { + t.Errorf("TestNewInnerTag_PvV1 failed: [%v]", err) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("TestNewInnerTag_PvV1 failed: want [%v], got [%v]", want, got) + } +} + +func TestNewInnerTag_PodV1(t *testing.T) { + // arrange + meta := metaV1.TypeMeta{Kind: podV1Kind, APIVersion: podV1ApiVersion} + want := &PodV1Tag{} + + // act + got, err := NewInnerTag(meta) + + // assert + if err != nil { + t.Errorf("TestNewInnerTag_PodV1 failed: [%v]", err) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("TestNewInnerTag_PodV1 failed: want [%v], got [%v]", want, got) + } +} + +func TestNewInnerTag_UnSupportedType(t *testing.T) { + // arrange + meta := metaV1.TypeMeta{Kind: "fakeKind", APIVersion: "fakeAPIVersion"} + want := fmt.Errorf("unsupported tag type [%v]", meta) + + // act + _, got := NewInnerTag(meta) + + // assert + if !reflect.DeepEqual(got, want) { + t.Errorf("TestNewInnerTag_UnSupportedType failed: want [%v], got [%v]", want, got) + } +} diff --git a/controller/resource/persistent_volume_v1.go b/controller/resource/persistent_volume_v1.go new file mode 100644 index 0000000..d6661b9 --- /dev/null +++ b/controller/resource/persistent_volume_v1.go @@ -0,0 +1,131 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package resource defines some support resource interface for topology +package resource + +import ( + "fmt" + + coreV1 "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + apiXuanwuV1 "github.com/huawei/csm/v2/client/apis/xuanwu/v1" + "github.com/huawei/csm/v2/controller/utils" + "github.com/huawei/csm/v2/controller/utils/consts" + "github.com/huawei/csm/v2/utils/log" + "github.com/huawei/csm/v2/utils/resource" +) + +const ( + persistentVolumeV1Kind = "PersistentVolume" + persistentVolumeV1ApiVersion = "v1" +) + +// PersistentVolumeV1Tag represents a persistent volume in v1 version +type PersistentVolumeV1Tag struct { + name string +} + +// InitFromResourceInfo init PersistentVolumeV1Tag struct from resource info +func (p *PersistentVolumeV1Tag) InitFromResourceInfo(info apiXuanwuV1.ResourceInfo) { + p.name = info.Name +} + +// HasOwner persistent volume don't need mark owner +func (p *PersistentVolumeV1Tag) HasOwner() bool { + return false +} + +// GetOwner persistent volume don't need mark owner +func (p *PersistentVolumeV1Tag) GetOwner() (InnerTag, error) { + return nil, nil +} + +// Exists check if persistent volume exist in cluster +func (p *PersistentVolumeV1Tag) Exists() bool { + err := utils.RetryFunc(func() (bool, error) { + _, err := resource.Instance().GetPV(p.name) + if err == nil { + return true, nil + } + if apiErrors.IsNotFound(err) { + return true, err + } + return false, err + }, consts.RetryTimes, consts.RetryDurationInit, consts.RetryDurationMax) + + if err != nil { + log.Errorf("get PersistentVolume [%s] failed: [%v]", p.name, err) + return false + } + return true +} + +// Ready check if persistent volume is Bounded +func (p *PersistentVolumeV1Tag) Ready() bool { + err := utils.RetryFunc(func() (bool, error) { + pv, err := resource.Instance().GetPV(p.name) + if err != nil && apiErrors.IsNotFound(err) { + return true, err + } + if err != nil { + return false, err + } + if pv.Status.Phase != coreV1.VolumeBound { + return false, fmt.Errorf("pv [%s] is not in bound status", p.name) + } + return true, nil + }, consts.RetryTimes, consts.RetryDurationInit, consts.RetryDurationMax) + if err != nil { + log.Errorf("check PersistentVolume [%s] ready failed: [%v]", p.name, err) + return false + } + return true +} + +// IsDeleting check if persistent volume has DeletionTimestamp field +func (p *PersistentVolumeV1Tag) IsDeleting() bool { + pv := &coreV1.PersistentVolume{} + err := utils.RetryFunc(func() (bool, error) { + var err error + pv, err = resource.Instance().GetPV(p.name) + if err != nil && apiErrors.IsNotFound(err) { + return true, err + } + + if err != nil { + return false, err + } + + return true, nil + }, consts.RetryTimes, consts.RetryDurationInit, consts.RetryDurationMax) + if err != nil { + log.Errorf("check PersistentVolume [%s] deleting stage failed: [%v]", p.name, err) + return false + } + return pv.DeletionTimestamp != nil +} + +// ToResourceInfo converts PersistentVolumeV1Tag to resource info +func (p *PersistentVolumeV1Tag) ToResourceInfo() apiXuanwuV1.ResourceInfo { + return apiXuanwuV1.ResourceInfo{ + TypeMeta: metaV1.TypeMeta{ + Kind: persistentVolumeV1Kind, + APIVersion: persistentVolumeV1ApiVersion, + }, + Name: p.name, + } +} diff --git a/controller/resource/pod_v1.go b/controller/resource/pod_v1.go new file mode 100644 index 0000000..2f3115b --- /dev/null +++ b/controller/resource/pod_v1.go @@ -0,0 +1,200 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package resource defines some support resource interface for topology +package resource + +import ( + "fmt" + + coreV1 "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + apiXuanwuV1 "github.com/huawei/csm/v2/client/apis/xuanwu/v1" + "github.com/huawei/csm/v2/controller/utils" + "github.com/huawei/csm/v2/controller/utils/consts" + "github.com/huawei/csm/v2/utils/log" + "github.com/huawei/csm/v2/utils/resource" +) + +const ( + podV1Kind = "Pod" + podV1ApiVersion = "v1" +) + +// PodV1Tag represents a pod in v1 version +type PodV1Tag struct { + name string + namespace string +} + +// InitFromResourceInfo init PodV1Tag struct from resource info +func (p *PodV1Tag) InitFromResourceInfo(info apiXuanwuV1.ResourceInfo) { + p.name = info.Name + p.namespace = info.Namespace +} + +// HasOwner checks if pod has OwnerReferences field +func (p *PodV1Tag) HasOwner() bool { + pod := &coreV1.Pod{} + err := utils.RetryFunc(func() (bool, error) { + var err error + pod, err = resource.Instance().GetPodByNameSpaceAndName(p.namespace, p.name, metaV1.GetOptions{}) + if err != nil && apiErrors.IsNotFound(err) { + return true, err + } + if err != nil { + return false, err + } + return true, nil + }, consts.RetryTimes, consts.RetryDurationInit, consts.RetryDurationMax) + + if err != nil { + log.Errorf("get Pod [%s/%s] failed: [%v]", p.namespace, p.name, err) + return false + } + return len(pod.OwnerReferences) != 0 +} + +// GetOwner returns the list of owners inner tag of the pod +func (p *PodV1Tag) GetOwner() (InnerTag, error) { + var info apiXuanwuV1.ResourceInfo + err := utils.RetryFunc(func() (bool, error) { + pod, err := resource.Instance().GetPodByNameSpaceAndName(p.namespace, p.name, metaV1.GetOptions{}) + if err != nil && apiErrors.IsNotFound(err) { + return true, err + } + if err != nil { + return false, err + } + + info = p.getOwnerResourceInfo(pod, p.namespace) + + return true, nil + }, consts.RetryTimes, consts.RetryDurationInit, consts.RetryDurationMax) + + if err != nil { + log.Errorf("get Pod [%s/%s] failed: [%v]", p.namespace, p.name, err) + return nil, err + } + + innerTag, err := NewInnerTag(info.TypeMeta) + if err != nil { + log.Warningf("get pod [%s/%s] owner references failed: [%v]", p.namespace, p.name, err) + return nil, nil + } + + return innerTag, nil +} + +// Exists checks if the pod exists in cluster +func (p *PodV1Tag) Exists() bool { + err := utils.RetryFunc(func() (bool, error) { + _, err := resource.Instance().GetPodByNameSpaceAndName(p.namespace, p.name, metaV1.GetOptions{}) + if err == nil { + return true, nil + } + if apiErrors.IsNotFound(err) { + return true, err + } + return false, err + }, consts.RetryTimes, consts.RetryDurationInit, consts.RetryDurationMax) + + if err != nil { + log.Errorf("get Pod [%s/%s] failed: [%v]", p.namespace, p.name, err) + return false + } + return true +} + +// Ready checks whether the pod is in Running or Succeeded status +func (p *PodV1Tag) Ready() bool { + err := utils.RetryFunc(func() (bool, error) { + pod, err := resource.Instance().GetPodByNameSpaceAndName(p.namespace, p.name, metaV1.GetOptions{}) + if err != nil && apiErrors.IsNotFound(err) { + return true, err + } + if err != nil { + return false, err + } + if pod.Status.Phase == coreV1.PodFailed { + return true, err + } + if pod.Status.Phase != coreV1.PodRunning && pod.Status.Phase != coreV1.PodSucceeded { + return false, fmt.Errorf("pv is not in [%s/%s] status, "+ + "cur status [%s]", coreV1.PodRunning, coreV1.PodSucceeded, pod.Status.Phase) + } + return true, nil + }, consts.RetryTimes, consts.RetryDurationInit, consts.RetryDurationMax) + + if err != nil { + log.Errorf("check Pod [%s/%s] ready failed: [%v]", p.namespace, p.name, err) + return false + } + return true +} + +// IsDeleting checks whether the pod has DeletionTimestamp +func (p *PodV1Tag) IsDeleting() bool { + pod := &coreV1.Pod{} + err := utils.RetryFunc(func() (bool, error) { + var err error + pod, err = resource.Instance().GetPodByNameSpaceAndName(p.namespace, p.name, metaV1.GetOptions{}) + if err != nil && apiErrors.IsNotFound(err) { + return true, err + } + if err != nil { + return false, err + } + return true, nil + }, consts.RetryTimes, consts.RetryDurationInit, consts.RetryDurationMax) + + if err != nil { + log.Errorf("check Pod [%s/%s] ready failed: [%v]", p.namespace, p.name, err) + return false + } + return pod.DeletionTimestamp != nil +} + +// ToResourceInfo converts a PodV1Tag to a ResourceInfo +func (p *PodV1Tag) ToResourceInfo() apiXuanwuV1.ResourceInfo { + return apiXuanwuV1.ResourceInfo{ + TypeMeta: metaV1.TypeMeta{ + Kind: podV1Kind, + APIVersion: podV1ApiVersion, + }, + Namespace: p.namespace, + Name: p.name, + } +} + +func (p *PodV1Tag) getOwnerResourceInfo(pod *coreV1.Pod, namespace string) apiXuanwuV1.ResourceInfo { + var info apiXuanwuV1.ResourceInfo + for _, ref := range pod.OwnerReferences { + if !*ref.Controller { + continue + } + info = apiXuanwuV1.ResourceInfo{ + TypeMeta: metaV1.TypeMeta{ + Kind: ref.Kind, + APIVersion: ref.APIVersion, + }, + Namespace: namespace, + Name: ref.Name, + } + break + } + return info +} diff --git a/controller/resourcetopology/persistent_volume_sync.go b/controller/resourcetopology/persistent_volume_sync.go new file mode 100644 index 0000000..df33b2a --- /dev/null +++ b/controller/resourcetopology/persistent_volume_sync.go @@ -0,0 +1,123 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package resourcetopology defines to reconcile action of resources topologies +package resourcetopology + +import ( + "context" + + coreV1 "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + apiXuanwuV1 "github.com/huawei/csm/v2/client/apis/xuanwu/v1" + "github.com/huawei/csm/v2/config/cmi" + "github.com/huawei/csm/v2/controller/utils" + "github.com/huawei/csm/v2/controller/utils/consts" + "github.com/huawei/csm/v2/utils/log" +) + +func (ctrl *Controller) syncPersistentVolume(ctx context.Context, pv *coreV1.PersistentVolume) error { + log.AddContext(ctx).Infof("[pv-controller] start to sync pv [%s]", pv.Name) + defer log.AddContext(ctx).Infof("[pv-controller] finished sync pv [%s]", pv.Name) + + if pv.Status.Phase == coreV1.VolumeFailed { + log.AddContext(ctx).Debugf("[pv-controller] pv [%s] is in failed status, skip to next", pv.Name) + return nil + } + + rtName := getResourceTopologyName(pv.Name) + labelSelector := &metaV1.LabelSelector{ + MatchLabels: map[string]string{consts.VolumeHandleKeyLabel: utils.EncryptMD5(pv.Spec.CSI.VolumeHandle)}, + } + selector, err := metaV1.LabelSelectorAsSelector(labelSelector) + if err != nil { + return err + } + + rtList, err := ctrl.topologyInformer.Lister().List(selector) + if err != nil { + return err + } + + if len(rtList) > 0 { + if rtList[0].Name != rtName { + log.AddContext(ctx).Warningf("[pv-controller] current resource [%s] has bound to another rt [%s]", + pv.Spec.CSI.VolumeHandle, rtList[0].Name) + return nil + } + + if rtList[0].DeletionTimestamp != nil { + log.Warningf("[pv-controller] rt [%s] is deleting, "+ + "wait the deletion finished to recreated", rtName) + ctrl.volumeQueue.AddAfter(pv.Name, resourceRequeueInterval) + return nil + } + + return nil + } + + return ctrl.createResourceTopology(ctx, pv, rtName) +} + +func (ctrl *Controller) removeResourceTopology(ctx context.Context, pvName string) error { + rtName := getResourceTopologyName(pvName) + log.AddContext(ctx).Infof("[pv-controller] start to delete rt [%s]", rtName) + defer log.AddContext(ctx).Infof("[pv-controller] finished delete rt [%s]", rtName) + + err := ctrl.xuanwuClient.XuanwuV1().ResourceTopologies().Delete(ctx, rtName, metaV1.DeleteOptions{}) + if err != nil && !apiErrors.IsNotFound(err) { + return err + } + + log.AddContext(ctx).Infof("[pv-controller] rt [%s] deleted by pv [%s] success", rtName, pvName) + return nil +} + +func (ctrl *Controller) createResourceTopology(ctx context.Context, + pv *coreV1.PersistentVolume, rtName string) error { + log.AddContext(ctx).Debugf("[pv-controller] start to create rt [%s]", rtName) + defer log.AddContext(ctx).Debugf("[pv-controller] finished create rt [%s]", rtName) + + rtLabels := make(map[string]string) + rtLabels[consts.VolumeHandleKeyLabel] = utils.EncryptMD5(pv.Spec.CSI.VolumeHandle) + + topologySpec := apiXuanwuV1.ResourceTopologySpec{ + Provisioner: cmi.GetProviderName(), + VolumeHandle: pv.Spec.CSI.VolumeHandle, + Tags: []apiXuanwuV1.Tag{ + { + ResourceInfo: apiXuanwuV1.ResourceInfo{ + TypeMeta: metaV1.TypeMeta{Kind: consts.PersistentVolume, APIVersion: consts.KubernetesV1}, + Name: pv.Name, + }, + }, + }, + } + + rt := &apiXuanwuV1.ResourceTopology{ + TypeMeta: metaV1.TypeMeta{Kind: consts.TopologyKind, APIVersion: consts.XuanwuV1}, + ObjectMeta: metaV1.ObjectMeta{Name: rtName, Labels: rtLabels}, + Spec: topologySpec, + } + + _, err := ctrl.xuanwuClient.XuanwuV1().ResourceTopologies().Create(ctx, rt, metaV1.CreateOptions{}) + if err != nil { + return err + } + + log.AddContext(ctx).Infof("[pv-controller] rt [%s] created by pv [%s] success", rtName, pv.Name) + return nil +} diff --git a/controller/resourcetopology/pod_sync.go b/controller/resourcetopology/pod_sync.go new file mode 100644 index 0000000..f42d12b --- /dev/null +++ b/controller/resourcetopology/pod_sync.go @@ -0,0 +1,272 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package resourcetopology defines to reconcile action of resources topologies +package resourcetopology + +import ( + "context" + "fmt" + + coreV1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + apiXuanwuV1 "github.com/huawei/csm/v2/client/apis/xuanwu/v1" + "github.com/huawei/csm/v2/controller/utils/consts" + "github.com/huawei/csm/v2/utils/log" +) + +func (ctrl *Controller) syncPod(ctx context.Context, pod *coreV1.Pod) error { + log.AddContext(ctx).Infof("[pod-controller] start to sync pod [%s/%s] relate tags", + pod.Namespace, pod.Name) + defer log.AddContext(ctx).Infof("[pod-controller] finished sync pod [%s/%s] relate tags", + pod.Namespace, pod.Name) + + err := ctrl.podStore.Add(pod) + if err != nil { + return nil + } + + pvcNameList, retryErr := filterAvailablePvcInPod(pod) + + err = ctrl.syncPodTagByPvcList(ctx, pvcNameList, pod.Name, pod.Namespace) + if err != nil { + return err + } + + if retryErr != nil { + log.AddContext(ctx).Warningln(retryErr.Error()) + ctrl.podQueue.AddAfter(getPodKey(pod.Namespace, pod.Name), resourceRequeueInterval) + return nil + } + + return nil +} + +func (ctrl *Controller) syncPodTagByPvcList(ctx context.Context, + pvcNameList []string, podName, namespace string) error { + unAvailableRtNameList := make([]string, 0, len(pvcNameList)) + for _, pvcName := range pvcNameList { + pv, err := ctrl.getPvByPvcName(pvcName, namespace) + if errors.IsNotFound(err) { + continue + } + + if err != nil { + return err + } + + rtName := getResourceTopologyName(pv.Name) + rt, err := ctrl.topologyInformer.Lister().Get(rtName) + if err != nil && !errors.IsNotFound(err) { + return err + } + + if errors.IsNotFound(err) || rt.DeletionTimestamp != nil { + unAvailableRtNameList = append(unAvailableRtNameList, rtName) + continue + } + + if isTagExist(rt.Spec.Tags, podName, namespace) { + continue + } + + rt.Spec.Tags = addPodTag(rt.Spec.Tags, podName, namespace) + _, err = ctrl.xuanwuClient.XuanwuV1().ResourceTopologies().Update(ctx, rt, metaV1.UpdateOptions{}) + if err != nil { + return err + } + + log.AddContext(ctx).Infof("[pod-controller] add tag [%s/%s] to rt [%s] success", + namespace, podName, rt.Name) + } + + if len(unAvailableRtNameList) > 0 { + log.AddContext(ctx).Warningf("[pod-controller] resourceTopologies %v are not available, "+ + "need to retry later", unAvailableRtNameList) + ctrl.podQueue.AddAfter(getPodKey(namespace, podName), resourceRequeueInterval) + return nil + } + + return nil +} + +func (ctrl *Controller) removePodTag(ctx context.Context, key string) error { + log.AddContext(ctx).Infof("[pod-controller] start to delete pod [%s] relate tags", key) + defer log.AddContext(ctx).Infof("[pod-controller] finished delete pod [%s] relate tags", key) + podObj, isExist, err := ctrl.podStore.GetByKey(key) + if err != nil { + return err + } + + if !isExist { + log.AddContext(ctx).Errorf("[pod-controller] can not find the pod [%s] in cache", key) + return nil + } + + pod, ok := podObj.(*coreV1.Pod) + if !ok { + log.AddContext(ctx).Errorf("[pod-controller] invalid struct of the pod [%s] in cache", key) + return err + } + + for _, volume := range pod.Spec.Volumes { + if volume.PersistentVolumeClaim == nil { + continue + } + + rt, err := ctrl.getRtByPvcName(volume.PersistentVolumeClaim.ClaimName, pod.Namespace) + if errors.IsNotFound(err) { + log.AddContext(ctx).Debugf("[pod-controller] can not find the resource: [%v], skip to next", err) + continue + } + + if err != nil { + return err + } + + if rt.DeletionTimestamp != nil { + log.AddContext(ctx).Debugf("[pod-controller] rt [%s] is deleting, no need to delete tag", rt.Name) + continue + } + + if !isTagExist(rt.Spec.Tags, pod.Name, pod.Namespace) { + continue + } + + log.AddContext(ctx).Debugf("[pod-controller] try to delete rt [%s] tag [%s]", rt.Name, key) + rt.Spec.Tags = deletePodTag(rt.Spec.Tags, pod.Name, pod.Namespace) + _, err = ctrl.xuanwuClient.XuanwuV1().ResourceTopologies().Update(ctx, rt, metaV1.UpdateOptions{}) + if err != nil { + return err + } + } + + return ctrl.podStore.Delete(pod) +} + +func filterAvailablePvcInPod(pod *coreV1.Pod) ([]string, error) { + // running container set + runningContainerNameSet := make(map[string]bool) + for _, containerStatus := range pod.Status.ContainerStatuses { + if containerStatus.State.Running != nil { + runningContainerNameSet[containerStatus.Name] = true + } + } + + // pv-pvc map + volumeMap := make(map[string]string) + for _, volume := range pod.Spec.Volumes { + if volume.PersistentVolumeClaim != nil { + volumeMap[volume.Name] = volume.PersistentVolumeClaim.ClaimName + } + } + + // container-pvcList map + containerMap := make(map[string][]string) + for _, container := range pod.Spec.Containers { + containerMap[container.Name] = make([]string, 0) + for _, volume := range container.VolumeMounts { + containerMap[container.Name] = append(containerMap[container.Name], volumeMap[volume.Name]) + } + + for _, volume := range container.VolumeDevices { + containerMap[container.Name] = append(containerMap[container.Name], volumeMap[volume.Name]) + } + } + + pvcNameList := make([]string, 0) + needRetryContainerList := make([]string, 0) + for containerName, pvcList := range containerMap { + if runningContainerNameSet[containerName] { + pvcNameList = append(pvcNameList, pvcList...) + } else if len(pvcList) > 0 { + needRetryContainerList = append(needRetryContainerList, containerName) + } + } + + if len(needRetryContainerList) > 0 { + return pvcNameList, fmt.Errorf("[pod-controller] containers %v in pod [%s] are not running, "+ + "need to retry later", needRetryContainerList, pod.Name) + } + return pvcNameList, nil +} + +func (ctrl *Controller) getPvByPvcName(pvcName, namespace string) (*coreV1.PersistentVolume, error) { + pvc, err := ctrl.claimInformer.Lister().PersistentVolumeClaims(namespace).Get(pvcName) + if err != nil { + return nil, err + } + + return ctrl.volumeInformer.Lister().Get(pvc.Spec.VolumeName) +} + +func (ctrl *Controller) getRtByPvcName(pvcName, namespace string) (*apiXuanwuV1.ResourceTopology, error) { + pvc, err := ctrl.claimInformer.Lister().PersistentVolumeClaims(namespace).Get(pvcName) + if err != nil { + return nil, err + } + + pv, err := ctrl.volumeInformer.Lister().Get(pvc.Spec.VolumeName) + if err != nil { + return nil, err + } + + rtName := getResourceTopologyName(pv.Name) + return ctrl.topologyInformer.Lister().Get(rtName) +} + +func isTagExist(tags []apiXuanwuV1.Tag, podName, namespace string) bool { + for _, tag := range tags { + if tag.Name == podName && tag.Namespace == namespace { + return true + } + } + + return false +} + +func addPodTag(tags []apiXuanwuV1.Tag, podName, namespace string) []apiXuanwuV1.Tag { + for _, tag := range tags { + if tag.Name == podName && tag.Namespace == namespace { + return tags + } + } + + addTag := apiXuanwuV1.Tag{ + ResourceInfo: apiXuanwuV1.ResourceInfo{ + TypeMeta: metaV1.TypeMeta{Kind: consts.Pod, APIVersion: consts.KubernetesV1}, + Namespace: namespace, + Name: podName, + }, + } + + tags = append(tags, addTag) + return tags +} + +func deletePodTag(tags []apiXuanwuV1.Tag, podName, namespace string) []apiXuanwuV1.Tag { + for index, tag := range tags { + if tag.Name == podName && tag.Namespace == namespace { + return append(tags[:index], tags[index+1:]...) + } + } + + return tags +} + +func getPodKey(namespace, name string) string { + return namespace + "/" + name +} diff --git a/controller/resourcetopology/resourcetopology_controller.go b/controller/resourcetopology/resourcetopology_controller.go new file mode 100644 index 0000000..91a8a03 --- /dev/null +++ b/controller/resourcetopology/resourcetopology_controller.go @@ -0,0 +1,407 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package resourcetopology defines to reconcile action of resources topologies +package resourcetopology + +import ( + "context" + "fmt" + "time" + + coreV1 "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/util/wait" + informersCoreV1 "k8s.io/client-go/informers/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/workqueue" + + apiXuanwuV1 "github.com/huawei/csm/v2/client/apis/xuanwu/v1" + controllerConfig "github.com/huawei/csm/v2/config/topology" + cmiGrpc "github.com/huawei/csm/v2/grpc/lib/go/cmi" + xuanwuClient "github.com/huawei/csm/v2/pkg/client/clientset/versioned" + xuanwuClientInformers "github.com/huawei/csm/v2/pkg/client/informers/externalversions/xuanwu/v1" + "github.com/huawei/csm/v2/utils/log" +) + +// Controller defines the resourceTopology controller parameters +type Controller struct { + cmiClient *cmiGrpc.ClientSet + kubeClient kubernetes.Interface + xuanwuClient xuanwuClient.Interface + eventRecorder record.EventRecorder + reSyncPeriod time.Duration + + topologyQueue workqueue.RateLimitingInterface + topologyInformer xuanwuClientInformers.ResourceTopologyInformer + + volumeQueue workqueue.RateLimitingInterface + volumeInformer informersCoreV1.PersistentVolumeInformer + + claimInformer informersCoreV1.PersistentVolumeClaimInformer + + podQueue workqueue.RateLimitingInterface + podInformer informersCoreV1.PodInformer + podStore cache.Store +} + +// ControllerRequest is a request for new controller +type ControllerRequest struct { + CmiClient *cmiGrpc.ClientSet + KubeClient kubernetes.Interface + XuanwuClient xuanwuClient.Interface + TopologyInformer xuanwuClientInformers.ResourceTopologyInformer + VolumeInformer informersCoreV1.PersistentVolumeInformer + ClaimInformer informersCoreV1.PersistentVolumeClaimInformer + PodInformer informersCoreV1.PodInformer + ReSyncPeriod time.Duration + EventRecorder record.EventRecorder +} + +// NewController return a new ResourceTopologyController +func NewController(request ControllerRequest) *Controller { + rtRateLimiter := workqueue.NewItemExponentialFailureRateLimiter(controllerConfig.GetRtRetryBaseDelay(), + controllerConfig.GetRtRetryMaxDelay()) + pvRateLimiter := workqueue.NewItemExponentialFailureRateLimiter(controllerConfig.GetPvRetryBaseDelay(), + controllerConfig.GetPvRetryMaxDelay()) + podRateLimiter := workqueue.NewItemExponentialFailureRateLimiter(controllerConfig.GetPodRetryBaseDelay(), + controllerConfig.GetPodRetryMaxDelay()) + resourceTopologyQueueConfig := workqueue.RateLimitingQueueConfig{Name: "resourceTopology"} + volumeQueueConfig := workqueue.RateLimitingQueueConfig{Name: "persistentVolume"} + podQueueConfig := workqueue.RateLimitingQueueConfig{Name: "pod"} + + ctrl := &Controller{ + kubeClient: request.KubeClient, + xuanwuClient: request.XuanwuClient, + eventRecorder: request.EventRecorder, + reSyncPeriod: request.ReSyncPeriod, + cmiClient: request.CmiClient, + topologyQueue: workqueue.NewRateLimitingQueueWithConfig(rtRateLimiter, resourceTopologyQueueConfig), + topologyInformer: request.TopologyInformer, + volumeQueue: workqueue.NewRateLimitingQueueWithConfig(pvRateLimiter, volumeQueueConfig), + volumeInformer: request.VolumeInformer, + claimInformer: request.ClaimInformer, + podQueue: workqueue.NewRateLimitingQueueWithConfig(podRateLimiter, podQueueConfig), + podInformer: request.PodInformer, + podStore: cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc), + } + + ctrl.addEventFunc() + return ctrl +} + +func (ctrl *Controller) addEventFunc() { + ctrl.topologyInformer.Informer().AddEventHandler( + cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { ctrl.enqueueResourceTopology(obj) }, + UpdateFunc: func(oldObj, newObj interface{}) { ctrl.enqueueResourceTopology(newObj) }, + DeleteFunc: func(obj interface{}) { ctrl.enqueueResourceTopology(obj) }, + }, + ) + + ctrl.volumeInformer.Informer().AddEventHandler( + cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { ctrl.enqueuePersistentVolume(obj) }, + UpdateFunc: func(oldObj, newObj interface{}) { ctrl.enqueuePersistentVolume(newObj) }, + DeleteFunc: func(obj interface{}) { ctrl.enqueuePersistentVolume(obj) }, + }, + ) + + ctrl.claimInformer.Informer().AddEventHandler( + cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) {}, + UpdateFunc: func(oldObj, newObj interface{}) {}, + DeleteFunc: func(obj interface{}) {}, + }, + ) + + ctrl.podInformer.Informer().AddEventHandler( + cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { ctrl.enqueuePod(obj) }, + UpdateFunc: func(oldObj, newObj interface{}) { ctrl.enqueuePod(newObj) }, + DeleteFunc: func(obj interface{}) { ctrl.enqueuePod(obj) }, + }, + ) +} + +func (ctrl *Controller) enqueueResourceTopology(obj interface{}) { + if unknown, ok := obj.(cache.DeletedFinalStateUnknown); ok && unknown.Obj != nil { + obj = unknown.Obj + } + + if resourceTopology, ok := obj.(*apiXuanwuV1.ResourceTopology); ok { + if !checkResourceTopologyName(resourceTopology.Name) { + log.Debugf("unsupported prefix of resourceTopology [%s], skip to next", resourceTopology.Name) + return + } + + objName, err := cache.DeletionHandlingMetaNamespaceKeyFunc(resourceTopology) + if err != nil { + log.Errorf("fail to get key from object [%v] err: [%v]", resourceTopology, err) + return + } + + log.Infof("enqueued resourceTopology [%v] for sync", objName) + ctrl.topologyQueue.Add(objName) + } +} + +func (ctrl *Controller) enqueuePersistentVolume(obj interface{}) { + if unknown, ok := obj.(cache.DeletedFinalStateUnknown); ok && unknown.Obj != nil { + obj = unknown.Obj + } + + if pv, ok := obj.(*coreV1.PersistentVolume); ok { + if pv.Spec.CSI == nil { + log.Debugf("pv [%s] is not a csi pv, skip to next", pv.Name) + return + } + + if pv.Spec.CSI.Driver != controllerConfig.GetCSIDriverName() { + log.Debugf("pv [%s] driver [%s] is not supported, skip to next", pv.Name, pv.Spec.CSI.Driver) + return + } + + objName, err := cache.DeletionHandlingMetaNamespaceKeyFunc(pv) + if err != nil { + log.Errorf("fail to get key from object [%v] err: [%v]", pv, err) + return + } + + log.Infof("enqueued pv [%v] for sync", objName) + ctrl.volumeQueue.Add(objName) + } +} + +func (ctrl *Controller) enqueuePod(obj interface{}) { + if unknown, ok := obj.(cache.DeletedFinalStateUnknown); ok && unknown.Obj != nil { + obj = unknown.Obj + } + + if pod, ok := obj.(*coreV1.Pod); ok { + objName, err := cache.DeletionHandlingMetaNamespaceKeyFunc(pod) + if err != nil { + log.Errorf("fail to get key from object [%v] err: [%v]", pod, err) + return + } + + log.Infof("enqueued pod [%v] for sync", objName) + ctrl.podQueue.Add(objName) + } +} + +// Run defines the resourceTopology controller process +func (ctrl *Controller) Run(ctx context.Context, workers int, stopCh <-chan struct{}) { + defer ctrl.topologyQueue.ShutDown() + defer ctrl.podQueue.ShutDown() + defer ctrl.volumeQueue.ShutDown() + log.AddContext(ctx).Infoln("starting topology controller") + defer log.AddContext(ctx).Infoln("shutting down topology controller") + + if !cache.WaitForCacheSync(stopCh, + ctrl.topologyInformer.Informer().HasSynced, + ctrl.volumeInformer.Informer().HasSynced, + ctrl.claimInformer.Informer().HasSynced, + ctrl.podInformer.Informer().HasSynced) { + log.AddContext(ctx).Errorln("cannot sync caches") + return + } + + log.AddContext(ctx).Infoln("starting workers") + for i := 0; i < workers; i++ { + go wait.Until(func() { ctrl.runResourceTopologyWorker(ctx) }, time.Second, stopCh) + go wait.Until(func() { ctrl.runPersistentVolumeWorker(ctx) }, time.Second, stopCh) + go wait.Until(func() { ctrl.runPodWorker(ctx) }, time.Second, stopCh) + } + log.AddContext(ctx).Infoln("started workers") + defer log.AddContext(ctx).Infoln("shutting down workers") + if stopCh != nil { + sign := <-stopCh + log.AddContext(ctx).Infof("resourceTopology controller exited, reason: [%v]", sign) + } +} + +func (ctrl *Controller) runResourceTopologyWorker(ctx context.Context) { + for { + if processNext := ctrl.processNextResourceTopologyWorkItem(ctx); !processNext { + break + } + } +} + +func (ctrl *Controller) runPersistentVolumeWorker(ctx context.Context) { + for { + if processNext := ctrl.processNextPersistentVolumeWorkItem(ctx); !processNext { + break + } + } +} + +func (ctrl *Controller) runPodWorker(ctx context.Context) { + for { + if processNext := ctrl.processNextPodWorkItem(ctx); !processNext { + break + } + } +} + +func (ctrl *Controller) processNextResourceTopologyWorkItem(ctx context.Context) bool { + obj, shutdown := ctrl.topologyQueue.Get() + if shutdown { + log.AddContext(ctx).Infof("processNextResourceTopologyWorkItem obj: [%v], shutdown: [%v]", obj, shutdown) + return false + } + + err := ctrl.handle(ctx, obj, ctrl.topologyQueue, ctrl.handleResourceTopologyWork) + if err != nil { + log.AddContext(ctx).Errorln(err) + return false + } + return true +} + +func (ctrl *Controller) processNextPersistentVolumeWorkItem(ctx context.Context) bool { + obj, shutdown := ctrl.volumeQueue.Get() + if shutdown { + log.AddContext(ctx).Infof("processNextPersistentVolumeWorkItem obj: [%v], shutdown: [%v]", obj, shutdown) + return false + } + + err := ctrl.handle(ctx, obj, ctrl.volumeQueue, ctrl.handlePersistentVolumeWork) + if err != nil { + log.AddContext(ctx).Errorln(err) + return false + } + return true +} + +func (ctrl *Controller) processNextPodWorkItem(ctx context.Context) bool { + obj, shutdown := ctrl.podQueue.Get() + if shutdown { + log.AddContext(ctx).Infof("processNextPodWorkItem obj: [%v], shutdown: [%v]", obj, shutdown) + return false + } + + err := ctrl.handle(ctx, obj, ctrl.podQueue, ctrl.handlePodWork) + if err != nil { + log.AddContext(ctx).Errorln(err) + return false + } + return true +} + +func (ctrl *Controller) handle(ctx context.Context, obj interface{}, + queue workqueue.RateLimitingInterface, function func(ctx context.Context, key string) error) error { + defer queue.Done(obj) + var key string + var ok bool + + if key, ok = obj.(string); !ok { + queue.Forget(obj) + log.AddContext(ctx).Errorf("expected string in workqueue but got [%#v]", obj) + return nil + } + + log.AddContext(ctx).Infof("start handle object [%s]", key) + + ctx, err := log.SetRequestInfo(ctx) + if err != nil { + queue.AddRateLimited(key) + return fmt.Errorf("get requestIdCtx failed, error is [%v], requeuing key [%s]", err, key) + } + + if err = function(ctx, key); err != nil { + queue.AddRateLimited(key) + return fmt.Errorf("handle key [%s] failed: [%s], requeuing key [%s]", key, err.Error(), key) + } + + queue.Forget(obj) + log.AddContext(ctx).Infof("syncHandle object [%s] successfully", key) + return nil +} + +func (ctrl *Controller) handleResourceTopologyWork(ctx context.Context, key string) error { + _, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + log.AddContext(ctx).Errorf("invalid resource key: [%s]", key) + return nil + } + resourceTopology, err := ctrl.topologyInformer.Lister().Get(name) + if err != nil { + if apiErrors.IsNotFound(err) { + log.AddContext(ctx).Infof("resourceTopology [%s] is no longer exists, end this work", name) + return nil + } + log.AddContext(ctx).Errorf("get resourceTopology [%s] from the indexer cache failed", name) + return err + } + + // delete directly do nothing + if resourceTopology.ObjectMeta.DeletionTimestamp != nil { + return ctrl.deleteResourceTopology(ctx, resourceTopology) + } + + return ctrl.syncResourceTopology(ctx, resourceTopology) +} + +func (ctrl *Controller) handlePersistentVolumeWork(ctx context.Context, key string) error { + _, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + log.AddContext(ctx).Errorf("invalid resource key: [%s]", key) + return nil + } + pv, err := ctrl.volumeInformer.Lister().Get(name) + if err != nil { + if apiErrors.IsNotFound(err) { + log.AddContext(ctx).Infof("pv [%s] is no longer exists", name) + return ctrl.removeResourceTopology(ctx, name) + } + log.AddContext(ctx).Errorf("get pv [%s] from the indexer cache failed", name) + return err + } + + if pv.ObjectMeta.DeletionTimestamp != nil { + retryErr := fmt.Errorf("pv [%s] is deleting, wait the deletion finished to remove rt", name) + return retryErr + } + + return ctrl.syncPersistentVolume(ctx, pv) +} + +func (ctrl *Controller) handlePodWork(ctx context.Context, key string) error { + namespace, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + log.AddContext(ctx).Errorf("invalid resource key: [%s]", key) + return nil + } + pod, err := ctrl.podInformer.Lister().Pods(namespace).Get(name) + if err != nil { + if apiErrors.IsNotFound(err) { + log.AddContext(ctx).Infof("pod [%s/%s] is no longer exists", namespace, name) + return ctrl.removePodTag(ctx, key) + } + log.AddContext(ctx).Errorf("get pod [%s/%s] from the indexer cache failed", namespace, name) + return err + } + + if pod.ObjectMeta.DeletionTimestamp != nil { + retryErr := fmt.Errorf("pod [%s/%s] is deleting, "+ + "wait the deletion finished to remove rt label", namespace, name) + return retryErr + } + + return ctrl.syncPod(ctx, pod) +} diff --git a/controller/resourcetopology/resourcetopology_controller_core.go b/controller/resourcetopology/resourcetopology_controller_core.go new file mode 100644 index 0000000..03094b8 --- /dev/null +++ b/controller/resourcetopology/resourcetopology_controller_core.go @@ -0,0 +1,82 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package resourcetopology defines to reconcile action of resources topologies +package resourcetopology + +import ( + "context" + + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + apiXuanwuV1 "github.com/huawei/csm/v2/client/apis/xuanwu/v1" + "github.com/huawei/csm/v2/controller/utils/cmi" + grpc "github.com/huawei/csm/v2/grpc/lib/go/cmi" + "github.com/huawei/csm/v2/utils/log" +) + +// UpdateResourceTopologiesStatus updates resource topologies status +func (ctrl *Controller) UpdateResourceTopologiesStatus(ctx context.Context, + resourceTopologyCopy *apiXuanwuV1.ResourceTopology) (*apiXuanwuV1.ResourceTopology, error) { + resourceTopology, err := ctrl.xuanwuClient. + XuanwuV1(). + ResourceTopologies(). + UpdateStatus(ctx, resourceTopologyCopy, metaV1.UpdateOptions{}) + return resourceTopology, err +} + +// CmiCreateLabel create label by cmi grpc connection +func (ctrl *Controller) CmiCreateLabel(ctx context.Context, params *cmi.Params) error { + request := &grpc.CreateLabelRequest{ + VolumeId: params.VolumeId(), + LabelName: params.LabelName(), + Kind: params.Kind(), + } + + if params.ClusterName() != "" { + request.ClusterName = params.ClusterName() + } + + if params.Namespace() != "" { + request.Namespace = params.Namespace() + } + _, err := ctrl.cmiClient.LabelClient.CreateLabel(ctx, request) + if err != nil { + log.AddContext(ctx).Errorf("create label [%v] on storage failed: [%v]", params, err) + return err + } + + return err +} + +// CmiDeleteLabel delete label by cmi grpc connection +func (ctrl *Controller) CmiDeleteLabel(ctx context.Context, params *cmi.Params) error { + request := &grpc.DeleteLabelRequest{ + VolumeId: params.VolumeId(), + LabelName: params.LabelName(), + Kind: params.Kind(), + } + + if params.Namespace() != "" { + request.Namespace = params.Namespace() + } + + _, err := ctrl.cmiClient.LabelClient.DeleteLabel(ctx, request) + if err != nil { + log.AddContext(ctx).Errorf("delete label [%v] on storage failed: [%v]", params, err) + return err + } + + return err +} diff --git a/controller/resourcetopology/resourcetopology_controller_core_test.go b/controller/resourcetopology/resourcetopology_controller_core_test.go new file mode 100644 index 0000000..80dbdb0 --- /dev/null +++ b/controller/resourcetopology/resourcetopology_controller_core_test.go @@ -0,0 +1,77 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package resourcetopology +package resourcetopology + +import ( + "context" + "reflect" + "testing" + + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + apiXuanwuV1 "github.com/huawei/csm/v2/client/apis/xuanwu/v1" + fakeXuanwuClient "github.com/huawei/csm/v2/pkg/client/clientset/versioned/fake" +) + +func TestResourceTopologyController_UpdateResourceTopologiesStatus_Success(t *testing.T) { + // arrange + ctrl := &Controller{} + ctx := context.TODO() + rt := &apiXuanwuV1.ResourceTopology{} + want := &apiXuanwuV1.ResourceTopology{Status: apiXuanwuV1.ResourceTopologyStatus{ + Status: apiXuanwuV1.ResourceTopologyStatusNormal, + }} + + // mock + ctrl.xuanwuClient = fakeXuanwuClient.NewSimpleClientset() + + // expect + ctrl.xuanwuClient.XuanwuV1().ResourceTopologies().Create(ctx, rt, metaV1.CreateOptions{}) + + // act + got, err := ctrl.UpdateResourceTopologiesStatus(ctx, want) + + // assert + if err != nil { + t.Errorf("TestResourceTopologyController_UpdateResourceTopologiesStatus_Success failed: [%v]", err) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("TestResourceTopologyController_UpdateResourceTopologiesStatus_Success failed: "+ + "want: [%v],got: [%v]", want, got) + } +} + +func TestResourceTopologyController_UpdateResourceTopologiesStatus_Failed(t *testing.T) { + // arrange + ctrl := &Controller{} + ctx := context.TODO() + fakeRt := &apiXuanwuV1.ResourceTopology{Status: apiXuanwuV1.ResourceTopologyStatus{ + Status: apiXuanwuV1.ResourceTopologyStatusNormal, + }} + want := "resourcetopologies.xuanwu.huawei.io \"\" not found" + + // mock + ctrl.xuanwuClient = fakeXuanwuClient.NewSimpleClientset() + + // act + _, got := ctrl.UpdateResourceTopologiesStatus(ctx, fakeRt) + + // assert + if got == nil || got.Error() != want { + t.Errorf("TestResourceTopologyController_UpdateResourceTopologiesStatus_Success failed: "+ + "wantErr: [%v], gotErr: [%v]", want, got) + } +} diff --git a/controller/resourcetopology/resourcetopology_controller_delete.go b/controller/resourcetopology/resourcetopology_controller_delete.go new file mode 100644 index 0000000..3675ba2 --- /dev/null +++ b/controller/resourcetopology/resourcetopology_controller_delete.go @@ -0,0 +1,75 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package resourcetopology defines to reconcile action of resources topologies +package resourcetopology + +import ( + "context" + + coreV1 "k8s.io/api/core/v1" + + apiXuanwuV1 "github.com/huawei/csm/v2/client/apis/xuanwu/v1" + "github.com/huawei/csm/v2/controller/utils/consts" + "github.com/huawei/csm/v2/utils/log" +) + +func (ctrl *Controller) deleteResourceTopology(ctx context.Context, + resourceTopology *apiXuanwuV1.ResourceTopology) error { + log.AddContext(ctx).Infof("start deleteResourceTopology [%s]", resourceTopology.Name) + defer log.AddContext(ctx).Infof("end deleteResourceTopology [%s]", resourceTopology.Name) + + var err error + + // update the status of resourceTopology to Deleting + if resourceTopology.Status.Status != apiXuanwuV1.ResourceTopologyStatusDeleting { + resourceTopology, err = ctrl.updateResourceTopologyStatusPhase(ctx, + resourceTopology, apiXuanwuV1.ResourceTopologyStatusDeleting) + if err != nil { + return err + } + } + + // delete resources in status on storage + for _, tag := range resourceTopology.Status.Tags { + err = ctrl.CmiDeleteLabel(ctx, getCmiParams(resourceTopology, tag)) + if err != nil { + return err + } + } + + // reload resources in spec on cluster + for _, tag := range resourceTopology.Spec.Tags { + switch tag.Kind { + case consts.Pod: + ctrl.podQueue.Add(tag.Namespace + "/" + tag.Name) + break + case consts.PersistentVolume: + ctrl.volumeQueue.Add(tag.Name) + break + default: + log.AddContext(ctx).Errorf("unsupported tag type: [%s]", tag.Kind) + } + } + + // remove self finalizer + _, err = ctrl.deleteResourceTopologyFinalizers(ctx, resourceTopology, resourceTopologyFinalizerBySelf) + if err != nil { + ctrl.eventRecorder.Event(resourceTopology, coreV1.EventTypeWarning, + syncedFailedReason, failedUpdateResourceTopologyFinalizersMessage) + return err + } + + return nil +} diff --git a/controller/resourcetopology/resourcetopology_controller_public.go b/controller/resourcetopology/resourcetopology_controller_public.go new file mode 100644 index 0000000..ceb8a8d --- /dev/null +++ b/controller/resourcetopology/resourcetopology_controller_public.go @@ -0,0 +1,195 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package resourcetopology defines to reconcile action of resources topologies +package resourcetopology + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "time" + + coreV1 "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + apiXuanwuV1 "github.com/huawei/csm/v2/client/apis/xuanwu/v1" + "github.com/huawei/csm/v2/controller/utils" + "github.com/huawei/csm/v2/controller/utils/cmi" + "github.com/huawei/csm/v2/utils/log" +) + +const ( + updateReason = "Update" + updateFailedReason = "UpdateFailed" + syncedFailedReason = "Synced" + + failedUpdateResourceTopologyStatusPhaseMessage = "Failed to update ResourceTopology status into" + successUpdateResourceTopologyStatusPhaseMessage = "Success to update ResourceTopology status into" + failedUpdateResourceTopologyTagsFieldMessage = "Failed to update ResourceTopology tags field to" + successUpdateResourceTopologyTagsFieldMessage = "Success to update ResourceTopology tags field to" + failedUpdateResourceTopologyFinalizersMessage = "Failed to update ResourceTopology finalizers" + + resourceTopologyFinalizerBySelf = "resourcetopology.xuanwu.huawei.io/resourcetopology-protection" + + rtPrefix = "rt-" + updateRetryTimes = 10 + updateRetryPeriod = 100 * time.Millisecond + + resourceRequeueInterval = 10 * time.Second +) + +func (ctrl *Controller) updateResourceTopologyStatusPhase(ctx context.Context, + resourceTopology *apiXuanwuV1.ResourceTopology, + statusPhase apiXuanwuV1.ResourceTopologyStatusPhase) (*apiXuanwuV1.ResourceTopology, error) { + log.AddContext(ctx).Infof("update resourceTopology [%s] into status [%s]", resourceTopology.Name, statusPhase) + statusCopy := resourceTopology.Status.DeepCopy() + statusCopy.Status = statusPhase + + resourceTopologyNew, err := ctrl.updateResourceTopologyStatusStruct(ctx, resourceTopology, *statusCopy) + if err != nil { + ctrl.eventRecorder.Event(resourceTopology, coreV1.EventTypeWarning, updateFailedReason, + fmt.Sprintf("%s %s", failedUpdateResourceTopologyStatusPhaseMessage, statusPhase)) + return nil, err + } + + *resourceTopology = *resourceTopologyNew + ctrl.eventRecorder.Event(resourceTopology, coreV1.EventTypeNormal, updateReason, + fmt.Sprintf("%s %s", successUpdateResourceTopologyStatusPhaseMessage, statusPhase)) + return resourceTopology, nil +} + +func (ctrl *Controller) updateResourceTopologyStatusTagsWithRetry(ctx context.Context, + resourceTopology *apiXuanwuV1.ResourceTopology, tags []apiXuanwuV1.Tag) (*apiXuanwuV1.ResourceTopology, error) { + log.AddContext(ctx).Infof("update resourceTopology [%s] tags field in status", resourceTopology.Name) + rtCopy := resourceTopology.DeepCopy() + var err error + for attempt := 0; attempt < updateRetryTimes; attempt++ { + rtCopy.Status.Tags = tags + rtCopy, err = ctrl.xuanwuClient.XuanwuV1().ResourceTopologies(). + UpdateStatus(ctx, rtCopy, metaV1.UpdateOptions{}) + if err == nil { + ctrl.eventRecorder.Event(rtCopy, coreV1.EventTypeNormal, updateReason, + fmt.Sprintf("%s %v", successUpdateResourceTopologyTagsFieldMessage, tags)) + return rtCopy, nil + } + + if !apiErrors.IsConflict(err) { + ctrl.eventRecorder.Event(resourceTopology, coreV1.EventTypeWarning, updateFailedReason, + fmt.Sprintf("%s %v", failedUpdateResourceTopologyTagsFieldMessage, tags)) + return nil, err + } + + log.AddContext(ctx).Infof("conflict when trying to update resourceTopology [%s], need to try again", + resourceTopology.Name) + time.Sleep(updateRetryPeriod) + rtCopy, err = ctrl.xuanwuClient.XuanwuV1().ResourceTopologies(). + Get(ctx, resourceTopology.Name, metaV1.GetOptions{}) + if err != nil { + return nil, err + } + } + + return nil, fmt.Errorf("too many conflicts when trying to update resourceTopology [%s]", + resourceTopology.Name) +} + +func (ctrl *Controller) updateResourceTopologyStatusStruct(ctx context.Context, + resourceTopology *apiXuanwuV1.ResourceTopology, + status apiXuanwuV1.ResourceTopologyStatus) (*apiXuanwuV1.ResourceTopology, error) { + resourceTopologyCopy := resourceTopology.DeepCopy() + resourceTopologyCopy.Status = status + + resourceTopologyNew, err := ctrl.UpdateResourceTopologiesStatus(ctx, resourceTopologyCopy) + if err != nil { + errMsg := fmt.Sprintf("update resourceTopology [%s] status struct failed: [%v]", resourceTopology.Name, err) + log.AddContext(ctx).Errorln(errMsg) + return nil, errors.New(errMsg) + } + + *resourceTopology = *resourceTopologyNew + log.AddContext(ctx).Infof("update resourceTopology [%s] status struct succeed, the new status struct is [%v]", + resourceTopology.Name, resourceTopology.Status) + return resourceTopology, nil +} + +func (ctrl *Controller) addResourceTopologyFinalizers(ctx context.Context, + resourceTopology *apiXuanwuV1.ResourceTopology, target string) (*apiXuanwuV1.ResourceTopology, error) { + finalizers := resourceTopology.Finalizers + if utils.Contains(finalizers, target) { + return resourceTopology, nil + } + + resourceTopologyCopy := resourceTopology.DeepCopy() + resourceTopologyCopy.Finalizers = append(finalizers, target) + + resourceTopologyNew, err := ctrl.xuanwuClient.XuanwuV1().ResourceTopologies(). + Update(ctx, resourceTopologyCopy, metaV1.UpdateOptions{}) + if err != nil { + return nil, fmt.Errorf("add resourceTopology [%s] finalizers failed, errors is [%v]", + resourceTopology.Name, err) + } + + log.AddContext(ctx).Infof("add resourceTopology [%s] finalizers succeed, the new finalizers is [%v]", + resourceTopology.Name, resourceTopologyNew.Finalizers) + + return resourceTopologyNew, nil +} + +func (ctrl *Controller) deleteResourceTopologyFinalizers(ctx context.Context, + resourceTopology *apiXuanwuV1.ResourceTopology, target string) (*apiXuanwuV1.ResourceTopology, error) { + finalizers := resourceTopology.Finalizers + if !utils.Contains(finalizers, target) { + return resourceTopology, nil + } + + resourceTopologyCopy := resourceTopology.DeepCopy() + resourceTopologyCopy.Finalizers = utils.DeleteElementFromSlice(finalizers, target) + + resourceTopologyNew, err := ctrl.xuanwuClient.XuanwuV1().ResourceTopologies(). + Update(ctx, resourceTopologyCopy, metaV1.UpdateOptions{}) + if err != nil { + return nil, fmt.Errorf("delete resourceTopology [%s] finalizers failed, errors is [%v]", + resourceTopology.Name, err) + } + + log.AddContext(ctx).Infof("delete resourceTopology [%s] finalizers succeed, the new finalizers is [%v]", + resourceTopology.Name, resourceTopologyNew.Finalizers) + + return resourceTopologyNew, nil +} + +func getCmiParams(resourceTopology *apiXuanwuV1.ResourceTopology, tag apiXuanwuV1.Tag) *cmi.Params { + params := &cmi.Params{} + return params.SetVolumeId(resourceTopology.Spec.VolumeHandle). + SetKind(tag.Kind). + SetNamespace(tag.Namespace). + SetLabelName(tag.Name). + SetClusterName(os.Getenv("CLUSTER_NAME")) +} + +func checkResourceTopologyName(rtName string) bool { + return strings.HasPrefix(rtName, rtPrefix) +} + +func getResourceTopologyName(pvName string) string { + return rtPrefix + pvName +} + +func getPvNameByResourceTopologyName(rtName string) string { + return strings.TrimPrefix(rtName, rtPrefix) +} diff --git a/controller/resourcetopology/resourcetopology_controller_sync.go b/controller/resourcetopology/resourcetopology_controller_sync.go new file mode 100644 index 0000000..9224f25 --- /dev/null +++ b/controller/resourcetopology/resourcetopology_controller_sync.go @@ -0,0 +1,495 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package resourcetopology defines to reconcile action of resources topologies +package resourcetopology + +import ( + "context" + "errors" + "fmt" + "reflect" + "sort" + + coreV1 "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + apiXuanwuV1 "github.com/huawei/csm/v2/client/apis/xuanwu/v1" + controller "github.com/huawei/csm/v2/config/topology" + innerTag "github.com/huawei/csm/v2/controller/resource" + "github.com/huawei/csm/v2/controller/utils" + "github.com/huawei/csm/v2/controller/utils/cmi" + "github.com/huawei/csm/v2/controller/utils/consts" + grpc "github.com/huawei/csm/v2/grpc/lib/go/cmi" + "github.com/huawei/csm/v2/utils/log" +) + +type provisioner struct { + provider string + capability map[string]bool +} + +var ( + cmiProvisioner provisioner +) + +func (ctrl *Controller) syncResourceTopology(ctx context.Context, + resourceTopology *apiXuanwuV1.ResourceTopology) error { + log.AddContext(ctx).Infof("start to sync resourceTopology [%s]", resourceTopology.Name) + defer log.AddContext(ctx).Infof("finished sync resourceTopology [%s]", resourceTopology.Name) + + resourceTopologyNew, err := ctrl.addResourceTopologyFinalizers(ctx, + resourceTopology, resourceTopologyFinalizerBySelf) + if err != nil { + ctrl.eventRecorder.Event(resourceTopology, coreV1.EventTypeWarning, + syncedFailedReason, failedUpdateResourceTopologyFinalizersMessage) + return err + } + + addList, delList := getChangeList(resourceTopologyNew) + if len(addList) != 0 || len(delList) != 0 { + err = ctrl.provisionerCheck(ctx, resourceTopologyNew) + if err != nil { + return err + } + + log.AddContext(ctx).Infof("new tags [%v], delete tags [%v]", addList, delList) + resourceTopologyNew, err = ctrl.handlePendingStatus(ctx, resourceTopologyNew, delList, addList) + if err != nil { + return err + } + return nil + } + + if resourceTopologyNew.Status.Status != apiXuanwuV1.ResourceTopologyStatusNormal { + resourceTopologyNew, err = ctrl.updateResourceTopologyStatusPhase(ctx, resourceTopologyNew, + apiXuanwuV1.ResourceTopologyStatusNormal) + if err != nil { + return err + } + } + + // check resources + return ctrl.checkResourceTopology(ctx, resourceTopologyNew) +} + +func (ctrl *Controller) provisionerCheck(ctx context.Context, + resourceTopology *apiXuanwuV1.ResourceTopology) error { + // check if using right provisioner name + err := ctrl.checkProvisionerName(ctx, resourceTopology) + if err != nil { + return err + } + + // check if provisioner supports labels capability + err = ctrl.checkProvisionerCapability(ctx) + if err != nil { + return err + } + return nil +} + +func (ctrl *Controller) checkProvisionerName(ctx context.Context, + resourceTopology *apiXuanwuV1.ResourceTopology) error { + if cmiProvisioner.provider == "" { + info, err := ctrl.cmiClient.IdentityClient.GetProvisionerInfo(ctx, &grpc.GetProviderInfoRequest{}) + if err != nil { + return fmt.Errorf("error getting provisioner info: [%v]", err) + } + + cmiProvisioner.provider = info.Provider + } + + if resourceTopology.Spec.Provisioner == cmiProvisioner.provider { + return nil + } + + return fmt.Errorf("provider not correct, in resourceTopology is [%s], from cmi got: [%s]", + resourceTopology.Spec.Provisioner, cmiProvisioner.provider) +} + +func (ctrl *Controller) checkProvisionerCapability(ctx context.Context) error { + if cmiProvisioner.capability == nil || len(cmiProvisioner.capability) == 0 { + cmiProvisioner.capability = make(map[string]bool) + capabilities, err := ctrl.cmiClient.IdentityClient.GetProviderCapabilities(ctx, + &grpc.GetProviderCapabilitiesRequest{}) + if err != nil { + return errors.New("error getting provider capabilities") + } + + for _, capability := range capabilities.GetCapabilities() { + cmiProvisioner.capability[grpc.ProviderCapability_Type_name[int32(capability.Type)]] = true + } + } + + if cmiProvisioner.capability[grpc.ProviderCapability_Type_name[int32( + grpc.ProviderCapability_ProviderCapability_Label_Service)]] { + return nil + } + + return errors.New("cmi unsupported label capability") +} + +func (ctrl *Controller) handlePendingStatus(ctx context.Context, + resourceTopology *apiXuanwuV1.ResourceTopology, + delList []apiXuanwuV1.Tag, addList []apiXuanwuV1.Tag) (*apiXuanwuV1.ResourceTopology, error) { + var err error + resourceTopology, err = ctrl.updateResourceTopologyStatusPhase(ctx, resourceTopology, + apiXuanwuV1.ResourceTopologyStatusPending) + if err != nil { + return nil, err + } + + if len(delList) != 0 { + resourceTopology, err = ctrl.handleDeleteTags(ctx, resourceTopology, delList) + if err != nil { + return nil, err + } + } + if len(addList) != 0 { + resourceTopology, err = ctrl.handleAddTags(ctx, resourceTopology, addList) + if err != nil { + return nil, err + } + } + return resourceTopology, nil +} + +func (ctrl *Controller) handleAddTags(ctx context.Context, resourceTopology *apiXuanwuV1.ResourceTopology, + addList []apiXuanwuV1.Tag) (*apiXuanwuV1.ResourceTopology, error) { + log.AddContext(ctx).Infof("start to add tags [%v] to resourceTopology [%s]", addList, resourceTopology.Name) + defer log.AddContext(ctx).Infof("finished add tags [%v] to resourceTopology [%s]", + addList, resourceTopology.Name) + + var err error + for _, tag := range addList { + log.AddContext(ctx).Infof("trying to add tag [%v]", tag) + err = ctrl.CmiCreateLabel(ctx, getCmiParams(resourceTopology, tag)) + if err != nil { + return nil, err + } + } + + statusTags := append(resourceTopology.Status.Tags, addList...) + resourceTopology, err = ctrl.updateResourceTopologyStatusTagsWithRetry(ctx, resourceTopology, statusTags) + if err != nil { + return nil, err + } + + return resourceTopology, nil +} + +func (ctrl *Controller) rollBack(ctx context.Context, + resourceTopology *apiXuanwuV1.ResourceTopology, tag apiXuanwuV1.Tag) { + log.AddContext(ctx).Infof("rolling back resource topology tag [%v]", tag) + rollBackErr := ctrl.CmiDeleteLabel(ctx, getCmiParams(resourceTopology, tag)) + if rollBackErr != nil { + log.AddContext(ctx).Errorf("roll back label on storage err: [%v]", rollBackErr) + } +} + +func (ctrl *Controller) addTagToSlice(ctx context.Context, tags []apiXuanwuV1.Tag, + tag apiXuanwuV1.Tag, params *cmi.Params) ([]apiXuanwuV1.Tag, error) { + if contains := utils.Contains(controller.GetSupportResources(), tag.Kind); !contains { + log.Infof("tag kind [%s] is unsupported or already added, skipping", tag.Kind) + return tags, nil + } + + inner, err := innerTag.NewInnerTag(tag.TypeMeta) + if err != nil { + log.AddContext(ctx).Errorf("new inner tag failed: [%v]", err) + return tags, err + } + inner.InitFromResourceInfo(tag.ResourceInfo) + + // try to add owner of the resource + if inner.HasOwner() { + ownerInfo, tags, err := ctrl.addOwnerTag(ctx, tags, inner, params) + if err != nil { + return tags, err + } + if utils.Contains(controller.GetSupportResources(), ownerInfo.Kind) { + tag.Owner = ownerInfo + } + } + + // check resource enable added + if err := addResourceCheck(inner); err != nil { + return tags, err + } + + err = ctrl.CmiCreateLabel(ctx, params) + if err != nil { + return nil, err + } + + tags = append(tags, tag) + return tags, nil +} + +func (ctrl *Controller) addOwnerTag(ctx context.Context, tags []apiXuanwuV1.Tag, + inner innerTag.InnerTag, params *cmi.Params) (apiXuanwuV1.ResourceInfo, []apiXuanwuV1.Tag, error) { + ownerInfo, err := getOwnerInfo(inner) + if err != nil { + return apiXuanwuV1.ResourceInfo{}, tags, err + } + // if the owner kind is not supported, kind in owner info will be empty + if ownerInfo.Kind == "" { + return apiXuanwuV1.ResourceInfo{}, tags, nil + } + + tags, err = ctrl.addTagToSlice(ctx, tags, apiXuanwuV1.Tag{ResourceInfo: ownerInfo}, params) + if err != nil { + return apiXuanwuV1.ResourceInfo{}, tags, err + } + return ownerInfo, tags, nil +} + +func (ctrl *Controller) handleDeleteTags(ctx context.Context, + resourceTopology *apiXuanwuV1.ResourceTopology, + delList []apiXuanwuV1.Tag) (*apiXuanwuV1.ResourceTopology, error) { + log.AddContext(ctx).Infof("start to remove tags [%v] from resourceTopology [%s]", + delList, resourceTopology.Name) + defer log.AddContext(ctx).Infof("finished remove tags [%v] from resourceTopology [%s]", + delList, resourceTopology.Name) + + statusTags := resourceTopology.Status.Tags + + var err error + for _, tag := range delList { + log.AddContext(ctx).Infof("trying to delete tag [%v]", tag) + err = ctrl.CmiDeleteLabel(ctx, getCmiParams(resourceTopology, tag)) + if err != nil { + return nil, err + } + statusTags = deleteTag(statusTags, tag) + } + + resourceTopology, err = ctrl.updateResourceTopologyStatusTagsWithRetry(ctx, resourceTopology, statusTags) + if err != nil { + return nil, err + } + + return resourceTopology, nil +} + +func (ctrl *Controller) deleteTagFromSlice(ctx context.Context, tags []apiXuanwuV1.Tag, + tag apiXuanwuV1.Tag, params *cmi.Params) ([]apiXuanwuV1.Tag, error) { + inner, err := innerTag.NewInnerTag(tag.TypeMeta) + if err != nil { + log.AddContext(ctx).Errorf("new inner tag failed: [%v]", err) + return tags, err + } + + inner.InitFromResourceInfo(tag.ResourceInfo) + if inner.HasOwner() { + tags, err = ctrl.deleteOwnerTag(ctx, tags, inner, params) + if err != nil { + return tags, err + } + } + + if inner.Exists() && !inner.IsDeleting() { + return tags, fmt.Errorf("resource of tag [%v] still exist but not in deleting status", inner) + } + + err = utils.RetryFunc(func() (bool, error) { + if inner.Exists() && inner.IsDeleting() { + return false, fmt.Errorf("resource [%s] still deleting, check again later", tag) + } + return true, nil + }, consts.RetryTimes, consts.RetryDurationInit, consts.RetryDurationMax) + if err != nil { + return tags, fmt.Errorf("delete tag [%v] failed: [%v]", tag, err) + } + + err = ctrl.CmiDeleteLabel(ctx, params) + if err != nil { + return tags, err + } + + tags = deleteTag(tags, tag) + + return tags, nil +} + +func (ctrl *Controller) deleteOwnerTag(ctx context.Context, tags []apiXuanwuV1.Tag, + inner innerTag.InnerTag, params *cmi.Params) ([]apiXuanwuV1.Tag, error) { + ownerInfo, err := getOwnerInfo(inner) + if err != nil { + return tags, err + } + // if the owner kind is not supported, kind in owner info will be empty + if ownerInfo.Kind == "" { + return tags, nil + } + + tags, err = ctrl.deleteTagFromSlice(ctx, tags, apiXuanwuV1.Tag{ResourceInfo: ownerInfo}, params) + if err != nil { + return tags, err + } + return tags, nil +} + +func (ctrl *Controller) checkResourceTopology(ctx context.Context, + resourceTopology *apiXuanwuV1.ResourceTopology) error { + log.AddContext(ctx).Infof("start to check resources of resourceTopology [%s]", resourceTopology.Name) + defer log.AddContext(ctx).Infof("finished check resources of resourceTopology [%s]", resourceTopology.Name) + + // check whether pv exists + pvName := getPvNameByResourceTopologyName(resourceTopology.Name) + _, err := ctrl.volumeInformer.Lister().Get(pvName) + if apiErrors.IsNotFound(err) { + log.Errorf("pv [%s] is not exists, start to delete resourceTopology [%s]", pvName, resourceTopology.Name) + return ctrl.xuanwuClient.XuanwuV1().ResourceTopologies(). + Delete(ctx, resourceTopology.Name, metaV1.DeleteOptions{}) + } + + if err != nil { + return err + } + + // check whether pods exist + tags := resourceTopology.Spec.Tags + for _, tag := range resourceTopology.Status.Tags { + if tag.Kind == consts.PersistentVolume { + continue + } + + exist, err := ctrl.isPodExisted(ctx, tag.ResourceInfo) + if err != nil { + return err + } + + if !exist { + tags = deleteTag(tags, tag) + } + } + + if reflect.DeepEqual(resourceTopology.Spec.Tags, tags) { + return nil + } + + // update resourceTopology spec tags + resourceTopology.Spec.Tags = tags + _, err = ctrl.xuanwuClient.XuanwuV1().ResourceTopologies().Update(ctx, resourceTopology, metaV1.UpdateOptions{}) + + return err +} + +func (ctrl *Controller) isPodExisted(ctx context.Context, resourceInfo apiXuanwuV1.ResourceInfo) (bool, error) { + if resourceInfo.Kind != consts.Pod { + return false, fmt.Errorf("unsupported resource tag type [%s]", resourceInfo.Kind) + } + + _, err := ctrl.podInformer.Lister().Pods(resourceInfo.Namespace).Get(resourceInfo.Name) + if apiErrors.IsNotFound(err) { + log.AddContext(ctx).Errorf("pod [%s] is not existed, try to delete tag", resourceInfo.Name) + return false, nil + } + + if err != nil { + return false, err + } + + return true, nil +} + +func getOwnerInfo(inner innerTag.InnerTag) (apiXuanwuV1.ResourceInfo, error) { + ownerInnerTag, err := inner.GetOwner() + if err != nil { + return apiXuanwuV1.ResourceInfo{}, err + } + if ownerInnerTag == nil { + return apiXuanwuV1.ResourceInfo{}, nil + } + ownerInfo := ownerInnerTag.ToResourceInfo() + return ownerInfo, nil +} + +func deleteTag(tags []apiXuanwuV1.Tag, target apiXuanwuV1.Tag) []apiXuanwuV1.Tag { + idx := 0 + for i, tag := range tags { + if target.ResourceInfo == tag.ResourceInfo { + idx = i + break + } + } + tags = append(tags[:idx], tags[idx+1:]...) + return tags +} + +func addResourceCheck(inner innerTag.InnerTag) error { + if !inner.Exists() { + return fmt.Errorf("resource of tag [%v] does not exist", inner) + } + + if !inner.Ready() { + return fmt.Errorf("resource of tag [%v] not ready", inner) + } + + if inner.IsDeleting() { + return fmt.Errorf("resource of tag [%v] is deleting", inner) + } + return nil +} + +func getChangeList(topology *apiXuanwuV1.ResourceTopology) ([]apiXuanwuV1.Tag, []apiXuanwuV1.Tag) { + spec := getPodAndPvTags(topology.Spec.Tags) + status := getPodAndPvTags(topology.Status.Tags) + return getAddTagsList(status, spec), getDeleteTagsList(spec, status) +} + +func getDeleteTagsList(spec []apiXuanwuV1.Tag, status []apiXuanwuV1.Tag) []apiXuanwuV1.Tag { + return getChangedTags(spec, status) +} + +func getAddTagsList(status []apiXuanwuV1.Tag, spec []apiXuanwuV1.Tag) []apiXuanwuV1.Tag { + add := getChangedTags(status, spec) + + // if need to add PersistentVolume label to storage, must add first + sort.Slice(add, func(i, j int) bool { + if add[i].Kind == consts.PersistentVolume { + return true + } + return false + }) + return add +} + +func getChangedTags(origin []apiXuanwuV1.Tag, newList []apiXuanwuV1.Tag) []apiXuanwuV1.Tag { + set := make(map[apiXuanwuV1.ResourceInfo]struct{}) + for _, tag := range origin { + set[tag.ResourceInfo] = struct{}{} + } + + var add []apiXuanwuV1.Tag + for _, tag := range newList { + if _, ok := set[tag.ResourceInfo]; !ok { + add = append(add, tag) + } + } + return add +} + +func getPodAndPvTags(tags []apiXuanwuV1.Tag) []apiXuanwuV1.Tag { + var result []apiXuanwuV1.Tag + for _, tag := range tags { + if tag.Kind == consts.Pod || tag.Kind == consts.PersistentVolume { + result = append(result, apiXuanwuV1.Tag{ResourceInfo: tag.ResourceInfo}) + } + } + return result +} diff --git a/controller/resourcetopology/resourcetopology_controller_sync_test.go b/controller/resourcetopology/resourcetopology_controller_sync_test.go new file mode 100644 index 0000000..b93a88b --- /dev/null +++ b/controller/resourcetopology/resourcetopology_controller_sync_test.go @@ -0,0 +1,132 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package resourcetopology +package resourcetopology + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + + apiXuanwuV1 "github.com/huawei/csm/v2/client/apis/xuanwu/v1" + fakeXuanwuClient "github.com/huawei/csm/v2/pkg/client/clientset/versioned/fake" +) + +const defaultBufferSize = 2048 + +func TestResourceTopologyController_syncResourceTopology_SuccessOnNoChange(t *testing.T) { + // arrange + fakeClient := fakeXuanwuClient.NewSimpleClientset() + ctrl := &Controller{xuanwuClient: fakeClient, eventRecorder: record.NewFakeRecorder(defaultBufferSize)} + ctx := context.TODO() + rt := &apiXuanwuV1.ResourceTopology{ObjectMeta: metaV1.ObjectMeta{Name: "fakeResourcesTopology"}, + Spec: apiXuanwuV1.ResourceTopologySpec{Provisioner: "fakeProvisioner", VolumeHandle: "fakeVolumeHandle", + Tags: []apiXuanwuV1.Tag{{ResourceInfo: apiXuanwuV1.ResourceInfo{TypeMeta: metaV1.TypeMeta{ + Kind: "PersistentVolume", APIVersion: "v1"}, Name: "fakePersistentVolume"}}}}, + Status: apiXuanwuV1.ResourceTopologyStatus{Tags: []apiXuanwuV1.Tag{{ResourceInfo: apiXuanwuV1.ResourceInfo{ + TypeMeta: metaV1.TypeMeta{Kind: "PersistentVolume", APIVersion: "v1"}, Name: "fakePersistentVolume"}}}}} + + // mock + fakeClient.XuanwuV1().ResourceTopologies().Create(ctx, rt, metaV1.CreateOptions{}) + mock := gomonkey.NewPatches() + mock.ApplyPrivateMethod(ctrl, "checkResourceTopology", + func(ctx context.Context, resourceTopology *apiXuanwuV1.ResourceTopology) error { + return nil + }) + + // act + err := ctrl.syncResourceTopology(ctx, rt) + + // assert + if err != nil { + t.Errorf("TestResourceTopologyController_syncResourceTopology_SuccessOnNoChange failed: [%v]", err) + } + + // cleanup + t.Cleanup(func() { + fakeClient.XuanwuV1().ResourceTopologies().Delete(ctx, rt.Name, metaV1.DeleteOptions{}) + mock.Reset() + }) +} + +func TestResourceTopologyController_syncResourceTopology_FailOnRtNotExist(t *testing.T) { + // arrange + fakeClient := fakeXuanwuClient.NewSimpleClientset() + ctrl := &Controller{xuanwuClient: fakeClient, eventRecorder: record.NewFakeRecorder(defaultBufferSize)} + ctx := context.TODO() + rt := &apiXuanwuV1.ResourceTopology{ObjectMeta: metaV1.ObjectMeta{Name: "fakeResourcesTopology"}, + Spec: apiXuanwuV1.ResourceTopologySpec{Provisioner: "fakeProvisioner", VolumeHandle: "fakeVolumeHandle", + Tags: []apiXuanwuV1.Tag{{ResourceInfo: apiXuanwuV1.ResourceInfo{TypeMeta: metaV1.TypeMeta{ + Kind: "PersistentVolume", APIVersion: "v1"}, Name: "fakePersistentVolume"}}}}, + Status: apiXuanwuV1.ResourceTopologyStatus{Tags: []apiXuanwuV1.Tag{{ResourceInfo: apiXuanwuV1.ResourceInfo{ + TypeMeta: metaV1.TypeMeta{Kind: "PersistentVolume", APIVersion: "v1"}, Name: "fakePersistentVolume"}}}}} + wantErr := errors.New("add resourceTopology [fakeResourcesTopology] finalizers failed, " + + "errors is [resourcetopologies.xuanwu.huawei.io \"fakeResourcesTopology\" not found]") + + // act + err := ctrl.syncResourceTopology(ctx, rt) + + // assert + if !reflect.DeepEqual(err, wantErr) { + t.Errorf("TestResourceTopologyController_syncResourceTopology_FailOnRtNotExist "+ + "failed: want :[%v], got: [%v]", wantErr, err) + } +} + +func TestResourceTopologyController_syncResourceTopology_StatusToNormal(t *testing.T) { + // arrange + fakeClient := fakeXuanwuClient.NewSimpleClientset() + ctrl := &Controller{xuanwuClient: fakeClient, eventRecorder: record.NewFakeRecorder(defaultBufferSize)} + ctx := context.TODO() + rt := &apiXuanwuV1.ResourceTopology{ObjectMeta: metaV1.ObjectMeta{Name: "fakeResourcesTopology"}, + Spec: apiXuanwuV1.ResourceTopologySpec{Provisioner: "fakeProvisioner", VolumeHandle: "fakeVolumeHandle", + Tags: []apiXuanwuV1.Tag{{ResourceInfo: apiXuanwuV1.ResourceInfo{TypeMeta: metaV1.TypeMeta{ + Kind: "PersistentVolume", APIVersion: "v1"}, Name: "fakePersistentVolume"}}}}, + Status: apiXuanwuV1.ResourceTopologyStatus{Status: apiXuanwuV1.ResourceTopologyStatusPending, + Tags: []apiXuanwuV1.Tag{{ResourceInfo: apiXuanwuV1.ResourceInfo{TypeMeta: metaV1.TypeMeta{ + Kind: "PersistentVolume", APIVersion: "v1"}, Name: "fakePersistentVolume"}}}}} + + // mock + rt, _ = fakeClient.XuanwuV1().ResourceTopologies().Create(ctx, rt, metaV1.CreateOptions{}) + mock := gomonkey.NewPatches() + mock.ApplyPrivateMethod(ctrl, "checkResourceTopology", + func(ctx context.Context, resourceTopology *apiXuanwuV1.ResourceTopology) error { + return nil + }) + + // act + err := ctrl.syncResourceTopology(ctx, rt) + + // assert + if err != nil { + t.Errorf("TestResourceTopologyController_syncResourceTopology_StatusToNormal failed: [%v]", err) + } + rt, _ = fakeClient.XuanwuV1().ResourceTopologies().Get(ctx, rt.Name, metaV1.GetOptions{}) + if rt.Status.Status != apiXuanwuV1.ResourceTopologyStatusNormal { + t.Errorf("TestResourceTopologyController_syncResourceTopology_StatusToNormal failed: "+ + "want status: [%s], got status: [%s]", apiXuanwuV1.ResourceTopologyStatusNormal, rt.Status.Status) + } + + // cleanup + t.Cleanup(func() { + fakeClient.XuanwuV1().ResourceTopologies().Delete(ctx, rt.Name, metaV1.DeleteOptions{}) + mock.Reset() + }) +} diff --git a/controller/utils/client_set.go b/controller/utils/client_set.go new file mode 100644 index 0000000..0d67c15 --- /dev/null +++ b/controller/utils/client_set.go @@ -0,0 +1,204 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package utils is a package that provides utilities for controllers +package utils + +import ( + "fmt" + + apiV1 "k8s.io/api/core/v1" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + coreV1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/record" + + cmiGrpc "github.com/huawei/csm/v2/grpc/lib/go/cmi" + xuanwuClient "github.com/huawei/csm/v2/pkg/client/clientset/versioned" + "github.com/huawei/csm/v2/utils/log" +) + +// ClientsSet contains all clients needed by controller +type ClientsSet struct { + Config *rest.Config + CmiClient *cmiGrpc.ClientSet + KubeClient kubernetes.Interface + XuanwuClient xuanwuClient.Interface + DynamicClient dynamic.Interface + EventBroadcaster record.EventBroadcaster + EventRecorder record.EventRecorder + CmiAddress string +} + +const ( + eventComponentName = "huawei-csm" +) + +var ( + initFuncList = []func(*ClientsSet) error{ + initKubeClient, + initXuanwuClient, + initDynamicClient, + initEventBroadcaster, + initEventRecorder, + initCmiClient, + } +) + +// NewClientsSet creates a new clients set with the given kube config +func NewClientsSet(config string, cmiAddress string) (*ClientsSet, error) { + var kubeConfig *rest.Config + var err error + if config != "" { + kubeConfig, err = clientcmd.BuildConfigFromFlags("", config) + } else { + kubeConfig, err = rest.InClusterConfig() + } + if err != nil { + log.Errorf("getting kubeConfig [%s] err: [%v]", config, err) + return nil, err + } + + clientsSet := &ClientsSet{} + clientsSet.Config = kubeConfig + clientsSet.CmiAddress = cmiAddress + + for _, initFunction := range initFuncList { + err := initFunction(clientsSet) + if err != nil { + return nil, err + } + } + + return clientsSet, nil +} + +func initKubeClient(c *ClientsSet) error { + log.Infoln("initial kubernetes client") + defer log.Infoln("initial kubernetes client success") + if c.KubeClient != nil { + return nil + } + + kubeClient, err := kubernetes.NewForConfig(c.Config) + if err != nil { + log.Errorf("init kubernetes client error: [%v]", err) + return err + } + + c.KubeClient = kubeClient + return nil +} + +func initXuanwuClient(c *ClientsSet) error { + log.Infoln("initial xuanwu client") + defer log.Infoln("initial xuanwu client success") + if c.XuanwuClient != nil { + return nil + } + + client, err := xuanwuClient.NewForConfig(c.Config) + if err != nil { + log.Errorf("init xuanwu client error: [%v]", err) + return err + } + + c.XuanwuClient = client + return nil +} + +func initDynamicClient(c *ClientsSet) error { + log.Infoln("initial dynamic client") + if c.DynamicClient != nil { + return nil + } + + client, err := dynamic.NewForConfig(c.Config) + if err != nil { + log.Errorf("init dynamic client error: [%v]", err) + return err + } + + c.DynamicClient = client + log.Infoln("initial dynamic client success") + return nil +} + +func initEventBroadcaster(c *ClientsSet) error { + log.Infoln("initial event broadcaster") + if c.EventBroadcaster != nil { + return nil + } + + if c.KubeClient == nil { + client, err := kubernetes.NewForConfig(c.Config) + if err != nil { + log.Errorf("init kubernetes client error: [%v]", err) + return err + } + c.KubeClient = client + } + + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartStructuredLogging(0) + eventBroadcaster.StartRecordingToSink(&coreV1.EventSinkImpl{Interface: c.KubeClient.CoreV1().Events("")}) + + c.EventBroadcaster = eventBroadcaster + log.Infoln("initial event broadcaster success") + return nil +} + +func initEventRecorder(c *ClientsSet) error { + log.Infoln("initial event recorder") + if c.EventRecorder != nil { + return nil + } + + if c.KubeClient == nil { + client, err := kubernetes.NewForConfig(c.Config) + if err != nil { + log.Errorf("init xuanwu client error: [%v]", err) + return err + } + c.KubeClient = client + } + + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartRecordingToSink( + &coreV1.EventSinkImpl{Interface: c.KubeClient.CoreV1().Events(apiV1.NamespaceAll)}) + c.EventRecorder = eventBroadcaster.NewRecorder( + scheme.Scheme, apiV1.EventSource{Component: fmt.Sprintf(eventComponentName)}) + log.Infoln("initial event recorder success") + return nil +} + +func initCmiClient(c *ClientsSet) error { + log.Infoln("initial cmi client") + if c.CmiClient != nil { + return nil + } + + cmiClientSet, err := cmiGrpc.GetClientSet(c.CmiAddress) + if err != nil { + return fmt.Errorf("error getting client set of cmi: [%v]", err) + } + cmiClientSet.Conn.Connect() + c.CmiClient = cmiClientSet + + log.Infoln("initial cmi client success") + return nil +} diff --git a/controller/utils/client_set_test.go b/controller/utils/client_set_test.go new file mode 100644 index 0000000..2240e17 --- /dev/null +++ b/controller/utils/client_set_test.go @@ -0,0 +1,331 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package utils +package utils + +import ( + "errors" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + fakeDynamicClient "k8s.io/client-go/dynamic/fake" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + + xuanwuClient "github.com/huawei/csm/v2/pkg/client/clientset/versioned" + fakeXuanwuClient "github.com/huawei/csm/v2/pkg/client/clientset/versioned/fake" +) + +func TestNewClientsSet_EmptyConfigEmptyInitFuncList_Success(t *testing.T) { + // arrange + config := "" + wantClientsSet := &ClientsSet{Config: &rest.Config{}} + + // mock + mock := gomonkey.NewPatches() + + // expect + mock.ApplyFunc(rest.InClusterConfig, func() (*rest.Config, error) { + return &rest.Config{}, nil + }).ApplyGlobalVar(&initFuncList, []func(*ClientsSet) error{}) + + // act + clients, err := NewClientsSet(config, "") + + // assert + if err != nil { + t.Errorf("TestNewClientsSet_EmptyConfig_Success failed: [%v]", err) + } + if !reflect.DeepEqual(clients, wantClientsSet) { + t.Errorf("TestNewClientsSet_EmptyConfig_Success failed, want: [%v], get [%v]", wantClientsSet, clients) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestNewClientsSet_EmptyConfig_Success(t *testing.T) { + // arrange + config := "" + simpleCsiXuanwuClient := fakeXuanwuClient.NewSimpleClientset() + simpleKubeClient := fake.NewSimpleClientset() + simpleDynamicClient := fakeDynamicClient.NewSimpleDynamicClient(scheme.Scheme) + wantClientsSet := &ClientsSet{ + Config: &rest.Config{}, + XuanwuClient: simpleCsiXuanwuClient, + KubeClient: simpleKubeClient, + DynamicClient: simpleDynamicClient, + } + + // mock + mock := gomonkey.NewPatches() + + // expect + mock.ApplyFunc(rest.InClusterConfig, func() (*rest.Config, error) { + return &rest.Config{}, nil + }).ApplyGlobalVar(&initFuncList, []func(*ClientsSet) error{ + initXuanwuClient, + initKubeClient, + initDynamicClient, + }).ApplyFunc(initXuanwuClient, func(c *ClientsSet) error { + c.XuanwuClient = simpleCsiXuanwuClient + return nil + }).ApplyFunc(initKubeClient, func(c *ClientsSet) error { + c.KubeClient = simpleKubeClient + return nil + }).ApplyFunc(initDynamicClient, func(c *ClientsSet) error { + c.DynamicClient = simpleDynamicClient + return nil + }) + + // act + clients, err := NewClientsSet(config, "") + + // assert + if err != nil { + t.Errorf("TestNewClientsSet_EmptyConfig_Success failed: %v", err) + } + if !reflect.DeepEqual(clients, wantClientsSet) { + t.Errorf("TestNewClientsSet_EmptyConfig_Success failed, want: %v,get %v", wantClientsSet, clients) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestNewClientsSet_EmptyConfigEmptyInitFuncList_Fail(t *testing.T) { + // arrange + config := "" + wantErr := errors.New("getting kubeConfig [] err: [fake error]") + + // mock + mock := gomonkey.NewPatches() + + // expect + mock.ApplyFunc(rest.InClusterConfig, func() (*rest.Config, error) { + return nil, errors.New("fake error") + }).ApplyGlobalVar(&initFuncList, []func(*ClientsSet) error{}) + + // act + clients, err := NewClientsSet(config, "/cmi/cmi.sock") + + // assert + if reflect.DeepEqual(err, wantErr) { + t.Errorf("TestNewClientsSet_EmptyConfig_Fail failed, want: %v, got: %v", wantErr, err) + } + if clients != nil { + t.Errorf("TestNewClientsSet_EmptyConfig_Fail failed want nil clients set, got: %v", clients) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestNewClientsSet_EmptyConfigInitFuncErr_Fail(t *testing.T) { + // arrange + config := "" + wantErr := errors.New("init kubernetes client error: [fake error]") + + // mock + mock := gomonkey.NewPatches() + + // expect + mock.ApplyFunc(rest.InClusterConfig, func() (*rest.Config, error) { + return &rest.Config{}, nil + }).ApplyGlobalVar(&initFuncList, []func(*ClientsSet) error{ + initKubeClient, + }).ApplyFunc(initKubeClient, func(c *ClientsSet) error { + return errors.New("fake error") + }) + + // act + clients, err := NewClientsSet(config, "/cmi/cmi.sock") + + // assert + if reflect.DeepEqual(err, wantErr) { + t.Errorf("TestNewClientsSet_EmptyConfig_Fail failed, want: %v, got: %v", wantErr, err) + } + if clients != nil { + t.Errorf("TestNewClientsSet_EmptyConfig_Fail failed want nil clients set, got: %v", clients) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func Test_initKubeClient_NilClient_Success(t *testing.T) { + // arrange + c := &ClientsSet{Config: &rest.Config{}} + want := &ClientsSet{Config: &rest.Config{}, KubeClient: &kubernetes.Clientset{}} + + // mock + mock := gomonkey.NewPatches() + + // expect + mock.ApplyFunc(kubernetes.NewForConfig, func(c *rest.Config) (*kubernetes.Clientset, error) { + return &kubernetes.Clientset{}, nil + }) + + // act + err := initKubeClient(c) + + // assert + if err != nil { + t.Errorf("Test_initKubeClient_NilClient_Success err: [%v]", err) + } + if !reflect.DeepEqual(c, want) { + t.Errorf("Test_initKubeClient_NilClient_Success failed: want: [%v], got: [%v]", want, c) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func Test_initKubeClient_NilClient_Fail(t *testing.T) { + // arrange + c := &ClientsSet{Config: &rest.Config{}} + wantErr := errors.New("fake error") + + // mock + mock := gomonkey.NewPatches() + + // expect + mock.ApplyFunc(kubernetes.NewForConfig, func(c *rest.Config) (*kubernetes.Clientset, error) { + return nil, errors.New("fake error") + }) + + // act + gotErr := initKubeClient(c) + + // assert + if !reflect.DeepEqual(gotErr, wantErr) { + t.Errorf("Test_initKubeClient_NilClient_Fail failed: wantErr: [%v], gotErr: [%v]", gotErr, wantErr) + } + if c.KubeClient != nil { + t.Error("Test_initKubeClient_NilClient_Fail failed, kube client should be nil") + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func Test_initKubeClient_WithClient_Success(t *testing.T) { + // arrange + kubeClient := &kubernetes.Clientset{} + c := &ClientsSet{Config: &rest.Config{}, KubeClient: kubeClient} + + // act + err := initKubeClient(c) + + // assert + if err != nil { + t.Errorf("Test_initKubeClient_WithClient_Success failed, err: [%v]", err) + } + if c.KubeClient != kubeClient { + t.Error("Test_initKubeClient_WithClient_Success failed, kube client changed") + } +} + +func Test_initCsiClient_NilClient_Success(t *testing.T) { + // arrange + c := &ClientsSet{Config: &rest.Config{}} + want := &ClientsSet{Config: &rest.Config{}, XuanwuClient: &xuanwuClient.Clientset{}} + + // mock + mock := gomonkey.NewPatches() + + // expect + mock.ApplyFunc(xuanwuClient.NewForConfig, func(c *rest.Config) (*xuanwuClient.Clientset, error) { + return &xuanwuClient.Clientset{}, nil + }) + + // act + err := initXuanwuClient(c) + + // assert + if err != nil { + t.Errorf("Test_initCsiClient_NilClient_Success err: [%v]", err) + } + if !reflect.DeepEqual(c, want) { + t.Errorf("Test_initCsiClient_NilClient_Success failed: want: [%v], got: [%v]", want, c) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func Test_initCsiClient_NilClient_Fail(t *testing.T) { + // arrange + c := &ClientsSet{Config: &rest.Config{}} + wantErr := errors.New("fake error") + + // mock + mock := gomonkey.NewPatches() + + // expect + mock.ApplyFunc(xuanwuClient.NewForConfig, func(c *rest.Config) (*xuanwuClient.Clientset, error) { + return nil, errors.New("fake error") + }) + + // act + gotErr := initXuanwuClient(c) + + // assert + if !reflect.DeepEqual(gotErr, wantErr) { + t.Errorf("Test_initCsiClient_NilClient_Fail failed: wantErr: [%v], gotErr: [%v]", gotErr, wantErr) + } + if c.KubeClient != nil { + t.Error("Test_initCsiClient_NilClient_Fail failed, kube client should be nil") + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func Test_initCsiClient_WithClient_Success(t *testing.T) { + // arrange + csiClient := &xuanwuClient.Clientset{} + c := &ClientsSet{Config: &rest.Config{}, XuanwuClient: csiClient} + + // act + err := initXuanwuClient(c) + + // assert + if err != nil { + t.Errorf("Test_initCsiClient_WithClient_Success failed, err: [%v]", err) + } + if c.XuanwuClient != csiClient { + t.Error("Test_initCsiClient_WithClient_Success failed, csi client changed") + } +} diff --git a/controller/utils/cmi/cmi.go b/controller/utils/cmi/cmi.go new file mode 100644 index 0000000..7c48791 --- /dev/null +++ b/controller/utils/cmi/cmi.go @@ -0,0 +1,80 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package cmi provides CreateLabel and DeleteLabel interface for cmi +package cmi + +// Params used to call cmi grpc interface +type Params struct { + volumeId string + labelName string + kind string + namespace string + clusterName string +} + +// VolumeId get volume id +func (p *Params) VolumeId() string { + return p.volumeId +} + +// LabelName get label name +func (p *Params) LabelName() string { + return p.labelName +} + +// Kind get kind +func (p *Params) Kind() string { + return p.kind +} + +// Namespace get namespace +func (p *Params) Namespace() string { + return p.namespace +} + +// ClusterName get cluster name +func (p *Params) ClusterName() string { + return p.clusterName +} + +// SetVolumeId sets volumeId field +func (p *Params) SetVolumeId(volumeId string) *Params { + p.volumeId = volumeId + return p +} + +// SetLabelName sets labelName field +func (p *Params) SetLabelName(labelName string) *Params { + p.labelName = labelName + return p +} + +// SetKind sets kind field +func (p *Params) SetKind(kind string) *Params { + p.kind = kind + return p +} + +// SetNamespace sets namespace field +func (p *Params) SetNamespace(namespace string) *Params { + p.namespace = namespace + return p +} + +// SetClusterName sets clusterName field +func (p *Params) SetClusterName(clusterName string) *Params { + p.clusterName = clusterName + return p +} diff --git a/controller/utils/consts/consts.go b/controller/utils/consts/consts.go new file mode 100644 index 0000000..faa41b7 --- /dev/null +++ b/controller/utils/consts/consts.go @@ -0,0 +1,50 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package consts constants for topology services +package consts + +import "time" + +const ( + // RetryTimes default function retry times + RetryTimes = 10 + // RetryDurationInit default function retry init duration + RetryDurationInit = 500 * time.Millisecond + // RetryDurationMax default function retry max duration + RetryDurationMax = 10 * time.Second +) + +const ( + // Pod is a tag type Pod + Pod = "Pod" + + // PersistentVolume is a tag type PersistentVolume + PersistentVolume = "PersistentVolume" + + // TopologyKind is topology resource kind + TopologyKind = "ResourceTopology" + + // KubernetesV1 is kubernetes v1 api version + KubernetesV1 = "v1" + + // XuanwuV1 is xuanwu v1 api version + XuanwuV1 = "xuanwu.huawei.io/v1" + + // AnnDynamicallyProvisioned is annotation pointed to provider + AnnDynamicallyProvisioned = "pv.kubernetes.io/provisioned-by" + + // VolumeHandleKeyLabel is label key used to search volume handle + VolumeHandleKeyLabel = "volumehandlekey" +) diff --git a/controller/utils/utils.go b/controller/utils/utils.go new file mode 100644 index 0000000..68875c3 --- /dev/null +++ b/controller/utils/utils.go @@ -0,0 +1,160 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package utils is a package that provides utilities for controllers +package utils + +import ( + "context" + "crypto/md5" + "encoding/hex" + "math/rand" + "os" + "os/signal" + "reflect" + "runtime" + "syscall" + "time" + + "github.com/pkg/errors" + admissionV1 "k8s.io/api/admission/v1" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/huawei/csm/v2/utils/log" +) + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +// GetTrueAdmissionResponse is used to get trueAdmissionResponse +func GetTrueAdmissionResponse() *admissionV1.AdmissionResponse { + return &admissionV1.AdmissionResponse{ + Allowed: true, + } +} + +// GetFalseAdmissionResponse is used to get falseAdmissionResponse with err +func GetFalseAdmissionResponse(err error) *admissionV1.AdmissionResponse { + return &admissionV1.AdmissionResponse{ + Allowed: false, + Result: &metaV1.Status{ + Message: err.Error(), + }, + } +} + +// WaitExitSignal is used to wait exits signal, components e.g. webhook, controller +func WaitExitSignal(ctx context.Context, components string) { + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, syscall.SIGINT, syscall.SIGILL, syscall.SIGKILL, syscall.SIGTERM) + stopSignal := <-signalChan + log.AddContext(ctx).Warningf("stop %s, stopSignal is [%v]", components, stopSignal) + close(signalChan) +} + +// WaitSignal stop the main when stop signals are received +func WaitSignal(ctx context.Context, signalChan chan os.Signal) { + if signalChan == nil { + log.AddContext(ctx).Errorln("the channel should not be nil") + return + } + + signal.Notify(signalChan, syscall.SIGINT, syscall.SIGILL, syscall.SIGKILL, syscall.SIGTERM) + stopSignal := <-signalChan + log.AddContext(ctx).Warningf("stop main, stopSignal is [%v]", stopSignal) +} + +// RetryFunc retry function, depend on following params and return error +func RetryFunc(function func() (bool, error), retryTimes int, + retryDurationInit, retryDurationMax time.Duration) error { + duration := retryDurationInit + name := runtime.FuncForPC(reflect.ValueOf(function).Pointer()).Name() + stop := false + var err error + for i := 0; i < retryTimes; i++ { + stop, err = function() + if stop { + return err + } + if err != nil { + log.Errorf("retry function [%s] for [%d] times failed: [%v]", name, i+1, err) + } + + time.Sleep(duration) + jitter := time.Duration(rand.Int63n(int64(duration))) + duration = (duration + jitter) * 2 + if duration > retryDurationMax { + duration = duration / 4 + } + } + + return errors.Wrap(err, "exceeded retry limit") +} + +// Contains returns true if slice s contains item target +func Contains[T comparable](s []T, target T) bool { + for _, item := range s { + if item == target { + return true + } + } + return false +} + +// DeleteElementFromSlice is used to delete element from slice +func DeleteElementFromSlice[T comparable](s []T, target T) []T { + ret := make([]T, 0, len(s)) + for _, item := range s { + if item != target { + ret = append(ret, item) + } + } + return ret +} + +// GetNameSpaceFromEnv get the namespace from the env +func GetNameSpaceFromEnv(namespaceEnv, defaultNamespace string) string { + ns := os.Getenv(namespaceEnv) + if ns == "" { + ns = defaultNamespace + } + + return ns +} + +// HasDifference check if there is difference between two slices +func HasDifference[T comparable](s1 []T, s2 []T) bool { + if len(s1) != len(s2) { + return true + } + + hMap := make(map[T]struct{}) + for _, item := range s2 { + hMap[item] = struct{}{} + } + + for _, item := range s1 { + if _, ok := hMap[item]; !ok { + return true + } + } + + return false +} + +func EncryptMD5(text string) string { + hash := md5.Sum([]byte(text)) + return hex.EncodeToString(hash[:]) +} diff --git a/controller/utils/utils_test.go b/controller/utils/utils_test.go new file mode 100644 index 0000000..09e2f82 --- /dev/null +++ b/controller/utils/utils_test.go @@ -0,0 +1,221 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package utils is a package that provides utilities for controllers +package utils + +import ( + "os" + "reflect" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + "github.com/pkg/errors" + admissionV1 "k8s.io/api/admission/v1" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestGetTrueAdmissionResponse(t *testing.T) { + // arrange + want := &admissionV1.AdmissionResponse{ + Allowed: true, + } + + // act + got := GetTrueAdmissionResponse() + + // assert + if !reflect.DeepEqual(got, want) { + t.Errorf("GetTrueAdmissionResponse failed: want [%v], got: [%v]", got, want) + } +} + +func TestGetFalseAdmissionResponse(t *testing.T) { + // arrange + err := errors.New("fake error") + want := &admissionV1.AdmissionResponse{ + Allowed: false, + Result: &metaV1.Status{ + Message: errors.New("fake error").Error(), + }, + } + + // act + got := GetFalseAdmissionResponse(err) + + // assert + if !reflect.DeepEqual(got, want) { + t.Errorf("GetFalseAdmissionResponse failed: want [%v], got: [%v]", got, want) + } +} + +func TestRetryFunc_Success(t *testing.T) { + // arrange + retryTimes := 2 + retryDurationInit := 1 * time.Second + retryDurationMax := 10 * time.Second + fakeFunc := func() (bool, error) { + return true, nil + } + + // act + err := RetryFunc(fakeFunc, retryTimes, retryDurationInit, retryDurationMax) + + // assert + if err != nil { + t.Errorf("TestRetryFunc_Success failed: [%v]", err) + } +} + +func TestRetryFunc_Fail(t *testing.T) { + // arrange + retryTimes := 2 + retryDurationInit := 1 * time.Second + retryDurationMax := 10 * time.Second + fakeFunc := func() (bool, error) { + return false, errors.New("fake error") + } + wantErr := errors.New("exceeded retry limit: fake error") + + // act + got := RetryFunc(fakeFunc, retryTimes, retryDurationInit, retryDurationMax) + + // assert + if got.Error() != wantErr.Error() { + t.Errorf("TestRetryFunc_Fail failed: wantErr: [%v], got: [%v]", wantErr, got) + } +} + +func TestContains_True(t *testing.T) { + // arrange + s := []string{"a", "b", "c", "d"} + target := "c" + + // act + ok := Contains(s, target) + + // assert + if !ok { + t.Errorf("TestContains_True: got: [%t], want: [%t]", ok, true) + } +} + +func TestContains_False(t *testing.T) { + // arrange + s := []string{"a", "b", "c", "d"} + target := "e" + + // act + ok := Contains(s, target) + + // assert + if ok { + t.Errorf("TestContains_True: got: [%t], want: [%t]", ok, false) + } +} + +func TestGetNameSpaceFromEnv_DefaultNs(t *testing.T) { + // arrange + want := "fakeNamespace" + + // mock + mock := gomonkey.NewPatches() + + // expect + mock.ApplyFunc(os.Getenv, func(s string) string { + return "fakeNamespace" + }) + + // act + got := GetNameSpaceFromEnv("", "") + + // assert + if got != want { + t.Errorf("TestGetNameSpaceFromEnv_DefaultNs: want: [%v], got: [%v]", want, got) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestGetNameSpaceFromEnv_SpecNs(t *testing.T) { + // arrange + want := "fakeNamespace" + + // mock + mock := gomonkey.NewPatches() + + // expect + mock.ApplyFunc(os.Getenv, func(s string) string { + return "" + }) + + // act + got := GetNameSpaceFromEnv("", "fakeNamespace") + + // assert + if got != want { + t.Errorf("TestGetNameSpaceFromEnv_DefaultNs: want: [%v], got: [%v]", want, got) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestHasDifference_False(t *testing.T) { + // arrange + s1 := []string{"a", "b", "c", "d"} + s2 := []string{"a", "b", "c", "d"} + + // act + got := HasDifference(s1, s2) + + // assert + if got { + t.Errorf("TestHasDifference_False: got: [%t], want: [%t]", got, false) + } +} + +func TestHasDifference_DiffLength(t *testing.T) { + // arrange + s1 := []int{1, 2, 3, 4, 5} + s2 := []int{1, 2, 3} + + // act + got := HasDifference(s1, s2) + + // assert + if !got { + t.Errorf("TestHasDifference_DiffLength: got: [%t], want: [%t]", got, true) + } +} + +func TestHasDifference_DiffEle(t *testing.T) { + // arrange + s1 := []int{1, 2, 3, 4, 5} + s2 := []int{1, 2, 3, 1, 2} + + // act + got := HasDifference(s1, s2) + + // assert + if !got { + t.Errorf("TestHasDifference_DiffEle: got: [%t], want: [%t]", got, true) + } +} diff --git a/docs/eSDK Huawei Storage Kubernetes CSM Plugins V2.1.0 User Guide 01.pdf b/docs/eSDK Huawei Storage Kubernetes CSM Plugins V2.1.0 User Guide 01.pdf new file mode 100644 index 0000000..aa26f1e Binary files /dev/null and b/docs/eSDK Huawei Storage Kubernetes CSM Plugins V2.1.0 User Guide 01.pdf differ diff --git "a/docs/eSDK Huawei Storage Kubernetes CSM Plugins V2.1.0 \347\224\250\346\210\267\346\214\207\345\215\227 01.pdf" "b/docs/eSDK Huawei Storage Kubernetes CSM Plugins V2.1.0 \347\224\250\346\210\267\346\214\207\345\215\227 01.pdf" new file mode 100644 index 0000000..961ecdd Binary files /dev/null and "b/docs/eSDK Huawei Storage Kubernetes CSM Plugins V2.1.0 \347\224\250\346\210\267\346\214\207\345\215\227 01.pdf" differ diff --git a/example/.keepdir b/example/.keepdir new file mode 100644 index 0000000..e69de29 diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..a75bcbc --- /dev/null +++ b/go.mod @@ -0,0 +1,70 @@ +module github.com/huawei/csm/v2 + +go 1.22 + +require ( + github.com/Huawei/eSDK_K8S_Plugin/v4 v4.6.0 + github.com/agiledragon/gomonkey/v2 v2.9.0 + github.com/golang/protobuf v1.5.4 + github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v1.11.1 + github.com/sirupsen/logrus v1.9.0 + github.com/spf13/cobra v1.8.0 + github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace + google.golang.org/grpc v1.57.2 + google.golang.org/protobuf v1.34.2 + k8s.io/api v0.29.2 + k8s.io/apimachinery v0.29.2 + k8s.io/client-go v0.29.2 + k8s.io/code-generator v0.27.6 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/evanphx/json-patch v5.6.0+incompatible // indirect + github.com/go-logr/logr v1.3.0 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.3 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/google/gnostic v0.5.7-v3refs // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/imdario/mergo v0.3.12 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.26.0 // indirect + github.com/prometheus/procfs v0.6.0 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/oauth2 v0.23.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/term v0.24.0 // indirect + golang.org/x/text v0.18.0 // indirect + golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01 // indirect + k8s.io/klog/v2 v2.110.1 // indirect + k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect + k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect +) diff --git a/grpc/.keepdir b/grpc/.keepdir new file mode 100644 index 0000000..e69de29 diff --git a/grpc/lib/go/cmi/client.go b/grpc/lib/go/cmi/client.go new file mode 100644 index 0000000..2b41438 --- /dev/null +++ b/grpc/lib/go/cmi/client.go @@ -0,0 +1,68 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package cmi provides grpc clients +package cmi + +import ( + "fmt" + "strings" + + "google.golang.org/grpc" + + "github.com/huawei/csm/v2/utils/log" +) + +// ClientSet provides clients to access services +type ClientSet struct { + LabelClient LabelServiceClient + CollectorClient CollectorClient + IdentityClient IdentityClient + Conn *grpc.ClientConn +} + +// GetClientSet get client set +func GetClientSet(address string) (*ClientSet, error) { + connect, err := buildGrpcConnect(address) + if err != nil { + return nil, err + } + return &ClientSet{ + LabelClient: NewLabelServiceClient(connect), + CollectorClient: NewCollectorClient(connect), + IdentityClient: NewIdentityClient(connect), + Conn: connect, + }, nil +} + +func buildGrpcConnect(address string) (*grpc.ClientConn, error) { + log.Infof("Connecting to %s", address) + + unixPrefix := "unix://" + if strings.HasPrefix(address, "/") { + // It looks like filesystem path. + address = unixPrefix + address + } + + if !strings.HasPrefix(address, unixPrefix) { + return nil, fmt.Errorf("invalid unix domain path [%s]", address) + } + + dialOptions := []grpc.DialOption{ + grpc.WithInsecure()} + + return grpc.Dial(address, dialOptions...) +} diff --git a/grpc/lib/go/cmi/cmi.pb.go b/grpc/lib/go/cmi/cmi.pb.go new file mode 100644 index 0000000..82d91a0 --- /dev/null +++ b/grpc/lib/go/cmi/cmi.pb.go @@ -0,0 +1,1610 @@ +// +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. +// +//Licensed under the Apache License, Version 2.0 (the "License"); +//you may not use this file except in compliance with the License. +//You may obtain a copy of the License at +//http://www.apache.org/licenses/LICENSE-2.0 +//Unless required by applicable law or agreed to in writing, software +//distributed under the License is distributed on an "AS IS" BASIS, +//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//See the License for the specific language governing permissions and +//limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.26.0 +// protoc v3.9.1 +// source: cmi.proto + +package cmi + +import ( + context "context" + wrappers "github.com/golang/protobuf/ptypes/wrappers" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ProviderCapability_Type int32 + +const ( + // If CMI implements ProviderCapability_Label_Service capability + // then it must implement CreateLabel RPC and DeleteLabel RPC. + ProviderCapability_ProviderCapability_Label_Service ProviderCapability_Type = 0 + // If CMI implements ProviderCapability_Label_Service capability + // then it must implement Collect RPC call for fetching storage information. + ProviderCapability_ProviderCapability_Collect_Service ProviderCapability_Type = 1 +) + +// Enum value maps for ProviderCapability_Type. +var ( + ProviderCapability_Type_name = map[int32]string{ + 0: "ProviderCapability_Label_Service", + 1: "ProviderCapability_Collect_Service", + } + ProviderCapability_Type_value = map[string]int32{ + "ProviderCapability_Label_Service": 0, + "ProviderCapability_Collect_Service": 1, + } +) + +func (x ProviderCapability_Type) Enum() *ProviderCapability_Type { + p := new(ProviderCapability_Type) + *p = x + return p +} + +func (x ProviderCapability_Type) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ProviderCapability_Type) Descriptor() protoreflect.EnumDescriptor { + return file_cmi_proto_enumTypes[0].Descriptor() +} + +func (ProviderCapability_Type) Type() protoreflect.EnumType { + return &file_cmi_proto_enumTypes[0] +} + +func (x ProviderCapability_Type) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ProviderCapability_Type.Descriptor instead. +func (ProviderCapability_Type) EnumDescriptor() ([]byte, []int) { + return file_cmi_proto_rawDescGZIP(), []int{8, 0} +} + +type CreateLabelRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // This field is REQUIRED.Value of this field is unique for a volume. + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + // This field is REQUIRED.Value of this field is label name. + LabelName string `protobuf:"bytes,2,opt,name=label_name,json=labelName,proto3" json:"label_name,omitempty"` + // This field is REQUIRED.Value of this field is label kind, e.g. Pod, PersistentVolume... + Kind string `protobuf:"bytes,3,opt,name=kind,proto3" json:"kind,omitempty"` + // This field is OPTIONAL. This allows to specify the namespace when kind is Pod + // If not specified, will use 'default' as the default namespace + Namespace string `protobuf:"bytes,4,opt,name=namespace,proto3" json:"namespace,omitempty"` + // This field is OPTIONAL.This allows to specify the cluster name when kind is PersistentVolume + ClusterName string `protobuf:"bytes,5,opt,name=cluster_name,json=clusterName,proto3" json:"cluster_name,omitempty"` + // Specific parameters passed in as opaque key-value pairs. + // This field is OPTIONAL. The CMI is responsible for parsing and validating these parameters. + Parameters map[string]string `protobuf:"bytes,6,rep,name=parameters,proto3" json:"parameters,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *CreateLabelRequest) Reset() { + *x = CreateLabelRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_cmi_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CreateLabelRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateLabelRequest) ProtoMessage() {} + +func (x *CreateLabelRequest) ProtoReflect() protoreflect.Message { + mi := &file_cmi_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateLabelRequest.ProtoReflect.Descriptor instead. +func (*CreateLabelRequest) Descriptor() ([]byte, []int) { + return file_cmi_proto_rawDescGZIP(), []int{0} +} + +func (x *CreateLabelRequest) GetVolumeId() string { + if x != nil { + return x.VolumeId + } + return "" +} + +func (x *CreateLabelRequest) GetLabelName() string { + if x != nil { + return x.LabelName + } + return "" +} + +func (x *CreateLabelRequest) GetKind() string { + if x != nil { + return x.Kind + } + return "" +} + +func (x *CreateLabelRequest) GetNamespace() string { + if x != nil { + return x.Namespace + } + return "" +} + +func (x *CreateLabelRequest) GetClusterName() string { + if x != nil { + return x.ClusterName + } + return "" +} + +func (x *CreateLabelRequest) GetParameters() map[string]string { + if x != nil { + return x.Parameters + } + return nil +} + +type CreateLabelResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Indicates if success or not + Success *wrappers.BoolValue `protobuf:"bytes,1,opt,name=success,proto3" json:"success,omitempty"` +} + +func (x *CreateLabelResponse) Reset() { + *x = CreateLabelResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_cmi_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CreateLabelResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateLabelResponse) ProtoMessage() {} + +func (x *CreateLabelResponse) ProtoReflect() protoreflect.Message { + mi := &file_cmi_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateLabelResponse.ProtoReflect.Descriptor instead. +func (*CreateLabelResponse) Descriptor() ([]byte, []int) { + return file_cmi_proto_rawDescGZIP(), []int{1} +} + +func (x *CreateLabelResponse) GetSuccess() *wrappers.BoolValue { + if x != nil { + return x.Success + } + return nil +} + +type DeleteLabelRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // This field is REQUIRED.Value of this field is unique for a volume. + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + // This field is REQUIRED.Value of this field is label name. + LabelName string `protobuf:"bytes,2,opt,name=label_name,json=labelName,proto3" json:"label_name,omitempty"` + // This field is REQUIRED.Value of this field is label kind, e.g. Pod, PersistentVolume... + Kind string `protobuf:"bytes,3,opt,name=kind,proto3" json:"kind,omitempty"` + // This field is OPTIONAL. This allows to specify the namespace when kind is Pod + // If not specified, will use 'default' as the default namespace + Namespace string `protobuf:"bytes,4,opt,name=namespace,proto3" json:"namespace,omitempty"` +} + +func (x *DeleteLabelRequest) Reset() { + *x = DeleteLabelRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_cmi_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DeleteLabelRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteLabelRequest) ProtoMessage() {} + +func (x *DeleteLabelRequest) ProtoReflect() protoreflect.Message { + mi := &file_cmi_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteLabelRequest.ProtoReflect.Descriptor instead. +func (*DeleteLabelRequest) Descriptor() ([]byte, []int) { + return file_cmi_proto_rawDescGZIP(), []int{2} +} + +func (x *DeleteLabelRequest) GetVolumeId() string { + if x != nil { + return x.VolumeId + } + return "" +} + +func (x *DeleteLabelRequest) GetLabelName() string { + if x != nil { + return x.LabelName + } + return "" +} + +func (x *DeleteLabelRequest) GetKind() string { + if x != nil { + return x.Kind + } + return "" +} + +func (x *DeleteLabelRequest) GetNamespace() string { + if x != nil { + return x.Namespace + } + return "" +} + +type DeleteLabelResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Indicates if success or not + Success *wrappers.BoolValue `protobuf:"bytes,1,opt,name=success,proto3" json:"success,omitempty"` +} + +func (x *DeleteLabelResponse) Reset() { + *x = DeleteLabelResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_cmi_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DeleteLabelResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteLabelResponse) ProtoMessage() {} + +func (x *DeleteLabelResponse) ProtoReflect() protoreflect.Message { + mi := &file_cmi_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteLabelResponse.ProtoReflect.Descriptor instead. +func (*DeleteLabelResponse) Descriptor() ([]byte, []int) { + return file_cmi_proto_rawDescGZIP(), []int{3} +} + +func (x *DeleteLabelResponse) GetSuccess() *wrappers.BoolValue { + if x != nil { + return x.Success + } + return nil +} + +// Probe request to check health/availability +type ProbeRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *ProbeRequest) Reset() { + *x = ProbeRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_cmi_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ProbeRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProbeRequest) ProtoMessage() {} + +func (x *ProbeRequest) ProtoReflect() protoreflect.Message { + mi := &file_cmi_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProbeRequest.ProtoReflect.Descriptor instead. +func (*ProbeRequest) Descriptor() ([]byte, []int) { + return file_cmi_proto_rawDescGZIP(), []int{4} +} + +// Response to indicate health/availability status +type ProbeResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Indicates if healthy/available or not + Ready *wrappers.BoolValue `protobuf:"bytes,1,opt,name=ready,proto3" json:"ready,omitempty"` +} + +func (x *ProbeResponse) Reset() { + *x = ProbeResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_cmi_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ProbeResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProbeResponse) ProtoMessage() {} + +func (x *ProbeResponse) ProtoReflect() protoreflect.Message { + mi := &file_cmi_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProbeResponse.ProtoReflect.Descriptor instead. +func (*ProbeResponse) Descriptor() ([]byte, []int) { + return file_cmi_proto_rawDescGZIP(), []int{5} +} + +func (x *ProbeResponse) GetReady() *wrappers.BoolValue { + if x != nil { + return x.Ready + } + return nil +} + +type GetProviderCapabilitiesRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *GetProviderCapabilitiesRequest) Reset() { + *x = GetProviderCapabilitiesRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_cmi_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetProviderCapabilitiesRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetProviderCapabilitiesRequest) ProtoMessage() {} + +func (x *GetProviderCapabilitiesRequest) ProtoReflect() protoreflect.Message { + mi := &file_cmi_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetProviderCapabilitiesRequest.ProtoReflect.Descriptor instead. +func (*GetProviderCapabilitiesRequest) Descriptor() ([]byte, []int) { + return file_cmi_proto_rawDescGZIP(), []int{6} +} + +type GetProviderCapabilitiesResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // All the capabilities that the CMI supports. This field is OPTIONAL. + Capabilities []*ProviderCapability `protobuf:"bytes,1,rep,name=capabilities,proto3" json:"capabilities,omitempty"` +} + +func (x *GetProviderCapabilitiesResponse) Reset() { + *x = GetProviderCapabilitiesResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_cmi_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetProviderCapabilitiesResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetProviderCapabilitiesResponse) ProtoMessage() {} + +func (x *GetProviderCapabilitiesResponse) ProtoReflect() protoreflect.Message { + mi := &file_cmi_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetProviderCapabilitiesResponse.ProtoReflect.Descriptor instead. +func (*GetProviderCapabilitiesResponse) Descriptor() ([]byte, []int) { + return file_cmi_proto_rawDescGZIP(), []int{7} +} + +func (x *GetProviderCapabilitiesResponse) GetCapabilities() []*ProviderCapability { + if x != nil { + return x.Capabilities + } + return nil +} + +type ProviderCapability struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Type ProviderCapability_Type `protobuf:"varint,1,opt,name=type,proto3,enum=cmi.v1.ProviderCapability_Type" json:"type,omitempty"` +} + +func (x *ProviderCapability) Reset() { + *x = ProviderCapability{} + if protoimpl.UnsafeEnabled { + mi := &file_cmi_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ProviderCapability) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProviderCapability) ProtoMessage() {} + +func (x *ProviderCapability) ProtoReflect() protoreflect.Message { + mi := &file_cmi_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProviderCapability.ProtoReflect.Descriptor instead. +func (*ProviderCapability) Descriptor() ([]byte, []int) { + return file_cmi_proto_rawDescGZIP(), []int{8} +} + +func (x *ProviderCapability) GetType() ProviderCapability_Type { + if x != nil { + return x.Type + } + return ProviderCapability_ProviderCapability_Label_Service +} + +type GetProviderInfoRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *GetProviderInfoRequest) Reset() { + *x = GetProviderInfoRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_cmi_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetProviderInfoRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetProviderInfoRequest) ProtoMessage() {} + +func (x *GetProviderInfoRequest) ProtoReflect() protoreflect.Message { + mi := &file_cmi_proto_msgTypes[9] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetProviderInfoRequest.ProtoReflect.Descriptor instead. +func (*GetProviderInfoRequest) Descriptor() ([]byte, []int) { + return file_cmi_proto_rawDescGZIP(), []int{9} +} + +type GetProviderInfoResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // This field is REQUIRED. Value of this field is unique name for CMI. + Provider string `protobuf:"bytes,1,opt,name=provider,proto3" json:"provider,omitempty"` +} + +func (x *GetProviderInfoResponse) Reset() { + *x = GetProviderInfoResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_cmi_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetProviderInfoResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetProviderInfoResponse) ProtoMessage() {} + +func (x *GetProviderInfoResponse) ProtoReflect() protoreflect.Message { + mi := &file_cmi_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetProviderInfoResponse.ProtoReflect.Descriptor instead. +func (*GetProviderInfoResponse) Descriptor() ([]byte, []int) { + return file_cmi_proto_rawDescGZIP(), []int{10} +} + +func (x *GetProviderInfoResponse) GetProvider() string { + if x != nil { + return x.Provider + } + return "" +} + +type CollectRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // This field is REQUIRED. Value of this field is StorageBlackClaim name. + // See StorageBlackClaim resource for details. + BackendName string `protobuf:"bytes,1,opt,name=backend_name,json=backendName,proto3" json:"backend_name,omitempty"` + // This field is REQUIRED. Value of this field is collect type. + // Indicates that type of data to be collected by Collect request. + CollectType string `protobuf:"bytes,2,opt,name=collect_type,json=collectType,proto3" json:"collect_type,omitempty"` + // This field is REQUIRED. Value of this field is metrics type. + // Allowed values: + // object: will collect object data, e.g. controller's cpu usage... + // performance: will collect performance data, e.g. controller's read bandwidth... + MetricsType string `protobuf:"bytes,3,opt,name=metrics_type,json=metricsType,proto3" json:"metrics_type,omitempty"` + // This field is REQUIRED when metrics_type is performance + Indicators []string `protobuf:"bytes,4,rep,name=indicators,proto3" json:"indicators,omitempty"` +} + +func (x *CollectRequest) Reset() { + *x = CollectRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_cmi_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CollectRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CollectRequest) ProtoMessage() {} + +func (x *CollectRequest) ProtoReflect() protoreflect.Message { + mi := &file_cmi_proto_msgTypes[11] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CollectRequest.ProtoReflect.Descriptor instead. +func (*CollectRequest) Descriptor() ([]byte, []int) { + return file_cmi_proto_rawDescGZIP(), []int{11} +} + +func (x *CollectRequest) GetBackendName() string { + if x != nil { + return x.BackendName + } + return "" +} + +func (x *CollectRequest) GetCollectType() string { + if x != nil { + return x.CollectType + } + return "" +} + +func (x *CollectRequest) GetMetricsType() string { + if x != nil { + return x.MetricsType + } + return "" +} + +func (x *CollectRequest) GetIndicators() []string { + if x != nil { + return x.Indicators + } + return nil +} + +type CollectResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // This field is REQUIRED. + // See CollectRequest's backend_name for details. + BackendName string `protobuf:"bytes,1,opt,name=backend_name,json=backendName,proto3" json:"backend_name,omitempty"` + // This field is REQUIRED. + // See CollectRequest's collect_type for details. + CollectType string `protobuf:"bytes,2,opt,name=collect_type,json=collectType,proto3" json:"collect_type,omitempty"` + // This field is REQUIRED. + // See CollectRequest's metrics_type for details. + MetricsType string `protobuf:"bytes,3,opt,name=metrics_type,json=metricsType,proto3" json:"metrics_type,omitempty"` + // The list of collected data. + Details []*CollectDetail `protobuf:"bytes,4,rep,name=details,proto3" json:"details,omitempty"` +} + +func (x *CollectResponse) Reset() { + *x = CollectResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_cmi_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CollectResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CollectResponse) ProtoMessage() {} + +func (x *CollectResponse) ProtoReflect() protoreflect.Message { + mi := &file_cmi_proto_msgTypes[12] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CollectResponse.ProtoReflect.Descriptor instead. +func (*CollectResponse) Descriptor() ([]byte, []int) { + return file_cmi_proto_rawDescGZIP(), []int{12} +} + +func (x *CollectResponse) GetBackendName() string { + if x != nil { + return x.BackendName + } + return "" +} + +func (x *CollectResponse) GetCollectType() string { + if x != nil { + return x.CollectType + } + return "" +} + +func (x *CollectResponse) GetMetricsType() string { + if x != nil { + return x.MetricsType + } + return "" +} + +func (x *CollectResponse) GetDetails() []*CollectDetail { + if x != nil { + return x.Details + } + return nil +} + +type CollectDetail struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // This field is REQUIRED. Value of this field is a map of data to be collected. + // Collected data are specified in as key-value pairs. + Data map[string]string `protobuf:"bytes,6,rep,name=data,proto3" json:"data,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *CollectDetail) Reset() { + *x = CollectDetail{} + if protoimpl.UnsafeEnabled { + mi := &file_cmi_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CollectDetail) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CollectDetail) ProtoMessage() {} + +func (x *CollectDetail) ProtoReflect() protoreflect.Message { + mi := &file_cmi_proto_msgTypes[13] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CollectDetail.ProtoReflect.Descriptor instead. +func (*CollectDetail) Descriptor() ([]byte, []int) { + return file_cmi_proto_rawDescGZIP(), []int{13} +} + +func (x *CollectDetail) GetData() map[string]string { + if x != nil { + return x.Data + } + return nil +} + +var File_cmi_proto protoreflect.FileDescriptor + +var file_cmi_proto_rawDesc = []byte{ + 0x0a, 0x09, 0x63, 0x6d, 0x69, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, 0x63, 0x6d, 0x69, + 0x2e, 0x76, 0x31, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x77, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x73, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x22, 0xb0, 0x02, 0x0a, 0x12, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4c, 0x61, + 0x62, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x76, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, 0x61, 0x62, 0x65, 0x6c, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6c, 0x61, 0x62, + 0x65, 0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, + 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, + 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x75, 0x73, + 0x74, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, + 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x4a, 0x0a, 0x0a, 0x70, + 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x2a, 0x2e, 0x63, 0x6d, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4c, + 0x61, 0x62, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x50, 0x61, 0x72, 0x61, + 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, 0x61, 0x72, + 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x1a, 0x3d, 0x0a, 0x0f, 0x50, 0x61, 0x72, 0x61, 0x6d, + 0x65, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x4b, 0x0a, 0x13, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x34, 0x0a, + 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x42, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, + 0x65, 0x73, 0x73, 0x22, 0x82, 0x01, 0x0a, 0x12, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4c, 0x61, + 0x62, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x76, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, 0x61, 0x62, 0x65, 0x6c, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6c, 0x61, 0x62, + 0x65, 0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, + 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, + 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x22, 0x4b, 0x0a, 0x13, 0x44, 0x65, 0x6c, 0x65, + 0x74, 0x65, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x34, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x42, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x07, 0x73, 0x75, + 0x63, 0x63, 0x65, 0x73, 0x73, 0x22, 0x0e, 0x0a, 0x0c, 0x50, 0x72, 0x6f, 0x62, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x41, 0x0a, 0x0d, 0x50, 0x72, 0x6f, 0x62, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x42, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x75, + 0x65, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x22, 0x20, 0x0a, 0x1e, 0x47, 0x65, 0x74, 0x50, + 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, + 0x69, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x61, 0x0a, 0x1f, 0x47, 0x65, + 0x74, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, + 0x69, 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, + 0x0c, 0x63, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x63, 0x6d, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, + 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x52, + 0x0c, 0x63, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x22, 0x9f, 0x01, + 0x0a, 0x12, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, + 0x6c, 0x69, 0x74, 0x79, 0x12, 0x33, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0e, 0x32, 0x1f, 0x2e, 0x63, 0x6d, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x76, + 0x69, 0x64, 0x65, 0x72, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x2e, 0x54, + 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x54, 0x0a, 0x04, 0x54, 0x79, 0x70, + 0x65, 0x12, 0x24, 0x0a, 0x20, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x61, 0x70, + 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x5f, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x5f, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x10, 0x00, 0x12, 0x26, 0x0a, 0x22, 0x50, 0x72, 0x6f, 0x76, 0x69, + 0x64, 0x65, 0x72, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x5f, 0x43, 0x6f, + 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x5f, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x10, 0x01, 0x22, + 0x18, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x49, 0x6e, + 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x35, 0x0a, 0x17, 0x47, 0x65, 0x74, + 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, + 0x22, 0x99, 0x01, 0x0a, 0x0e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x5f, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x62, 0x61, 0x63, 0x6b, 0x65, + 0x6e, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, + 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, + 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x6d, 0x65, 0x74, + 0x72, 0x69, 0x63, 0x73, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0b, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1e, 0x0a, 0x0a, + 0x69, 0x6e, 0x64, 0x69, 0x63, 0x61, 0x74, 0x6f, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x0a, 0x69, 0x6e, 0x64, 0x69, 0x63, 0x61, 0x74, 0x6f, 0x72, 0x73, 0x22, 0xab, 0x01, 0x0a, + 0x0f, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x21, 0x0a, 0x0c, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x4e, + 0x61, 0x6d, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x5f, 0x74, + 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6c, 0x6c, 0x65, + 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, + 0x73, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x73, 0x54, 0x79, 0x70, 0x65, 0x12, 0x2f, 0x0a, 0x07, 0x64, 0x65, 0x74, + 0x61, 0x69, 0x6c, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x63, 0x6d, 0x69, + 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x44, 0x65, 0x74, 0x61, 0x69, + 0x6c, 0x52, 0x07, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x22, 0x7d, 0x0a, 0x0d, 0x43, 0x6f, + 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x12, 0x33, 0x0a, 0x04, 0x64, + 0x61, 0x74, 0x61, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x63, 0x6d, 0x69, 0x2e, + 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, + 0x2e, 0x44, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, + 0x1a, 0x37, 0x0a, 0x09, 0x44, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x32, 0x89, 0x02, 0x0a, 0x08, 0x49, 0x64, + 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x36, 0x0a, 0x05, 0x50, 0x72, 0x6f, 0x62, 0x65, 0x12, + 0x14, 0x2e, 0x63, 0x6d, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x62, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x63, 0x6d, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x50, + 0x72, 0x6f, 0x62, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x57, + 0x0a, 0x12, 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, + 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1e, 0x2e, 0x63, 0x6d, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, + 0x74, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x63, 0x6d, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, + 0x74, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6c, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x50, 0x72, + 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, + 0x65, 0x73, 0x12, 0x26, 0x2e, 0x63, 0x6d, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, + 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, + 0x69, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x63, 0x6d, 0x69, + 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, + 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x32, 0xa2, 0x01, 0x0a, 0x0c, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x1a, 0x2e, 0x63, 0x6d, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x43, + 0x72, 0x65, 0x61, 0x74, 0x65, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6d, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, + 0x65, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x12, + 0x1a, 0x2e, 0x63, 0x6d, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4c, + 0x61, 0x62, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6d, + 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4c, 0x61, 0x62, 0x65, 0x6c, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x32, 0x49, 0x0a, 0x09, 0x43, 0x6f, + 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x3c, 0x0a, 0x07, 0x43, 0x6f, 0x6c, 0x6c, 0x65, + 0x63, 0x74, 0x12, 0x16, 0x2e, 0x63, 0x6d, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, + 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6d, 0x69, + 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x0c, 0x5a, 0x0a, 0x6c, 0x69, 0x62, 0x2f, 0x67, 0x6f, 0x3b, + 0x63, 0x6d, 0x69, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_cmi_proto_rawDescOnce sync.Once + file_cmi_proto_rawDescData = file_cmi_proto_rawDesc +) + +func file_cmi_proto_rawDescGZIP() []byte { + file_cmi_proto_rawDescOnce.Do(func() { + file_cmi_proto_rawDescData = protoimpl.X.CompressGZIP(file_cmi_proto_rawDescData) + }) + return file_cmi_proto_rawDescData +} + +var file_cmi_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_cmi_proto_msgTypes = make([]protoimpl.MessageInfo, 16) +var file_cmi_proto_goTypes = []interface{}{ + (ProviderCapability_Type)(0), // 0: cmi.v1.ProviderCapability.Type + (*CreateLabelRequest)(nil), // 1: cmi.v1.CreateLabelRequest + (*CreateLabelResponse)(nil), // 2: cmi.v1.CreateLabelResponse + (*DeleteLabelRequest)(nil), // 3: cmi.v1.DeleteLabelRequest + (*DeleteLabelResponse)(nil), // 4: cmi.v1.DeleteLabelResponse + (*ProbeRequest)(nil), // 5: cmi.v1.ProbeRequest + (*ProbeResponse)(nil), // 6: cmi.v1.ProbeResponse + (*GetProviderCapabilitiesRequest)(nil), // 7: cmi.v1.GetProviderCapabilitiesRequest + (*GetProviderCapabilitiesResponse)(nil), // 8: cmi.v1.GetProviderCapabilitiesResponse + (*ProviderCapability)(nil), // 9: cmi.v1.ProviderCapability + (*GetProviderInfoRequest)(nil), // 10: cmi.v1.GetProviderInfoRequest + (*GetProviderInfoResponse)(nil), // 11: cmi.v1.GetProviderInfoResponse + (*CollectRequest)(nil), // 12: cmi.v1.CollectRequest + (*CollectResponse)(nil), // 13: cmi.v1.CollectResponse + (*CollectDetail)(nil), // 14: cmi.v1.CollectDetail + nil, // 15: cmi.v1.CreateLabelRequest.ParametersEntry + nil, // 16: cmi.v1.CollectDetail.DataEntry + (*wrappers.BoolValue)(nil), // 17: google.protobuf.BoolValue +} +var file_cmi_proto_depIdxs = []int32{ + 15, // 0: cmi.v1.CreateLabelRequest.parameters:type_name -> cmi.v1.CreateLabelRequest.ParametersEntry + 17, // 1: cmi.v1.CreateLabelResponse.success:type_name -> google.protobuf.BoolValue + 17, // 2: cmi.v1.DeleteLabelResponse.success:type_name -> google.protobuf.BoolValue + 17, // 3: cmi.v1.ProbeResponse.ready:type_name -> google.protobuf.BoolValue + 9, // 4: cmi.v1.GetProviderCapabilitiesResponse.capabilities:type_name -> cmi.v1.ProviderCapability + 0, // 5: cmi.v1.ProviderCapability.type:type_name -> cmi.v1.ProviderCapability.Type + 14, // 6: cmi.v1.CollectResponse.details:type_name -> cmi.v1.CollectDetail + 16, // 7: cmi.v1.CollectDetail.data:type_name -> cmi.v1.CollectDetail.DataEntry + 5, // 8: cmi.v1.Identity.Probe:input_type -> cmi.v1.ProbeRequest + 10, // 9: cmi.v1.Identity.GetProvisionerInfo:input_type -> cmi.v1.GetProviderInfoRequest + 7, // 10: cmi.v1.Identity.GetProviderCapabilities:input_type -> cmi.v1.GetProviderCapabilitiesRequest + 1, // 11: cmi.v1.LabelService.CreateLabel:input_type -> cmi.v1.CreateLabelRequest + 3, // 12: cmi.v1.LabelService.DeleteLabel:input_type -> cmi.v1.DeleteLabelRequest + 12, // 13: cmi.v1.Collector.Collect:input_type -> cmi.v1.CollectRequest + 6, // 14: cmi.v1.Identity.Probe:output_type -> cmi.v1.ProbeResponse + 11, // 15: cmi.v1.Identity.GetProvisionerInfo:output_type -> cmi.v1.GetProviderInfoResponse + 8, // 16: cmi.v1.Identity.GetProviderCapabilities:output_type -> cmi.v1.GetProviderCapabilitiesResponse + 2, // 17: cmi.v1.LabelService.CreateLabel:output_type -> cmi.v1.CreateLabelResponse + 4, // 18: cmi.v1.LabelService.DeleteLabel:output_type -> cmi.v1.DeleteLabelResponse + 13, // 19: cmi.v1.Collector.Collect:output_type -> cmi.v1.CollectResponse + 14, // [14:20] is the sub-list for method output_type + 8, // [8:14] is the sub-list for method input_type + 8, // [8:8] is the sub-list for extension type_name + 8, // [8:8] is the sub-list for extension extendee + 0, // [0:8] is the sub-list for field type_name +} + +func init() { file_cmi_proto_init() } +func file_cmi_proto_init() { + if File_cmi_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_cmi_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CreateLabelRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cmi_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CreateLabelResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cmi_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DeleteLabelRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cmi_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DeleteLabelResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cmi_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ProbeRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cmi_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ProbeResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cmi_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetProviderCapabilitiesRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cmi_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetProviderCapabilitiesResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cmi_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ProviderCapability); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cmi_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetProviderInfoRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cmi_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetProviderInfoResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cmi_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CollectRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cmi_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CollectResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cmi_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CollectDetail); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_cmi_proto_rawDesc, + NumEnums: 1, + NumMessages: 16, + NumExtensions: 0, + NumServices: 3, + }, + GoTypes: file_cmi_proto_goTypes, + DependencyIndexes: file_cmi_proto_depIdxs, + EnumInfos: file_cmi_proto_enumTypes, + MessageInfos: file_cmi_proto_msgTypes, + }.Build() + File_cmi_proto = out.File + file_cmi_proto_rawDesc = nil + file_cmi_proto_goTypes = nil + file_cmi_proto_depIdxs = nil +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConnInterface + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion6 + +// IdentityClient is the client API for Identity service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type IdentityClient interface { + // Get CMI running status. + Probe(ctx context.Context, in *ProbeRequest, opts ...grpc.CallOption) (*ProbeResponse, error) + // Get CMI information, e.g. provider's name. + GetProvisionerInfo(ctx context.Context, in *GetProviderInfoRequest, opts ...grpc.CallOption) (*GetProviderInfoResponse, error) + // Get CMI capabilities + // Though it, can know which interfaces have been implemented by CMI. + GetProviderCapabilities(ctx context.Context, in *GetProviderCapabilitiesRequest, opts ...grpc.CallOption) (*GetProviderCapabilitiesResponse, error) +} + +type identityClient struct { + cc grpc.ClientConnInterface +} + +func NewIdentityClient(cc grpc.ClientConnInterface) IdentityClient { + return &identityClient{cc} +} + +func (c *identityClient) Probe(ctx context.Context, in *ProbeRequest, opts ...grpc.CallOption) (*ProbeResponse, error) { + out := new(ProbeResponse) + err := c.cc.Invoke(ctx, "/cmi.v1.Identity/Probe", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *identityClient) GetProvisionerInfo(ctx context.Context, in *GetProviderInfoRequest, opts ...grpc.CallOption) (*GetProviderInfoResponse, error) { + out := new(GetProviderInfoResponse) + err := c.cc.Invoke(ctx, "/cmi.v1.Identity/GetProvisionerInfo", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *identityClient) GetProviderCapabilities(ctx context.Context, in *GetProviderCapabilitiesRequest, opts ...grpc.CallOption) (*GetProviderCapabilitiesResponse, error) { + out := new(GetProviderCapabilitiesResponse) + err := c.cc.Invoke(ctx, "/cmi.v1.Identity/GetProviderCapabilities", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// IdentityServer is the server API for Identity service. +type IdentityServer interface { + // Get CMI running status. + Probe(context.Context, *ProbeRequest) (*ProbeResponse, error) + // Get CMI information, e.g. provider's name. + GetProvisionerInfo(context.Context, *GetProviderInfoRequest) (*GetProviderInfoResponse, error) + // Get CMI capabilities + // Though it, can know which interfaces have been implemented by CMI. + GetProviderCapabilities(context.Context, *GetProviderCapabilitiesRequest) (*GetProviderCapabilitiesResponse, error) +} + +// UnimplementedIdentityServer can be embedded to have forward compatible implementations. +type UnimplementedIdentityServer struct { +} + +func (*UnimplementedIdentityServer) Probe(context.Context, *ProbeRequest) (*ProbeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Probe not implemented") +} +func (*UnimplementedIdentityServer) GetProvisionerInfo(context.Context, *GetProviderInfoRequest) (*GetProviderInfoResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetProvisionerInfo not implemented") +} +func (*UnimplementedIdentityServer) GetProviderCapabilities(context.Context, *GetProviderCapabilitiesRequest) (*GetProviderCapabilitiesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetProviderCapabilities not implemented") +} + +func RegisterIdentityServer(s *grpc.Server, srv IdentityServer) { + s.RegisterService(&_Identity_serviceDesc, srv) +} + +func _Identity_Probe_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ProbeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(IdentityServer).Probe(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/cmi.v1.Identity/Probe", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(IdentityServer).Probe(ctx, req.(*ProbeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Identity_GetProvisionerInfo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetProviderInfoRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(IdentityServer).GetProvisionerInfo(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/cmi.v1.Identity/GetProvisionerInfo", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(IdentityServer).GetProvisionerInfo(ctx, req.(*GetProviderInfoRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Identity_GetProviderCapabilities_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetProviderCapabilitiesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(IdentityServer).GetProviderCapabilities(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/cmi.v1.Identity/GetProviderCapabilities", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(IdentityServer).GetProviderCapabilities(ctx, req.(*GetProviderCapabilitiesRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _Identity_serviceDesc = grpc.ServiceDesc{ + ServiceName: "cmi.v1.Identity", + HandlerType: (*IdentityServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Probe", + Handler: _Identity_Probe_Handler, + }, + { + MethodName: "GetProvisionerInfo", + Handler: _Identity_GetProvisionerInfo_Handler, + }, + { + MethodName: "GetProviderCapabilities", + Handler: _Identity_GetProviderCapabilities_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "cmi.proto", +} + +// LabelServiceClient is the client API for LabelService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type LabelServiceClient interface { + // Create label + CreateLabel(ctx context.Context, in *CreateLabelRequest, opts ...grpc.CallOption) (*CreateLabelResponse, error) + // Delete label + DeleteLabel(ctx context.Context, in *DeleteLabelRequest, opts ...grpc.CallOption) (*DeleteLabelResponse, error) +} + +type labelServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewLabelServiceClient(cc grpc.ClientConnInterface) LabelServiceClient { + return &labelServiceClient{cc} +} + +func (c *labelServiceClient) CreateLabel(ctx context.Context, in *CreateLabelRequest, opts ...grpc.CallOption) (*CreateLabelResponse, error) { + out := new(CreateLabelResponse) + err := c.cc.Invoke(ctx, "/cmi.v1.LabelService/CreateLabel", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *labelServiceClient) DeleteLabel(ctx context.Context, in *DeleteLabelRequest, opts ...grpc.CallOption) (*DeleteLabelResponse, error) { + out := new(DeleteLabelResponse) + err := c.cc.Invoke(ctx, "/cmi.v1.LabelService/DeleteLabel", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// LabelServiceServer is the server API for LabelService service. +type LabelServiceServer interface { + // Create label + CreateLabel(context.Context, *CreateLabelRequest) (*CreateLabelResponse, error) + // Delete label + DeleteLabel(context.Context, *DeleteLabelRequest) (*DeleteLabelResponse, error) +} + +// UnimplementedLabelServiceServer can be embedded to have forward compatible implementations. +type UnimplementedLabelServiceServer struct { +} + +func (*UnimplementedLabelServiceServer) CreateLabel(context.Context, *CreateLabelRequest) (*CreateLabelResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method CreateLabel not implemented") +} +func (*UnimplementedLabelServiceServer) DeleteLabel(context.Context, *DeleteLabelRequest) (*DeleteLabelResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteLabel not implemented") +} + +func RegisterLabelServiceServer(s *grpc.Server, srv LabelServiceServer) { + s.RegisterService(&_LabelService_serviceDesc, srv) +} + +func _LabelService_CreateLabel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateLabelRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(LabelServiceServer).CreateLabel(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/cmi.v1.LabelService/CreateLabel", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(LabelServiceServer).CreateLabel(ctx, req.(*CreateLabelRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _LabelService_DeleteLabel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteLabelRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(LabelServiceServer).DeleteLabel(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/cmi.v1.LabelService/DeleteLabel", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(LabelServiceServer).DeleteLabel(ctx, req.(*DeleteLabelRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _LabelService_serviceDesc = grpc.ServiceDesc{ + ServiceName: "cmi.v1.LabelService", + HandlerType: (*LabelServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "CreateLabel", + Handler: _LabelService_CreateLabel_Handler, + }, + { + MethodName: "DeleteLabel", + Handler: _LabelService_DeleteLabel_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "cmi.proto", +} + +// CollectorClient is the client API for Collector service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type CollectorClient interface { + // collect storage info + Collect(ctx context.Context, in *CollectRequest, opts ...grpc.CallOption) (*CollectResponse, error) +} + +type collectorClient struct { + cc grpc.ClientConnInterface +} + +func NewCollectorClient(cc grpc.ClientConnInterface) CollectorClient { + return &collectorClient{cc} +} + +func (c *collectorClient) Collect(ctx context.Context, in *CollectRequest, opts ...grpc.CallOption) (*CollectResponse, error) { + out := new(CollectResponse) + err := c.cc.Invoke(ctx, "/cmi.v1.Collector/Collect", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// CollectorServer is the server API for Collector service. +type CollectorServer interface { + // collect storage info + Collect(context.Context, *CollectRequest) (*CollectResponse, error) +} + +// UnimplementedCollectorServer can be embedded to have forward compatible implementations. +type UnimplementedCollectorServer struct { +} + +func (*UnimplementedCollectorServer) Collect(context.Context, *CollectRequest) (*CollectResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Collect not implemented") +} + +func RegisterCollectorServer(s *grpc.Server, srv CollectorServer) { + s.RegisterService(&_Collector_serviceDesc, srv) +} + +func _Collector_Collect_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CollectRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CollectorServer).Collect(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/cmi.v1.Collector/Collect", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CollectorServer).Collect(ctx, req.(*CollectRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _Collector_serviceDesc = grpc.ServiceDesc{ + ServiceName: "cmi.v1.Collector", + HandlerType: (*CollectorServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Collect", + Handler: _Collector_Collect_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "cmi.proto", +} diff --git a/grpc/proto/cmi.proto b/grpc/proto/cmi.proto new file mode 100644 index 0000000..c304eeb --- /dev/null +++ b/grpc/proto/cmi.proto @@ -0,0 +1,178 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +syntax = "proto3"; +package cmi.v1; + +import "google/protobuf/wrappers.proto"; +option go_package = "lib/go;cmi"; + +message CreateLabelRequest{ + // This field is REQUIRED.Value of this field is unique for a volume. + string volume_id = 1; + + // This field is REQUIRED.Value of this field is label name. + string label_name = 2; + + // This field is REQUIRED.Value of this field is label kind, e.g. Pod, PersistentVolume... + string kind = 3; + + // This field is OPTIONAL. This allows to specify the namespace when kind is Pod + // If not specified, will use 'default' as the default namespace + string namespace = 4; + + // This field is OPTIONAL.This allows to specify the cluster name when kind is PersistentVolume + string cluster_name = 5; + + // Specific parameters passed in as opaque key-value pairs. + // This field is OPTIONAL. The CMI is responsible for parsing and validating these parameters. + map parameters = 6; +} + +message CreateLabelResponse{ + // Indicates if success or not + google.protobuf.BoolValue success = 1; +} + +message DeleteLabelRequest{ + // This field is REQUIRED.Value of this field is unique for a volume. + string volume_id = 1; + + // This field is REQUIRED.Value of this field is label name. + string label_name = 2; + + // This field is REQUIRED.Value of this field is label kind, e.g. Pod, PersistentVolume... + string kind = 3; + + // This field is OPTIONAL. This allows to specify the namespace when kind is Pod + // If not specified, will use 'default' as the default namespace + string namespace = 4; +} + +message DeleteLabelResponse{ + // Indicates if success or not + google.protobuf.BoolValue success = 1; +} + +// Probe request to check health/availability +message ProbeRequest{} + +// Response to indicate health/availability status +message ProbeResponse { + // Indicates if healthy/available or not + google.protobuf.BoolValue ready = 1; +} + +message GetProviderCapabilitiesRequest{ + // Intentionally empty. +} + +message GetProviderCapabilitiesResponse{ + // All the capabilities that the CMI supports. This field is OPTIONAL. + repeated ProviderCapability capabilities = 1; +} + +message ProviderCapability{ + enum Type{ + // If CMI implements ProviderCapability_Label_Service capability + // then it must implement CreateLabel RPC and DeleteLabel RPC. + ProviderCapability_Label_Service = 0; + // If CMI implements ProviderCapability_Label_Service capability + // then it must implement Collect RPC call for fetching storage information. + ProviderCapability_Collect_Service = 1; + } + + Type type = 1; +} + +message GetProviderInfoRequest{ + // Intentionally empty. +} + +message GetProviderInfoResponse{ + // This field is REQUIRED. Value of this field is unique name for CMI. + string provider = 1; +} + +message CollectRequest{ + // This field is REQUIRED. Value of this field is StorageBlackClaim name. + // See StorageBlackClaim resource for details. + string backend_name = 1; + + // This field is REQUIRED. Value of this field is collect type. + // Indicates that type of data to be collected by Collect request. + string collect_type = 2; + + // This field is REQUIRED. Value of this field is metrics type. + // Allowed values: + // object: will collect object data, e.g. controller's cpu usage... + // performance: will collect performance data, e.g. controller's read bandwidth... + string metrics_type = 3; + + // This field is REQUIRED when metrics_type is performance + repeated string indicators = 4; +} + +message CollectResponse{ + // This field is REQUIRED. + // See CollectRequest's backend_name for details. + string backend_name = 1; + + // This field is REQUIRED. + // See CollectRequest's collect_type for details. + string collect_type = 2; + + // This field is REQUIRED. + // See CollectRequest's metrics_type for details. + string metrics_type = 3; + + // The list of collected data. + repeated CollectDetail details = 4; +} + +message CollectDetail{ + // This field is REQUIRED. Value of this field is a map of data to be collected. + // Collected data are specified in as key-value pairs. + map data = 6; +} + +service Identity{ + // Get CMI running status. + rpc Probe(ProbeRequest) + returns (ProbeResponse) {} + + // Get CMI information, e.g. provider's name. + rpc GetProvisionerInfo(GetProviderInfoRequest) + returns (GetProviderInfoResponse) {} + + // Get CMI capabilities + // Though it, can know which interfaces have been implemented by CMI. + rpc GetProviderCapabilities(GetProviderCapabilitiesRequest) + returns (GetProviderCapabilitiesResponse) {} +} + +service LabelService{ + // Create label + rpc CreateLabel(CreateLabelRequest) + returns (CreateLabelResponse) {} + + // Delete label + rpc DeleteLabel(DeleteLabelRequest) + returns (DeleteLabelResponse) {} +} + +service Collector{ + // collect storage info + rpc Collect(CollectRequest) + returns (CollectResponse){} +} \ No newline at end of file diff --git a/helm/huawei-csm/.helmignore b/helm/huawei-csm/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/helm/huawei-csm/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/huawei-csm/Chart.yaml b/helm/huawei-csm/Chart.yaml new file mode 100644 index 0000000..112d2c6 --- /dev/null +++ b/helm/huawei-csm/Chart.yaml @@ -0,0 +1,16 @@ +apiVersion: v2 +name: huawei-csm +description: A Helm chart for deployment of CSM(Container Storage Monitor) Service + +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: "{{version}}" + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "{{version}}" diff --git a/helm/huawei-csm/common.sh b/helm/huawei-csm/common.sh new file mode 100644 index 0000000..3b4e767 --- /dev/null +++ b/helm/huawei-csm/common.sh @@ -0,0 +1,146 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# shellcheck disable=SC2034 +DRIVERDIR="${SCRIPTDIR}/../helm" + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +DARK_GRAY='\033[1;30m' +NC='\033[0m' # No Color + +function decho() { + if [ -n "${DEBUGLOG}" ]; then + echo "$@" | tee -a "${DEBUGLOG}" + fi +} + +function debuglog_only() { + if [ -n "${DEBUGLOG}" ]; then + echo "$@" >> "${DEBUGLOG}" + fi +} + +function log() { + case $1 in + separator) + decho "------------------------------------------------------" + ;; + error) + decho + log separator + # shellcheck disable=SC2059 + printf "${RED}Error: $2\n" + # shellcheck disable=SC2059 + printf "${RED}Installation cannot continue${NC}\n" + debuglog_only "Error: $2" + debuglog_only "Installation cannot continue" + exit 1 + ;; + uninstall_error) + log separator + # shellcheck disable=SC2059 + printf "${RED}Error: $2\n" + # shellcheck disable=SC2059 + printf "${RED}Uninstallation cannot continue${NC}\n" + debuglog_only "Error: $2" + debuglog_only "Uninstallation cannot continue" + exit 1 + ;; + step) + printf "|\n|- %-65s" "$2" + debuglog_only "${2}" + ;; + small_step) + printf "%-61s" "$2" + debuglog_only "${2}" + ;; + section) + log separator + printf "> %s\n" "$2" + debuglog_only "${2}" + log separator + ;; + smart_step) + if [[ $3 == "small" ]]; then + log small_step "$2" + else + log step "$2" + fi + ;; + arrow) + printf " %s\n %s" "|" "|--> " + ;; + step_success) + # shellcheck disable=SC2059 + printf "${GREEN}Success${NC}\n" + ;; + step_failure) + # shellcheck disable=SC2059 + printf "${RED}Failed${NC}\n" + ;; + step_warning) + # shellcheck disable=SC2059 + printf "${YELLOW}Warning${NC}\n" + ;; + info) + printf "${DARK_GRAY}%s${NC}\n" "$2" + ;; + passed) + # shellcheck disable=SC2059 + printf "${GREEN}Success${NC}\n" + ;; + warnings) + # shellcheck disable=SC2059 + printf "${YELLOW}Warnings:${NC}\n" + ;; + errors) + # shellcheck disable=SC2059 + printf "${RED}Errors:${NC}\n" + ;; + *) + echo -n "Unknown" + ;; + esac +} + +function run_command() { + local RC=0 + if [ -n "${DEBUGLOG}" ]; then + # shellcheck disable=SC2155 + local ME=$(basename "${0}") + echo "---------------" >> "${DEBUGLOG}" + # shellcheck disable=SC2145 + echo "${ME}:${BASH_LINENO[0]} - Running command: $@" >> "${DEBUGLOG}" + debuglog_only "Results:" + eval "$@" 2>&1 | tee -a "${DEBUGLOG}" + RC=${PIPESTATUS[0]} + echo "---------------" >> "${DEBUGLOG}" + else + eval "$@" + RC=$? + fi + return $RC +} + +# warning, with an option for users to continue +function warning() { + log separator + # shellcheck disable=SC2059 + printf "${YELLOW}WARNING:${NC}\n" + for N in "$@"; do + decho "$N" + done + decho +} diff --git a/helm/huawei-csm/crds/xuanwu.huawei.io_resourcetopologies.yaml b/helm/huawei-csm/crds/xuanwu.huawei.io_resourcetopologies.yaml new file mode 100644 index 0000000..a5418e3 --- /dev/null +++ b/helm/huawei-csm/crds/xuanwu.huawei.io_resourcetopologies.yaml @@ -0,0 +1,176 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null + name: resourcetopologies.xuanwu.huawei.io +spec: + group: xuanwu.huawei.io + names: + kind: ResourceTopology + listKind: ResourceTopologyList + plural: resourcetopologies + shortNames: + - rt + singular: resourcetopology + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.provisioner + name: Provisioner + type: string + - jsonPath: .spec.volumeHandle + name: VolumeHandle + type: string + - jsonPath: .status.status + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: ResourceTopology is the Schema for the ResourceTopologys API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ResourceTopologySpec defines the fields in Spec + properties: + provisioner: + description: Provisioner is the volume provisioner name + type: string + tags: + description: Tags defines pv and other relationships and ownership + items: + description: Tag defines pv and other relationships and ownership + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this + representation of an object. Servers should convert recognized + schemas to the latest internal value, and may reject unrecognized + values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint + the client submits requests to. Cannot be updated. In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: Name is the name of the resource + type: string + namespace: + description: NameSpace is the namespace of the resource + type: string + owner: + description: Owner defines who does the resource belongs to + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of + this representation of an object. Servers should convert + recognized schemas to the latest internal value, and may + reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST + resource this object represents. Servers may infer this + from the endpoint the client submits requests to. Cannot + be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: Name is the name of the resource + type: string + namespace: + description: NameSpace is the namespace of the resource + type: string + type: object + type: object + type: array + volumeHandle: + description: VolumeHandle is the backend name and identity of the + volume, format as . + type: string + required: + - provisioner + - tags + - volumeHandle + type: object + status: + description: ResourceTopologyStatus status of resource topology + properties: + status: + description: Status is the status of the ResourceTopology + type: string + tags: + description: Tags defines pv and other relationships and ownership + items: + description: Tag defines pv and other relationships and ownership + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this + representation of an object. Servers should convert recognized + schemas to the latest internal value, and may reject unrecognized + values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint + the client submits requests to. Cannot be updated. In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: Name is the name of the resource + type: string + namespace: + description: NameSpace is the namespace of the resource + type: string + owner: + description: Owner defines who does the resource belongs to + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of + this representation of an object. Servers should convert + recognized schemas to the latest internal value, and may + reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST + resource this object represents. Servers may infer this + from the endpoint the client submits requests to. Cannot + be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: Name is the name of the resource + type: string + namespace: + description: NameSpace is the namespace of the resource + type: string + type: object + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/helm/huawei-csm/grafana/OceanStor.json b/helm/huawei-csm/grafana/OceanStor.json new file mode 100644 index 0000000..6e3a585 --- /dev/null +++ b/helm/huawei-csm/grafana/OceanStor.json @@ -0,0 +1,4619 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 20, + "iteration": 1695864698601, + "links": [], + "panels": [ + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 92, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "qaalCSjVz" + }, + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 108, + "options": { + "content": "", + "mode": "html" + }, + "pluginVersion": "7.5.4", + "type": "text" + }, + { + "datasource": "${DataSource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-YlBl" + }, + "custom": { + "align": "center", + "displayMode": "color-text", + "filterable": false + }, + "mappings": [ + { + "from": "", + "id": 1, + "text": "Device Type", + "to": "", + "type": 1, + "value": "model" + }, + { + "from": "", + "id": 2, + "text": "Serial Number", + "to": "", + "type": 1, + "value": "sn" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 3, + "y": 1 + }, + "id": 98, + "maxPerRow": 2, + "options": { + "footer": { + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": false, + "sortBy": [] + }, + "pluginVersion": "7.5.4", + "repeat": null, + "repeatDirection": "h", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_array_basic_info{endpoint=~\"$StorageName\"}", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "$StorageName Device Information", + "transformations": [ + { + "id": "labelsToFields", + "options": {} + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "model", + "sn" + ] + } + } + }, + { + "id": "reduce", + "options": { + "includeTimeField": false, + "mode": "seriesToRows", + "reducers": [ + "last" + ] + } + } + ], + "type": "table" + }, + { + "datasource": "${DataSource}", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "from": "", + "id": 1, + "text": "NORMAL", + "to": "", + "type": 1, + "value": "1" + }, + { + "from": "", + "id": 2, + "text": "UNKNOWN", + "to": "", + "type": 1, + "value": "0" + }, + { + "from": "", + "id": 3, + "text": "FAULTY", + "to": "", + "type": 1, + "value": "2" + }, + { + "from": "", + "id": 4, + "text": "INCONSISTENT", + "to": "", + "type": 1, + "value": "9" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Running Status" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "from": "", + "id": 1, + "text": "UNKNOWN", + "to": "", + "type": 1, + "value": "0" + }, + { + "from": "", + "id": 2, + "text": "NORMAL", + "to": "", + "type": 1, + "value": "1" + }, + { + "from": "", + "id": 3, + "text": "RUNNING", + "to": "", + "type": 1, + "value": "2" + }, + { + "from": "", + "id": 4, + "text": "ONLINE", + "to": "", + "type": 1, + "value": "27" + }, + { + "from": "", + "id": 5, + "text": "OFFLINE", + "to": "", + "type": 1, + "value": "28" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 100, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 26 + }, + "textMode": "auto" + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_array_health_status{endpoint=~\"$StorageName\"}", + "hide": false, + "interval": "", + "legendFormat": "Health Status", + "refId": "C" + }, + { + "exemplar": true, + "expr": "huawei_storage_array_running_status{endpoint=~\"$StorageName\"}", + "hide": false, + "interval": "", + "legendFormat": "Running Status", + "refId": "D" + } + ], + "title": "$StorageName Device status", + "type": "stat" + }, + { + "datasource": "${DataSource}", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "from": "", + "id": 1, + "text": "NORMAL", + "to": "", + "type": 1, + "value": "1" + }, + { + "from": "", + "id": 2, + "text": "UNKNOWN", + "to": "", + "type": 1, + "value": "0" + }, + { + "from": "", + "id": 3, + "text": "FAULTY", + "to": "", + "type": 1, + "value": "2" + }, + { + "from": "", + "id": 4, + "text": "INCONSISTENT", + "to": "", + "type": 1, + "value": "9" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Running Status" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "from": "", + "id": 1, + "text": "UNKNOWN", + "to": "", + "type": 1, + "value": "0" + }, + { + "from": "", + "id": 2, + "text": "NORMAL", + "to": "", + "type": 1, + "value": "1" + }, + { + "from": "", + "id": 3, + "text": "RUNNING", + "to": "", + "type": 1, + "value": "2" + }, + { + "from": "", + "id": 4, + "text": "ONLINE", + "to": "", + "type": 1, + "value": "27" + }, + { + "from": "", + "id": 5, + "text": "OFFLINE", + "to": "", + "type": 1, + "value": "28" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 122, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 26 + }, + "textMode": "auto" + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_controller_health_status{endpoint=~\"$StorageName\", name=~\"$ControllerName\"}", + "hide": false, + "interval": "", + "legendFormat": "Health Status", + "refId": "C" + }, + { + "exemplar": true, + "expr": "huawei_storage_controller_running_status{endpoint=~\"$StorageName\", name=~\"$ControllerName\"}", + "hide": false, + "interval": "", + "legendFormat": "Running Status", + "refId": "D" + } + ], + "title": "$ControllerName Controller Status", + "type": "stat" + }, + { + "datasource": "${DataSource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "max": 200, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 0, + "y": 6 + }, + "id": 110, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "titleSize": 4, + "valueSize": 50 + }, + "textMode": "auto" + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DataSource}" + }, + "editorMode": "builder", + "expr": "huawei_storage_controller_total_iops{endpoint=~\"$StorageName\", name=~\"$ControllerName\"}", + "hide": true, + "legendFormat": "__auto", + "queryType": "randomWalk", + "range": true, + "refId": "A" + }, + { + "exemplar": true, + "expr": "huawei_storage_controller_total_iops{endpoint=~\"$StorageName\", name=~\"$ControllerName\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Total I/Os of controller $ControllerName", + "type": "stat" + }, + { + "datasource": "${DataSource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "max": 200, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "MiBs" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 3, + "y": 6 + }, + "id": 112, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 50 + }, + "textMode": "auto" + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_controller_total_bandwidth{endpoint=~\"$StorageName\", name=~\"$ControllerName\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Total bandwidth rate of controller $ControllerName", + "type": "stat" + }, + { + "datasource": "${DataSource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd", + "seriesBy": "last" + }, + "mappings": [], + "max": 200, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 6, + "y": 6 + }, + "id": 114, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 50 + }, + "textMode": "auto" + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_controller_avg_io_response_time{endpoint=~\"$StorageName\", name=~\"$ControllerName\"}", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "title": "Average I/O response time of controller $ControllerName", + "type": "stat" + }, + { + "datasource": "${DataSource}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "unit", + "value": "percent" + }, + { + "id": "max", + "value": 100 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 9, + "y": 6 + }, + "id": 54, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": { + "valueSize": 35 + } + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_controller_memory_usage{endpoint=~\"$StorageName\", name=~\"$ControllerName\"}", + "hide": false, + "interval": "", + "legendFormat": "Memory usage", + "refId": "C" + }, + { + "exemplar": true, + "expr": "huawei_storage_controller_cpu_usage{endpoint=~\"$StorageName\", name=~\"$ControllerName\"}", + "hide": false, + "interval": "", + "legendFormat": "CPU usage", + "refId": "D" + } + ], + "title": "Resource usage of controller $ControllerName", + "type": "bargauge" + }, + { + "datasource": "${DataSource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "max": 200, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "MiBs" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 15, + "y": 6 + }, + "id": 116, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 50 + }, + "textMode": "auto" + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_storagepool_total_bandwidth{endpoint=~\"$StorageName\", name=~\"$Pool\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Total bandwidth rate of storage pool $Pool", + "type": "stat" + }, + { + "datasource": "${DataSource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd", + "seriesBy": "last" + }, + "mappings": [], + "max": 200, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 18, + "y": 6 + }, + "id": 118, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 50 + }, + "textMode": "auto" + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_storagepool_avg_io_response_time{endpoint=~\"$StorageName\", name=~\"$Pool\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Average I/O response time of storage pool $Pool", + "type": "stat" + }, + { + "datasource": "${DataSource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "max": 200, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "continuous-YlBl" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 21, + "y": 6 + }, + "id": 120, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 50 + }, + "textMode": "auto" + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_storagepool_total_iops{endpoint=~\"$StorageName\", name=~\"$Pool\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Total number of I/Os in storage pool $Pool", + "type": "stat" + }, + { + "datasource": "${DataSource}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 106, + "options": { + "displayMode": "basic", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": {} + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "sort_desc(topk($FileSystemNum, huawei_storage_filesystem_capacity_usage{endpoint=~\"$StorageName\", name=~\".*\"}))", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "{{name}}", + "refId": "B" + } + ], + "title": "Top $FileSystemNum File System Capacity Usage of Storage Pool $Pool ", + "transformations": [], + "type": "bargauge" + }, + { + "datasource": "${DataSource}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 104, + "options": { + "displayMode": "basic", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": {} + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "sort_desc(topk($LunNum, huawei_storage_lun_capacity_usage{endpoint=~\"$StorageName\", name=~\".*\"}))", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "{{name}}", + "refId": "B" + } + ], + "title": "Top $LunNum Storage Pool $Pool Lun Capacity Usage Top", + "transformations": [], + "type": "bargauge" + } + ], + "title": "StorageDevice", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 30, + "panels": [ + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": "${DataSource}", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 2 + }, + "heatmap": {}, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 68, + "legend": { + "show": false + }, + "options": { + "calculate": true, + "calculation": { + "xBuckets": { + "mode": "size", + "value": "5m" + }, + "yBuckets": { + "mode": "size", + "scale": { + "type": "linear" + }, + "value": "10" + } + }, + "cellGap": 1, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "scale": "exponential", + "scheme": "RdYlBu", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "max": "100", + "min": 0, + "reverse": false + } + }, + "pluginVersion": "9.1.0", + "reverseYBuckets": false, + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_controller_memory_usage{endpoint=~\"$StorageName\", name=~\"$ControllerName\"}", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "$ControllerName", + "refId": "B" + } + ], + "title": "Memory usage of controller $ControllerName", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": "5m", + "yAxis": { + "decimals": null, + "format": "short", + "logBase": 1, + "max": "100", + "min": "0", + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": 10 + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": "${DataSource}", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 2 + }, + "heatmap": {}, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 62, + "legend": { + "show": false + }, + "options": { + "calculate": true, + "calculation": { + "xBuckets": { + "mode": "size", + "value": "5m" + }, + "yBuckets": { + "mode": "size", + "scale": { + "type": "linear" + }, + "value": "10" + } + }, + "cellGap": 1, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "scale": "exponential", + "scheme": "RdYlBu", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "max": "100", + "min": 0, + "reverse": false + } + }, + "pluginVersion": "9.1.0", + "reverseYBuckets": false, + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_controller_cpu_usage{endpoint=~\"$StorageName\", name=~\"$ControllerName\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "title": "CPU usage of controller $ControllerName", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": "5m", + "yAxis": { + "decimals": null, + "format": "short", + "logBase": 1, + "max": "100", + "min": "0", + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": 10 + }, + { + "datasource": "${DataSource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 26, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 8 + }, + "id": 64, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_controller_total_iops{endpoint=~\"$StorageName\", name=~\"$ControllerName\"}", + "hide": false, + "interval": "", + "legendFormat": "$ControllerName", + "refId": "B" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Total I/Os of controller $ControllerName", + "type": "timeseries" + }, + { + "datasource": "${DataSource}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 26, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 8 + }, + "id": 44, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_controller_read_iops{endpoint=~\"$StorageName\", name=~\"$ControllerName\"}", + "hide": false, + "interval": "", + "legendFormat": "$ControllerName", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Number of read I/Os of controller $ControllerName", + "type": "timeseries" + }, + { + "datasource": "${DataSource}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 26, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 8 + }, + "id": 46, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_controller_write_iops{endpoint=~\"$StorageName\", name=~\"$ControllerName\"}", + "hide": false, + "interval": "", + "legendFormat": "$ControllerName", + "refId": "B" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Number of write I/Os of controller $ControllerName ", + "type": "timeseries" + }, + { + "datasource": "${DataSource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 26, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "MiBs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 15 + }, + "id": 66, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_controller_total_bandwidth{endpoint=~\"$StorageName\", name=~\"$ControllerName\"}", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "$ControllerName", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Total bandwidth rate of controller $ControllerName", + "type": "timeseries" + }, + { + "datasource": "${DataSource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 26, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "MiBs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 15 + }, + "id": 50, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_controller_read_bandwidth{endpoint=~\"$StorageName\", name=~\"$ControllerName\"}", + "hide": false, + "interval": "", + "legendFormat": "$ControllerName", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Read bandwidth rate of controller $ControllerName", + "type": "timeseries" + }, + { + "datasource": "${DataSource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 26, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "MiBs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 15 + }, + "id": 52, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_controller_write_bandwidth{endpoint=~\"$StorageName\", name=~\"$ControllerName\"}", + "hide": false, + "interval": "", + "legendFormat": "$ControllerName", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Write bandwidth rate of controller $ControllerName", + "type": "timeseries" + }, + { + "datasource": "${DataSource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 26, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 42, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_controller_avg_io_response_time{endpoint=~\"$StorageName\", name=~\"$ControllerName\"}", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{name}}", + "refId": "B" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Average I/O response time of controller $ControllerName", + "type": "timeseries" + } + ], + "title": "Controller", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 18, + "panels": [ + { + "datasource": "${DataSource}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "links": [], + "mappings": [], + "noValue": "Capacity allocation of multiple storage pools cannot be displayed together.", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "gbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "FileSystem" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Lun" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 34, + "links": [], + "options": { + "displayLabels": [], + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "values": [ + "value" + ], + "width": 1 + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "exemplar": true, + "expr": "sum by(endpoint) (huawei_storage_lun_capacity{endpoint=~\"$StorageName\", name=~\".*\"})", + "hide": false, + "interval": "", + "legendFormat": "Lun", + "refId": "D" + }, + { + "exemplar": true, + "expr": "huawei_storage_storage_pool_total_capacity{endpoint=~\"$StorageName\", name=\"$Pool\"} - on(endpoint) sum by(endpoint) (huawei_storage_lun_capacity{endpoint=~\"$StorageName\"}) - sum by(endpoint) (huawei_storage_filesystem_capacity{endpoint=~\"$StorageName\", name=\"$Pool\"})", + "hide": false, + "interval": "", + "legendFormat": "Free", + "refId": "E" + }, + { + "exemplar": true, + "expr": "sum by(endpoint) (huawei_storage_filesystem_capacity{endpoint=~\"$StorageName\", name=~\".*\"})", + "hide": false, + "interval": "", + "legendFormat": "FileSystem", + "refId": "F" + } + ], + "title": "Capacity allocation of storage pool $Pool", + "type": "piechart" + }, + { + "datasource": "${DataSource}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "decimals": 0, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "gbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 4, + "y": 3 + }, + "id": 24, + "links": [], + "options": { + "displayLabels": [], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "exemplar": true, + "expr": "topk($LunNum, huawei_storage_lun_capacity{endpoint=~\"$StorageName\", name=~\".*\"})", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "{{name}}", + "refId": "C" + }, + { + "exemplar": true, + "expr": "huawei_storage_storage_pool_total_capacity{endpoint=~\"$StorageName\", name=\"$Pool\"} - on(endpoint) sum by(endpoint) (huawei_storage_lun_capacity{endpoint=~\"$StorageName\"})", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Free", + "refId": "D" + } + ], + "title": "Capacity allocation of LUNs in storage pool $Pool", + "type": "piechart" + }, + { + "datasource": "${DataSource}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "decimals": 0, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "gbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 12, + "y": 3 + }, + "id": 32, + "links": [], + "options": { + "displayLabels": [], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "exemplar": true, + "expr": "topk($FileSystemNum, huawei_storage_filesystem_capacity{endpoint=~\"$StorageName\", name=~\".*\"})", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "{{name}}", + "refId": "C" + }, + { + "exemplar": true, + "expr": "huawei_storage_storage_pool_total_capacity{endpoint=~\"$StorageName\", name=\"$Pool\"} - on(endpoint) sum by(endpoint) (huawei_storage_filesystem_capacity{endpoint=~\"$StorageName\"})", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Free", + "refId": "D" + } + ], + "title": "Capacity allocation of File System in storage pool $Pool", + "type": "piechart" + }, + { + "datasource": "${DataSource}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "gbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "空闲" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "已使用" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 20, + "y": 3 + }, + "id": 36, + "links": [], + "options": { + "displayLabels": [], + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "values": [ + "value" + ], + "width": 1 + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_storage_pool_free_capacity{endpoint=~\"$StorageName\", name=\"$Pool\"}", + "hide": false, + "interval": "", + "legendFormat": "Free", + "refId": "C" + }, + { + "exemplar": true, + "expr": "huawei_storage_storage_pool_used_capacity{endpoint=~\"$StorageName\", name=\"$Pool\"}", + "hide": false, + "interval": "", + "legendFormat": "Used", + "refId": "D" + } + ], + "title": "Capacity usage of storage pool $Pool", + "type": "piechart" + }, + { + "datasource": "${DataSource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 26, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "MiBs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 4, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_storagepool_total_bandwidth{endpoint=~\"$StorageName\", name=~\"$Pool\"}", + "hide": false, + "interval": "", + "legendFormat": "$Pool", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Total bandwidth rate of storage pool $Pool", + "type": "timeseries" + }, + { + "datasource": "${DataSource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 26, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "smooth", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 6, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_storagepool_avg_io_response_time{endpoint=~\"$StorageName\", name=~\"$Pool\"}", + "hide": false, + "interval": "", + "legendFormat": "{{name}}", + "refId": "B" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Average I/O response time of storage pool $Pool", + "type": "timeseries" + }, + { + "datasource": "${DataSource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 26, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "smooth", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 38, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DataSource}" + }, + "editorMode": "builder", + "expr": "huawei_storage_storagepool_total_iops{endpoint=~\"$StorageName\", name=~\"$Pool\"}", + "hide": false, + "legendFormat": "$Pool", + "queryType": "randomWalk", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DataSource}" + }, + "hide": false, + "queryType": "randomWalk", + "refId": "C" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Total number of I/Os in storage pool $Pool", + "type": "timeseries" + } + ], + "title": "Pool", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 12, + "panels": [ + { + "datasource": "${DataSource}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "links": [], + "mappings": [], + "noValue": "Capacity usage of multiple LUNs cannot be displayed together.", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "gbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "空闲" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "已使用" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 4 + }, + "id": 78, + "links": [], + "options": { + "displayLabels": [], + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "values": [ + "value" + ], + "width": 1 + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_lun_capacity{endpoint=~\"$StorageName\", name=\"$LunName\"} - (huawei_storage_lun_capacity_usage{endpoint=~\"$StorageName\", name=\"$LunName\"} / 100 * huawei_storage_lun_capacity{endpoint=~\"$StorageName\", name=\"$LunName\"})", + "hide": false, + "interval": "", + "legendFormat": "Free", + "refId": "A" + }, + { + "exemplar": true, + "expr": "(huawei_storage_lun_capacity_usage{endpoint=~\"$StorageName\", name=\"$LunName\"} / 100) * huawei_storage_lun_capacity{endpoint=~\"$StorageName\", name=\"$LunName\"}", + "hide": false, + "interval": "", + "legendFormat": "Used", + "refId": "B" + } + ], + "title": "Capacity usage of LUN $LunName", + "type": "piechart" + }, + { + "datasource": "${DataSource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 26, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 6, + "y": 4 + }, + "id": 124, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_lun_avg_io_response_time{endpoint=~\"$StorageName\", name=~\"$LunName\"}", + "hide": false, + "interval": "", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Average I/O response time of LUN $LunName", + "type": "timeseries" + }, + { + "datasource": "${DataSource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 26, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 0, + "y": 12 + }, + "id": 74, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_lun_total_iops{endpoint=~\"$StorageName\", name=~\"$LunName\"}", + "hide": false, + "interval": "", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Total I/O quantity of LUN $LunName", + "type": "timeseries" + }, + { + "datasource": "${DataSource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 26, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "MiBs" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 13, + "x": 11, + "y": 12 + }, + "id": 76, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_lun_total_bandwidth{endpoint=~\"$StorageName\", name=~\"$LunName\"}", + "hide": false, + "interval": "", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Total bandwidth rate of LUN $LunName", + "type": "timeseries" + } + ], + "title": "Lun", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 10, + "panels": [ + { + "datasource": "${DataSource}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "noValue": "Capacity allocation of multiple file system cannot be displayed together.", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "gbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 5 + }, + "id": 80, + "links": [], + "options": { + "displayLabels": [], + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "values": [ + "value" + ], + "width": 1 + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_filesystem_capacity{endpoint=~\"$StorageName\", name=\"$FileSystemName\"} - (huawei_storage_filesystem_capacity_usage{endpoint=~\"$StorageName\", name=\"$FileSystemName\"} / 100 * huawei_storage_filesystem_capacity{endpoint=~\"$StorageName\", name=\"$FileSystemName\"})", + "hide": false, + "interval": "", + "legendFormat": "Free", + "refId": "A" + }, + { + "exemplar": true, + "expr": "(huawei_storage_filesystem_capacity_usage{endpoint=~\"$StorageName\", name=\"$FileSystemName\"} / 100) * huawei_storage_filesystem_capacity{endpoint=~\"$StorageName\", name=\"$FileSystemName\"}", + "hide": false, + "interval": "", + "legendFormat": "Used", + "refId": "B" + } + ], + "title": "Capacity usage of file system $FileSystemName", + "type": "piechart" + }, + { + "datasource": "${DataSource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 26, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 6, + "y": 5 + }, + "id": 84, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_filesystem_ops{endpoint=~\"$StorageName\", name=~\"$FileSystemName\"}", + "interval": "", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Total I/O quantity of file system $FileSystemName", + "type": "timeseries" + }, + { + "datasource": "${DataSource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 26, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 0, + "y": 13 + }, + "id": 82, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_filesystem_avg_read_ops_response_time{endpoint=~\"$StorageName\", name=~\"$FileSystemName\"}", + "hide": false, + "interval": "", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Average read I/O response time of file system $FileSystemName", + "type": "timeseries" + }, + { + "datasource": "${DataSource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 26, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "smooth", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 13, + "x": 11, + "y": 13 + }, + "id": 86, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_filesystem_avg_write_ops_response_time{endpoint=~\"$StorageName\", name=~\"$FileSystemName\"}", + "hide": false, + "interval": "", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Average write I/O response time of file system $FileSystemName", + "type": "timeseries" + } + ], + "title": "FileSystem", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 128, + "panels": [ + { + "datasource": "${DataSource}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "noValue": "Capacity allocation of multiple file system cannot be displayed together.", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "gbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 6 + }, + "id": 126, + "links": [], + "options": { + "displayLabels": [], + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "values": [ + "value" + ], + "width": 1 + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_pv_capacity{backend=\"$StorageName\",pv_name=\"$PVName\"} - (huawei_storage_pv_capacity_usage{backend=~\"$StorageName\",pv_name=\"$PVName\"} / 100 * huawei_storage_pv_capacity{backend=~\"$StorageName\",pv_name=\"$PVName\"})", + "hide": false, + "interval": "", + "legendFormat": "Free", + "refId": "A" + }, + { + "exemplar": true, + "expr": "(huawei_storage_pv_capacity_usage{backend=~\"$StorageName\", pv_name=\"$PVName\"} / 100) * huawei_storage_pv_capacity{backend=~\"$StorageName\", pv_name=\"$PVName\"}\r", + "hide": false, + "interval": "", + "legendFormat": "Used", + "refId": "B" + } + ], + "title": "Capacity usage of PV $PVName", + "type": "piechart" + }, + { + "datasource": "${DataSource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 26, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 6, + "y": 6 + }, + "id": 136, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_pv_lun_pv_lun_total_iops{backend=~\"$StorageName\",pv_name=~\"$PVName\"}", + "interval": "", + "legendFormat": "{{pv_name}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Total I/O quantity of file system $FileSystemName", + "type": "timeseries" + }, + { + "datasource": "${DataSource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 26, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "pvc-17201a4a-a4b9-4259-ab2f-92999a6ca00b" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "graph": true, + "legend": false, + "tooltip": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 0, + "y": 14 + }, + "id": 138, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_pv_lun_avg_io_response_time{backend=~\"$StorageName\",pv_name=~\"$PVName\"}", + "hide": false, + "interval": "", + "legendFormat": "{{pv_name}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Average I/O response time of PV $PVName", + "type": "timeseries" + }, + { + "datasource": "${DataSource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 26, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "smooth", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "MiBs" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 13, + "x": 11, + "y": 14 + }, + "id": 140, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_pv_lun_total_bandwidth{backend=~\"$StorageName\",pv_name=~\"$PVName\"}", + "hide": false, + "interval": "", + "legendFormat": "{{pv_name}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Total bandwidth rate of PV $PVName", + "type": "timeseries" + } + ], + "title": "PV-Lun", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 142, + "panels": [ + { + "datasource": "${DataSource}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "noValue": "Capacity allocation of multiple file system cannot be displayed together.", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "gbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 7 + }, + "id": 144, + "links": [], + "options": { + "displayLabels": [], + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "values": [ + "value" + ], + "width": 1 + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_pv_capacity{backend=\"$StorageName\",pv_name=\"$PVName\"} - (huawei_storage_pv_capacity_usage{backend=~\"$StorageName\",pv_name=\"$PVName\"} / 100 * huawei_storage_pv_capacity{backend=~\"$StorageName\",pv_name=\"$PVName\"})", + "hide": false, + "interval": "", + "legendFormat": "Free", + "refId": "A" + }, + { + "exemplar": true, + "expr": "(huawei_storage_pv_capacity_usage{backend=~\"$StorageName\", pv_name=\"$PVName\"} / 100) * huawei_storage_pv_capacity{backend=~\"$StorageName\", pv_name=\"$PVName\"}\r", + "hide": false, + "interval": "", + "legendFormat": "Used", + "refId": "B" + } + ], + "title": "Capacity usage of PV $PVName", + "type": "piechart" + }, + { + "datasource": "${DataSource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 26, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 6, + "y": 7 + }, + "id": 130, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_pv_filesystem_ops{backend=~\"$StorageName\",pv_name=~\"$PVName\"}", + "interval": "", + "legendFormat": "{{pv_name}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Total I/O quantity of PV $PVName", + "type": "timeseries" + }, + { + "datasource": "${DataSource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 26, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 0, + "y": 15 + }, + "id": 132, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_pv_filesystem_avg_read_ops_response_time{backend=~\"$StorageName\",pv_name=~\"$PVName\"}", + "hide": false, + "interval": "", + "legendFormat": "{{pv_name}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Average read I/O response time of PV $PVName", + "type": "timeseries" + }, + { + "datasource": "${DataSource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 26, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "smooth", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 13, + "x": 11, + "y": 15 + }, + "id": 134, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.4", + "targets": [ + { + "exemplar": true, + "expr": "huawei_storage_pv_filesystem_avg_write_ops_response_time{backend=~\"$StorageName\",pv_name=~\"$PVName\"}", + "hide": false, + "interval": "", + "legendFormat": "{{pv_name}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Average write I/O response time of PV $PVName", + "type": "timeseries" + } + ], + "title": "PV-FIleSystem", + "type": "row" + } + ], + "refresh": "5s", + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "primary", + "value": "primary" + }, + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": "DataSource", + "multi": false, + "name": "DataSource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "dorado-iscsi-138", + "value": "dorado-iscsi-138" + }, + "datasource": "${DataSource}", + "definition": "label_values(huawei_storage_storage_pool_used_capacity{endpoint=~\".*\"},endpoint)", + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": "Storage Device", + "multi": false, + "name": "StorageName", + "options": [], + "query": { + "query": "label_values(huawei_storage_storage_pool_used_capacity{endpoint=~\".*\"},endpoint)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": false, + "text": "A", + "value": "A" + }, + "datasource": "${DataSource}", + "definition": "label_values(huawei_storage_controller_total_iops{endpoint=~\"$StorageName\",name=~\".*\"},name)", + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": "Controller", + "multi": false, + "name": "ControllerName", + "options": [], + "query": { + "query": "label_values(huawei_storage_controller_total_iops{endpoint=~\"$StorageName\",name=~\".*\"},name)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "test", + "value": "test" + }, + "datasource": "${DataSource}", + "definition": "label_values(huawei_storage_storagepool_total_iops{endpoint=~\"$StorageName\",name=~\".*\"},name)", + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": "Pool", + "multi": false, + "name": "Pool", + "options": [], + "query": { + "query": "label_values(huawei_storage_storagepool_total_iops{endpoint=~\"$StorageName\",name=~\".*\"},name)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": false, + "text": "10", + "value": "10" + }, + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": "Lun displayed quantity top", + "multi": false, + "name": "LunNum", + "options": [ + { + "selected": false, + "text": "1", + "value": "1" + }, + { + "selected": false, + "text": "2", + "value": "2" + }, + { + "selected": false, + "text": "3", + "value": "3" + }, + { + "selected": false, + "text": "4", + "value": "4" + }, + { + "selected": false, + "text": "5", + "value": "5" + }, + { + "selected": false, + "text": "6", + "value": "6" + }, + { + "selected": false, + "text": "7", + "value": "7" + }, + { + "selected": false, + "text": "8", + "value": "8" + }, + { + "selected": false, + "text": "9", + "value": "9" + }, + { + "selected": true, + "text": "10", + "value": "10" + } + ], + "query": "1,2,3,4,5,6,7,8,9,10", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "current": { + "selected": false, + "text": "10", + "value": "10" + }, + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": "File system displayed quantity top", + "multi": false, + "name": "FileSystemNum", + "options": [ + { + "selected": false, + "text": "1", + "value": "1" + }, + { + "selected": false, + "text": "2", + "value": "2" + }, + { + "selected": false, + "text": "3", + "value": "3" + }, + { + "selected": false, + "text": "4", + "value": "4" + }, + { + "selected": false, + "text": "5", + "value": "5" + }, + { + "selected": false, + "text": "6", + "value": "6" + }, + { + "selected": false, + "text": "7", + "value": "7" + }, + { + "selected": false, + "text": "8", + "value": "8" + }, + { + "selected": false, + "text": "9", + "value": "9" + }, + { + "selected": true, + "text": "10", + "value": "10" + } + ], + "query": "1,2,3,4,5,6,7,8,9,10", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "tags": [], + "text": [ + "pvc-a48431ac-df84-47d0-90d3-b82d72e2eaaf" + ], + "value": [ + "pvc-a48431ac-df84-47d0-90d3-b82d72e2eaaf" + ] + }, + "datasource": "${DataSource}", + "definition": "label_values(huawei_storage_lun_total_iops{endpoint=~\"$StorageName\",name=~\".*\"},name)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": "Lun", + "multi": true, + "name": "LunName", + "options": [], + "query": { + "query": "label_values(huawei_storage_lun_total_iops{endpoint=~\"$StorageName\",name=~\".*\"},name)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": true, + "tags": [], + "text": [ + "txwfs" + ], + "value": [ + "txwfs" + ] + }, + "datasource": "${DataSource}", + "definition": "label_values(huawei_storage_filesystem_capacity_usage{endpoint=~\"$StorageName\",name=~\".*\"},name)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": "FileSystem", + "multi": true, + "name": "FileSystemName", + "options": [], + "query": { + "query": "label_values(huawei_storage_filesystem_capacity_usage{endpoint=~\"$StorageName\",name=~\".*\"},name)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": false, + "text": "pvc-17201a4a-a4b9-4259-ab2f-92999a6ca00b", + "value": "pvc-17201a4a-a4b9-4259-ab2f-92999a6ca00b" + }, + "datasource": "${DataSource}", + "definition": "label_values(huawei_storage_pv_capacity{backend=~\"$StorageName\",pv_name=~\".*\"},pv_name)", + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": "pv", + "multi": true, + "name": "PVName", + "options": [ + { + "selected": true, + "text": "pvc-17201a4a-a4b9-4259-ab2f-92999a6ca00b", + "value": "pvc-17201a4a-a4b9-4259-ab2f-92999a6ca00b" + }, + { + "selected": false, + "text": "pvc-3f28ea0b-5fba-4151-bf69-c11333005eb4", + "value": "pvc-3f28ea0b-5fba-4151-bf69-c11333005eb4" + }, + { + "selected": false, + "text": "pvc-52ac4adf-4b09-49c9-8c80-ee8e59f76282", + "value": "pvc-52ac4adf-4b09-49c9-8c80-ee8e59f76282" + }, + { + "selected": false, + "text": "pvc-711d20d0-af21-441c-8294-30da379830cd", + "value": "pvc-711d20d0-af21-441c-8294-30da379830cd" + }, + { + "selected": false, + "text": "pvc-a48431ac-df84-47d0-90d3-b82d72e2eaaf", + "value": "pvc-a48431ac-df84-47d0-90d3-b82d72e2eaaf" + } + ], + "query": { + "query": "label_values(huawei_storage_pv_capacity{backend=~\"$StorageName\",pv_name=~\".*\"},pv_name)", + "refId": "StandardVariableQuery" + }, + "refresh": 0, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "OceanStor", + "uid": "bicDsaj4z", + "version": 11 +} \ No newline at end of file diff --git a/helm/huawei-csm/templates/_help.tpl b/helm/huawei-csm/templates/_help.tpl new file mode 100644 index 0000000..71a6196 --- /dev/null +++ b/helm/huawei-csm/templates/_help.tpl @@ -0,0 +1,17 @@ +{{- define "leader-election" -}} +{{- if gt ( (.Values.global).replicaCount | int ) 1 -}} +- --enable-leader-election=true +- --leader-lease-duration={{ ((.Values.global).leaderElection).leaseDuration | default "8s" }} +- --leader-renew-deadline={{ ((.Values.global).leaderElection).renewDeadline | default "6s" }} +- --leader-retry-period={{ ((.Values.global).leaderElection).retryPeriod | default "2s" }} +{{- else -}} +- --enable-leader-election=false +{{- end -}} +{{- end -}} + +{{- define "log" -}} +- --logging-module={{ .module | default "file" }} +- --log-level={{ .level | default "info" }} +- --log-file-size={{ .fileSize | default "20M" }} +- --max-backups={{ .maxBackups | default 9 }} +{{- end -}} \ No newline at end of file diff --git a/helm/huawei-csm/templates/csm-prometheus.yaml b/helm/huawei-csm/templates/csm-prometheus.yaml new file mode 100644 index 0000000..7d5f5e4 --- /dev/null +++ b/helm/huawei-csm/templates/csm-prometheus.yaml @@ -0,0 +1,304 @@ +{{ if ((.Values.features).prometheusCollector).enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csm-prometheus-sa + namespace: {{ (.Values.global).namespace | default "huawei-csm" }} + labels: + app: csm-prometheus-service +{{ if (((.Values.features).prometheusCollector).prometheusCollectorSSL).enabled }} +--- +apiVersion: v1 +kind: Secret +type: kubernetes.io/tls +metadata: + name: prometheus-ssl + namespace: {{ (.Values.global).namespace | default "huawei-csm" }} +data: + tls.crt: {{ .Files.Get (((.Values.features).prometheusCollector).prometheusCollectorSSL).certPath | b64enc }} + tls.key: {{ .Files.Get (((.Values.features).prometheusCollector).prometheusCollectorSSL).keyPath | b64enc }} +{{ end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus-collector-role + labels: + app: csm-prometheus-service +rules: + - apiGroups: [ "" ] + resources: [ "persistentvolumes","persistentvolumeclaims","pods" ] + verbs: [ "get","list" ] + - apiGroups: [ "xuanwu.huawei.io" ] + resources: [ "storagebackendclaims" ] + verbs: [ "get","list" ] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: prometheus-collector-binding + labels: + app: csm-prometheus-service +subjects: + - kind: ServiceAccount + name: csm-prometheus-sa + namespace: {{ (.Values.global).namespace | default "huawei-csm" }} +roleRef: + kind: ClusterRole + name: prometheus-collector-role + apiGroup: rbac.authorization.k8s.io + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cmi-collector-role + labels: + app: csm-prometheus-service +rules: + - apiGroups: [ "xuanwu.huawei.io" ] + resources: [ "storagebackendclaims" ] + verbs: [ "get" ] + - apiGroups: [ "" ] + resources: [ "secrets" ] + verbs: [ "get" ] + - apiGroups: [ "" ] + resources: [ "configmaps" ] + verbs: [ "create", "get", "update" ] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cmi-collector-binding + labels: + app: csm-prometheus-service +subjects: + - kind: ServiceAccount + name: csm-prometheus-sa + namespace: {{ (.Values.global).namespace | default "huawei-csm" }} +roleRef: + kind: ClusterRole + name: cmi-collector-role + apiGroup: rbac.authorization.k8s.io + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: csm-prometheus-service + namespace: {{ (.Values.global).namespace | default "huawei-csm" }} + labels: + app: csm-prometheus-service +spec: + replicas: {{ (.Values.global).replicaCount | default 1 }} + selector: + matchLabels: + app: csm-prometheus-service + template: + metadata: + labels: + app: csm-prometheus-service + spec: + {{- if ((.Values.features).prometheusCollector).nodeSelector }} + nodeSelector: + {{- toYaml ((.Values.features).prometheusCollector).nodeSelector | nindent 8 }} + {{- end }} + {{- if (.Values.global).balancedDeploy }} + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - csm-storage-service + topologyKey: kubernetes.io/hostname + weight: 100 + {{- end}} + tolerations: + - key: node.kubernetes.io/not-ready + operator: Exists + effect: NoExecute + tolerationSeconds: {{ ((.Values.global).tolerations).notReadySeconds | default 15 }} + - key: node.kubernetes.io/unreachable + operator: Exists + effect: NoExecute + tolerationSeconds: {{ ((.Values.global).tolerations).unreachableSeconds | default 15 }} + serviceAccount: csm-prometheus-sa + serviceAccountName: csm-prometheus-sa + containers: + - name: liveness-probe + args: + - --cmi-address={{ ((.Values.features).cmi).socket | default "/cmi/cmi.sock" }} + - --ip-address=$(POD_IP) + - --healthz-port={{ (.Values.global).healthPort | default 9808 }} + - --log-file-dir=/var/log/huawei-csm/csm-prometheus-service + - --log-file=liveness-probe + {{- include "log" (.Values.global).logging | nindent 12 }} + image: {{ required "Must provide the Values.global.imageRepo" .Values.global.imageRepo + }}{{ required "Must provide the .Values.images.livenessProbe" .Values.images.livenessProbe }} + imagePullPolicy: {{ (.Values.global).pullPolicy | default "IfNotPresent" }} + env: + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - mountPath: /cmi + name: socket-dir + - mountPath: /var/log + name: log + resources: + requests: + cpu: {{ .Values.containerResourcesSet.prometheusService.livenessProbe.requests.cpu }} + memory: {{ .Values.containerResourcesSet.prometheusService.livenessProbe.requests.memory }} + limits: + cpu: {{ .Values.containerResourcesSet.prometheusService.livenessProbe.limits.cpu }} + memory: {{ .Values.containerResourcesSet.prometheusService.livenessProbe.limits.memory }} + - name: prometheus-collector + image: {{ required "Must provide the Values.global.imageRepo" .Values.global.imageRepo + }}{{ required "Must provide the .Values.images.prometheusCollector" .Values.images.prometheusCollector }} + env: + - name: ENDPOINT + value: {{ ((.Values.features).cmi).socket | default "/cmi/cmi.sock" }} + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + imagePullPolicy: {{ (.Values.global).pullPolicy | default "IfNotPresent" }} + args: + - --cmi-address=$(ENDPOINT) + - --ip-address=$(POD_IP) + - --exporter-port=8887 + - --use-https={{(((.Values.features).prometheusCollector).prometheusCollectorSSL).enabled }} + - --log-file-dir=/var/log/huawei-csm/csm-prometheus-service + - --log-file=prometheus-collector + - --csi-driver-name={{ (.Values.global).csiDriverName }} + {{- include "log" .Values.global.logging | nindent 12 }} + volumeMounts: + - mountPath: /cmi + name: socket-dir + - mountPath: /var/log + name: log + - mountPath: /etc/localtime + name: host-time + {{- if (((.Values.features).prometheusCollector).prometheusCollectorSSL).enabled }} + - name: secret-volume + mountPath: /etc/secret-volume + readOnly: true + {{- end}} + livenessProbe: + failureThreshold: 5 + httpGet: + {{- if (((.Values.features).prometheusCollector).prometheusCollectorSSL).enabled }} + scheme: HTTPS + {{- end }} + path: /healthz + port: 8887 + initialDelaySeconds: 10 + periodSeconds: 60 + timeoutSeconds: 3 + resources: + requests: + cpu: {{ .Values.containerResourcesSet.prometheusService.prometheusCollector.requests.cpu }} + memory: {{ .Values.containerResourcesSet.prometheusService.prometheusCollector.requests.memory }} + limits: + cpu: {{ .Values.containerResourcesSet.prometheusService.prometheusCollector.limits.cpu }} + memory: {{ .Values.containerResourcesSet.prometheusService.prometheusCollector.limits.memory }} + - name: cmi-controller + image: {{ required "Must provide the Values.global.imageRepo" .Values.global.imageRepo + }}{{ required "Must provide the .Values.images.containerMonitorInterface" + .Values.images.containerMonitorInterface }} + env: + - name: ENDPOINT + value: {{ ((.Values.features).cmi).socket | default "/cmi/cmi.sock" }} + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + imagePullPolicy: {{ (.Values.global).pullPolicy | default "IfNotPresent" }} + args: + - --cmi-address=$(ENDPOINT) + - --cmi-name=cmi.huawei.com + - --page-size=100 + - --backend-namespace=huawei-csi + - --log-file-dir=/var/log/huawei-csm/csm-prometheus-service + - --log-file=cmi-service + {{- include "log" .Values.global.logging | nindent 12 }} + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 60 + timeoutSeconds: 3 + ports: + - containerPort: {{ (.Values.global).healthPort | default 9808 }} + name: healthz + protocol: TCP + volumeMounts: + - mountPath: /cmi + name: socket-dir + - mountPath: /var/log/ + name: log + - mountPath: /etc/localtime + name: host-time + resources: + requests: + cpu: {{ .Values.containerResourcesSet.prometheusService.cmiController.requests.cpu }} + memory: {{ .Values.containerResourcesSet.prometheusService.cmiController.requests.memory }} + limits: + cpu: {{ .Values.containerResourcesSet.prometheusService.cmiController.limits.cpu }} + memory: {{ .Values.containerResourcesSet.prometheusService.cmiController.limits.memory }} + volumes: + - emptyDir: { } + name: socket-dir + - hostPath: + path: /var/log/ + type: Directory + name: log + - hostPath: + path: /etc/localtime + type: File + name: host-time + {{- if (((.Values.features).prometheusCollector).prometheusCollectorSSL).enabled }} + - name: secret-volume + secret: + secretName: prometheus-ssl + defaultMode: 0400 + {{- end}} +--- +apiVersion: v1 +kind: Service +metadata: + name: csm-prometheus-service + namespace: {{ (.Values.global).namespace | default "huawei-csm" }} + labels: + app: csm-prometheus-service +spec: + selector: + app: csm-prometheus-service + type: NodePort + ports: + - name: prometheus-collector + protocol: TCP + port: 8887 + targetPort: 8887 + nodePort: {{ ((.Values.features).prometheusCollector).nodePort | default 30074 }} +{{ end }} diff --git a/helm/huawei-csm/templates/csm-storage.yaml b/helm/huawei-csm/templates/csm-storage.yaml new file mode 100644 index 0000000..623a3dc --- /dev/null +++ b/helm/huawei-csm/templates/csm-storage.yaml @@ -0,0 +1,274 @@ +{{ if ((.Values.features).storageTopo).enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: csm-storage-service + name: csm-storage-sa + namespace: {{ (.Values.global).namespace | default "huawei-csm" }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: topo-service-role + labels: + app: csm-storage-service +rules: + - apiGroups: [ "" ] + resources: [ "secrets", "events", "configmaps" ] + verbs: [ "create", "get", "update", "delete" ] + - apiGroups: [ "coordination.k8s.io" ] + resources: [ "leases" ] + verbs: [ "create", "get", "update", "delete" ] + - apiGroups: [ "xuanwu.huawei.io" ] + resources: [ "resourcetopologies", "resourcetopologies/status" ] + verbs: [ "create", "get", "list", "watch", "update", "delete" ] + - apiGroups: [ "*" ] + resources: [ "*" ] + verbs: [ "get", "list", "watch" ] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cmi-controller-role + labels: + app: csm-storage-service +rules: + - apiGroups: [ "xuanwu.huawei.io" ] + resources: [ "storagebackendclaims" ] + verbs: [ "get" ] + - apiGroups: [ "" ] + resources: [ "secrets" ] + verbs: [ "get" ] + - apiGroups: [ "" ] + resources: [ "configmaps" ] + verbs: [ "create", "get", "update" ] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: topo-service-binding + labels: + app: csm-storage-service +subjects: + - kind: ServiceAccount + name: csm-storage-sa + namespace: {{ (.Values.global).namespace | default "huawei-csm" }} +roleRef: + kind: ClusterRole + name: topo-service-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cmi-controller-binding + labels: + app: csm-storage-service +subjects: + - kind: ServiceAccount + name: csm-storage-sa + namespace: {{ (.Values.global).namespace | default "huawei-csm" }} +roleRef: + kind: ClusterRole + name: cmi-controller-role + apiGroup: rbac.authorization.k8s.io + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: csm-storage-service + name: csm-storage-service + namespace: {{ (.Values.global).namespace | default "huawei-csm" }} +spec: + progressDeadlineSeconds: 600 + replicas: {{ (.Values.global).replicaCount | default 1 }} + revisionHistoryLimit: 10 + selector: + matchLabels: + app: csm-storage-service + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + creationTimestamp: null + labels: + app: csm-storage-service + spec: + {{- if ((.Values.features).storageTopo).nodeSelector }} + nodeSelector: + {{- toYaml ((.Values.features).storageTopo).nodeSelector | nindent 8 }} + {{- end }} + {{- if (.Values.global).balancedDeploy }} + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - csm-prometheus-service + topologyKey: kubernetes.io/hostname + weight: 100 + {{- end}} + tolerations: + - key: node.kubernetes.io/not-ready + operator: Exists + effect: NoExecute + tolerationSeconds: {{ ((.Values.global).tolerations).notReadySeconds | default 15 }} + - key: node.kubernetes.io/unreachable + operator: Exists + effect: NoExecute + tolerationSeconds: {{ ((.Values.global).tolerations).unreachableSeconds | default 15 }} + serviceAccount: csm-storage-sa + serviceAccountName: csm-storage-sa + containers: + - name: liveness-probe + args: + - --cmi-address={{ ((.Values.features).cmi).socket | default "/cmi/cmi.sock" }} + - --ip-address=$(POD_IP) + - --healthz-port={{ (.Values.global).healthPort | default 9808 }} + - --log-file-dir=/var/log/huawei-csm/csm-storage-service + - --log-file=liveness-prob + {{- include "log" .Values.global.logging | nindent 12 }} + image: {{ required "Must provide the Values.global.imageRepo" .Values.global.imageRepo + }}{{ required "Must provide the .Values.images.livenessProbe" .Values.images.livenessProbe }} + imagePullPolicy: {{ (.Values.global).pullPolicy | default "IfNotPresent" }} + env: + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - mountPath: /cmi + name: socket-dir + - mountPath: /var/log + name: log + resources: + requests: + cpu: {{ .Values.containerResourcesSet.storageService.livenessProbe.requests.cpu }} + memory: {{ .Values.containerResourcesSet.storageService.livenessProbe.requests.memory }} + limits: + cpu: {{ .Values.containerResourcesSet.storageService.livenessProbe.limits.cpu }} + memory: {{ .Values.containerResourcesSet.storageService.livenessProbe.limits.memory }} + - name: cmi-controller + image: {{ required "Must provide the Values.global.imageRepo" .Values.global.imageRepo + }}{{ required "Must provide the .Values.images.containerMonitorInterface" + .Values.images.containerMonitorInterface }} + env: + - name: ENDPOINT + value: {{ ((.Values.features).cmi).socket | default "/cmi/cmi.sock" }} + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + imagePullPolicy: {{ (.Values.global).pullPolicy | default "IfNotPresent" }} + args: + - --cmi-address=$(ENDPOINT) + - --cmi-name=cmi.huawei.com + - --page-size=100 + - --backend-namespace=huawei-csi + - --log-file-dir=/var/log/huawei-csm/csm-storage-service + - --log-file=cmi-service + {{- include "log" .Values.global.logging | nindent 12 }} + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 60 + timeoutSeconds: 3 + ports: + - containerPort: {{ (.Values.global).healthPort | default 9808 }} + name: healthz + protocol: TCP + volumeMounts: + - mountPath: /cmi + name: socket-dir + - mountPath: /var/log/ + name: log + resources: + requests: + cpu: {{ .Values.containerResourcesSet.storageService.cmiController.requests.cpu }} + memory: {{ .Values.containerResourcesSet.storageService.cmiController.requests.memory }} + limits: + cpu: {{ .Values.containerResourcesSet.storageService.cmiController.limits.cpu }} + memory: {{ .Values.containerResourcesSet.storageService.cmiController.limits.memory }} + - args: + - --cmi-address=$(ENDPOINT) + - --rt-retry-base-delay={{ ((.Values.features).storageTopo).rtRetryBaseDelay | default "5s" }} + - --pv-retry-base-delay={{ ((.Values.features).storageTopo).pvRetryBaseDelay | default "5s" }} + - --pod-retry-base-delay={{ ((.Values.features).storageTopo).podRetryBaseDelay | default "5s" }} + - --rt-retry-max-delay={{ ((.Values.features).storageTopo).rtRetryMaxDelay | default "5m" }} + - --pv-retry-max-delay={{ ((.Values.features).storageTopo).pvRetryMaxDelay | default "1m" }} + - --pod-retry-max-delay={{ ((.Values.features).storageTopo).podRetryMaxDelay | default "1m" }} + - --resync-period={{ ((.Values.features).storageTopo).resyncPeriod | default "15m" }} + - --csi-driver-name={{ (.Values.global).csiDriverName }} + {{- include "leader-election" . | nindent 12 }} + - --log-file-dir=/var/log/huawei-csm/csm-storage-service + - --log-file=topo-service + {{- include "log" .Values.global.logging | nindent 12 }} + env: + - name: ENDPOINT + value: {{ ((.Values.features).cmi).socket | default "/cmi/cmi.sock" }} + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CLUSTER_NAME + value: {{ (.Values.cluster).name }} + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: {{ required "Must provide the Values.global.imageRepo" .Values.global.imageRepo + }}{{ required "Must provide the .Values.images.topoService" .Values.images.topoService }} + imagePullPolicy: {{ (.Values.global).pullPolicy | default "IfNotPresent" }} + name: topo-service + resources: + requests: + cpu: {{ .Values.containerResourcesSet.storageService.topoService.requests.cpu }} + memory: {{ .Values.containerResourcesSet.storageService.topoService.requests.memory }} + limits: + cpu: {{ .Values.containerResourcesSet.storageService.topoService.limits.cpu }} + memory: {{ .Values.containerResourcesSet.storageService.topoService.limits.memory }} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /cmi + name: socket-dir + - mountPath: /var/log + name: log + - mountPath: /etc/localtime + name: host-time + volumes: + - emptyDir: { } + name: socket-dir + - hostPath: + path: /var/log/ + type: Directory + name: log + - hostPath: + path: /etc/localtime + type: File + name: host-time +{{ end }} diff --git a/helm/huawei-csm/upload-image.sh b/helm/huawei-csm/upload-image.sh new file mode 100644 index 0000000..4e0d6cc --- /dev/null +++ b/helm/huawei-csm/upload-image.sh @@ -0,0 +1,199 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +PROG="${0}" +PUSH_IMAGE_OUTPUT="/tmp/huawei-csm-upload.$$.out" +VERSION={{version}} +IMAGE_LOCAL_PATH="/opt/huawei-csm/image/" +IMAGES=("csm-prometheus-collector" "csm-cmi" "csm-topo-service" "csm-liveness-probe") + +# export the name of the debug log, so child processes will see it +export DEBUGLOG="${SCRIPTDIR}/upload-debug.log" + +source "$SCRIPTDIR"/common.sh +if [ -f "${DEBUGLOG}" ]; then + rm -f "${DEBUGLOG}" +fi + +# usage will print command execution help and then exit +function usage() { + decho + decho "Help for $PROG" + decho + decho "Usage: $PROG options..." + decho "Options:" + decho " Required" + decho " --imageRepo[=] Project image repository address, such as 'docker.io/library/'" + + exit 0 +} + +while getopts ":h-:" optchar; do + case "${optchar}" in + -) + case "${OPTARG}" in + # IMAGE REPO + imageRepo) + imageRepo="${!OPTIND}" + OPTIND=$((OPTIND + 1)) + ;; + imageRepo=*) + imageRepo=${OPTARG#*=} + ;; + *) + decho "Unknown option --${OPTARG}" + decho "For help, run $PROG -h" + exit 1 + ;; + esac + ;; + h) + usage + ;; + *) + decho "Unknown option -${OPTARG}" + decho "For help, run $PROG -h" + exit 1 + ;; + esac +done + +# validate_params will validate the parameters passed in +function validate_params() { + # IMAGE REPO + if [ -z "${imageRepo}" ]; then + decho "imageRepo must be specified" + usage + exit 1 + fi +} + +# print header information +function header() { + log section "Uploading Huawei-CSM Images to Image Repository" +} + +# check docker command +function check_docker_command() { + docker --help >&/dev/null || { + decho "docker required for upload... exiting" + exit 2 + } +} + +# load all images locally +function load_images() { + # shellcheck disable=SC2086 + for image in ${IMAGES[@]}; do + run_command docker load -i ${IMAGE_LOCAL_PATH}${image}-${VERSION}.tar >/dev/null 2>&1 + if [ $? -ne 0 ]; then + error=1 + log step_failure + exit 2 + fi + done + log step_success +} + +# tag all images +function tag_images() { + for image in ${IMAGES[@]}; do + run_command docker tag ${image}:${VERSION} ${imageRepo}${image}:${VERSION} >/dev/null 2>&1 + if [ $? -ne 0 ]; then + error=1 + log step_failure + exit 2 + fi + done + log step_success +} + +# push all images +function push_images() { + error=0 + # shellcheck disable=SC2086 + for image in ${IMAGES[@]}; do + run_command docker push ${imageRepo}${image}:${VERSION} >"${PUSH_IMAGE_OUTPUT}" 2>&1 + if [ $? -ne 0 ]; then + error=1 + log step_failure + return 1 + fi + done + log step_success + return 0 +} + +function cleanup_images() { + # shellcheck disable=SC2086 + for image in ${IMAGES[@]}; do + run_command docker rmi ${imageRepo}${image}:${VERSION} >/dev/null 2>&1 + # shellcheck disable=SC2181 + if [ $? -ne 0 ]; then + error=1 + log step_failure + exit 2 + fi + run_command docker rmi ${image}:${VERSION} >/dev/null 2>&1 + if [ $? -ne 0 ]; then + error=1 + log step_failure + exit 2 + fi + done + log step_success +} + +# upload the xuanwu images to image repository +function upload() { + log step "Start to upload images" + + log step_success + + log arrow + log smart_step "Start to load the images" "small" + load_images + + log arrow + log smart_step "Start to tag the images" "small" + tag_images + + log arrow + log smart_step "Start to push the images to the image repository" "small" + push_images + if [ $? -ne 0 ]; then + cat "${PUSH_IMAGE_OUTPUT}" + warning "Can not push the image to ${imageRepo}, plz check the connection of the image repository." + cleanup_images + exit 1 + fi + + log arrow + log smart_step "Start to cleanup local images" "small" + cleanup_images + + log arrow + log smart_step "Images uploaded" "small" + log step_success +} + +# main used to upload the image step by step +function main() { + validate_params + check_docker_command + header + upload +} + +main diff --git a/helm/huawei-csm/values.yaml b/helm/huawei-csm/values.yaml new file mode 100644 index 0000000..1daec69 --- /dev/null +++ b/helm/huawei-csm/values.yaml @@ -0,0 +1,148 @@ +# Default values for huawei-csm. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +##################################Follows are REQUIRED################################## +# all global values across charts go here +global: + # the number of replicas of the pod. + # if the replicaCount greater than 1, then the function of leader election will be enabled. + replicaCount: 1 + imageRepo: + logging: + module: file + level: info + fileSize: 20M + maxBackups: 9 + # leaderElection configuration + leaderElection: + leaseDuration: "8s" + renewDeadline: "6s" + retryPeriod: "2s" + # csiDriverName: the csi driver name + # Default value: "csi.huawei.com" + csiDriverName: "csi.huawei.com" + # Allowed values: + # true: prefer to schedule pods with different services to different nodes + # false: use the default pod scheduling policy of Kubernetes + # Default value: true + # This field will be inactive when it conflicts with the nodeSelector. + balancedDeploy: true + +# all supported features +features: + # prometheusCollector: allowed prometheus use the storage to collect metrics + prometheusCollector: + # Allowed values: + # true: enable prometheus collect feature + # false: disable prometheus collect feature + # Default value: true + enabled: true + # nodePort: port the containers are provided to the prometheus + # Default value: 30074 + nodePort: 30074 + # prometheusCollectorSSL: parameters required to start https + prometheusCollectorSSL: + # Allowed values: + # true: enable https, when set it certPath and keyPath must set + # false: disable https, use http + # Default value: true + enabled: true + # The Path of cert, need to be placed in the huawei-csm directory + certPath: "" + # The Path of key, need to be placed in the huawei-csm directory + keyPath: "" + # nodeSelector: Define node selection constraints for prometheusCollector pods. + # For the pod to be eligible to run on a node, the node must have each + # of the indicated key-value pairs as labels. + # Leave as blank to consider all nodes + # Allowed values: map of key-value pairs + # Default value: None + nodeSelector: + # Uncomment if you wish the service scheduled to the node with Specific Labels + # kubernetes.io/hostname: "" + + # storageTopo: allow to provision pv/pod to storage. + storageTopo: + # Allowed values: + # true: enable prometheus collect feature + # false: disable prometheus collect feature + # Default value: true + enabled: true + # rtRetryInterval: the max delay for retrying a rt task + # Default value: "5m" + rtRetryMaxDelay: "5m" + # pvRetryIntervalMax: the max delay for retrying a pv task + # Default value: "1m" + pvRetryMaxDelay: "1m" + # podRetryInterval: the max delay for retrying a pod task + # Default value: "1m" + podRetryMaxDelay: "1m" + # resyncPeriod: the interval for refreshing the resourceTopologies on the cluster + # Default value: "15m" + resyncPeriod: "15m" + # nodeSelector: Define node selection constraints for storageTopo pods. + # For the pod to be eligible to run on a node, the node must have each + # of the indicated key-value pairs as labels. + # Leave as blank to consider all nodes + # Allowed values: map of key-value pairs + # Default value: None + nodeSelector: + # Uncomment if you wish the service scheduled to the node with Specific Labels + # kubernetes.io/hostname: "" + +cluster: + name: "kubernetes" + +images: + prometheusCollector: csm-prometheus-collector:{{version}} + topoService: csm-topo-service:{{version}} + containerMonitorInterface: csm-cmi:{{version}} + livenessProbe: csm-liveness-probe:{{version}} + +# limits and requests of containers +containerResourcesSet: + prometheusService: + livenessProbe: + requests: + cpu: 10m + memory: 128Mi + limits: + cpu: 100m + memory: 128Mi + prometheusCollector: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 300m + memory: 512Mi + cmiController: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 300m + memory: 512Mi + storageService: + livenessProbe: + requests: + cpu: 10m + memory: 128Mi + limits: + cpu: 100m + memory: 128Mi + cmiController: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 300m + memory: 512Mi + topoService: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 300m + memory: 512Mi diff --git a/manual/huawei-csm/templates/csm-prometheus.yaml b/manual/huawei-csm/templates/csm-prometheus.yaml new file mode 100644 index 0000000..c9b6540 --- /dev/null +++ b/manual/huawei-csm/templates/csm-prometheus.yaml @@ -0,0 +1,289 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csm-prometheus-sa + namespace: huawei-csm + labels: + app: csm-prometheus-service +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus-collector-role + labels: + app: csm-prometheus-service +rules: + - apiGroups: [ "" ] + resources: [ "persistentvolumes","persistentvolumeclaims","pods" ] + verbs: [ "get","list" ] + - apiGroups: [ "xuanwu.huawei.io" ] + resources: [ "storagebackendclaims" ] + verbs: [ "get","list" ] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: prometheus-collector-binding + labels: + app: csm-prometheus-service +subjects: + - kind: ServiceAccount + name: csm-prometheus-sa + namespace: huawei-csm +roleRef: + kind: ClusterRole + name: prometheus-collector-role + apiGroup: rbac.authorization.k8s.io + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cmi-collector-role + labels: + app: csm-prometheus-service +rules: + - apiGroups: [ "xuanwu.huawei.io" ] + resources: [ "storagebackendclaims" ] + verbs: [ "get" ] + - apiGroups: [ "" ] + resources: [ "secrets" ] + verbs: [ "get" ] + - apiGroups: [ "" ] + resources: [ "configmaps" ] + verbs: [ "create", "get", "update" ] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cmi-collector-binding + labels: + app: csm-prometheus-service +subjects: + - kind: ServiceAccount + name: csm-prometheus-sa + namespace: huawei-csm +roleRef: + kind: ClusterRole + name: cmi-collector-role + apiGroup: rbac.authorization.k8s.io + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: csm-prometheus-service + namespace: huawei-csm + labels: + app: csm-prometheus-service +spec: + replicas: 1 + selector: + matchLabels: + app: csm-prometheus-service + template: + metadata: + labels: + app: csm-prometheus-service + spec: +# uncomment if you wish to configure selection constraints for csm-prometheus-service pods +# nodeSelector: +# kubernetes.io/hostname: "" + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - csm-storage-service + topologyKey: kubernetes.io/hostname + weight: 100 + tolerations: + - key: node.kubernetes.io/not-ready + operator: Exists + effect: NoExecute + tolerationSeconds: 15 + - key: node.kubernetes.io/unreachable + operator: Exists + effect: NoExecute + tolerationSeconds: 15 + serviceAccount: csm-prometheus-sa + serviceAccountName: csm-prometheus-sa + containers: + - name: liveness-probe + args: + - --cmi-address=/cmi/cmi.sock + - --ip-address=$(POD_IP) + - --healthz-port=9808 + - --log-file-dir=/var/log/huawei-csm/csm-prometheus-service + - --log-file=liveness-probe + - --logging-module=file + - --log-level=info + - --log-file-size=20M + - --max-backups=9 + image: csm-liveness-probe:{{version}} + imagePullPolicy: IfNotPresent + env: + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - mountPath: /cmi + name: socket-dir + - mountPath: /var/log + name: log + resources: + requests: + cpu: 10m + memory: 128Mi + limits: + cpu: 100m + memory: 128Mi + - name: prometheus-collector + image: csm-prometheus-collector:{{version}} + env: + - name: ENDPOINT + value: /cmi/cmi.sock + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + imagePullPolicy: IfNotPresent + args: + - --cmi-address=$(ENDPOINT) + - --ip-address=$(POD_IP) + - --exporter-port=8887 + - --use-https=false # modify the value to "true" if configured the SSL cert + - --log-file-dir=/var/log/huawei-csm/csm-prometheus-service + - --log-file=prometheus-collector + - --csi-driver-name=csi.huawei.com + - --logging-module=file + - --log-level=info + - --log-file-size=20M + - --max-backups=9 + volumeMounts: + - mountPath: /cmi + name: socket-dir + - mountPath: /var/log + name: log + - mountPath: /etc/localtime + name: host-time +# uncomment if configured the SSL cert +# - name: secret-volume +# mountPath: /etc/secret-volume +# readOnly: true + livenessProbe: + failureThreshold: 5 + httpGet: +# uncomment if configured the SSL cert +# scheme: HTTPS + path: /healthz + port: 8887 + initialDelaySeconds: 10 + periodSeconds: 60 + timeoutSeconds: 3 + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 300m + memory: 512Mi + - name: cmi-controller + image: csm-cmi:{{version}} + env: + - name: ENDPOINT + value: /cmi/cmi.sock + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + imagePullPolicy: IfNotPresent + args: + - --cmi-address=$(ENDPOINT) + - --cmi-name=cmi.huawei.com + - --page-size=100 + - --backend-namespace=huawei-csi + - --log-file-dir=/var/log/huawei-csm/csm-prometheus-service + - --log-file=cmi-service + - --logging-module=file + - --log-level=info + - --log-file-size=20M + - --max-backups=9 + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 60 + timeoutSeconds: 3 + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + volumeMounts: + - mountPath: /cmi + name: socket-dir + - mountPath: /var/log/ + name: log + - mountPath: /etc/localtime + name: host-time + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 300m + memory: 512Mi + volumes: + - emptyDir: { } + name: socket-dir + - hostPath: + path: /var/log/ + type: Directory + name: log + - hostPath: + path: /etc/localtime + type: File + name: host-time +# uncomment if configured the SSL cert +# - name: secret-volume +# secret: +# secretName: prometheus-ssl +# defaultMode: 0400 +--- +apiVersion: v1 +kind: Service +metadata: + name: csm-prometheus-service + namespace: huawei-csm + labels: + app: csm-prometheus-service +spec: + selector: + app: csm-prometheus-service + type: NodePort + ports: + - name: prometheus-collector + protocol: TCP + port: 8887 + targetPort: 8887 + nodePort: 30074 diff --git a/manual/huawei-csm/templates/csm-storage.yaml b/manual/huawei-csm/templates/csm-storage.yaml new file mode 100644 index 0000000..63260e6 --- /dev/null +++ b/manual/huawei-csm/templates/csm-storage.yaml @@ -0,0 +1,277 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: csm-storage-service + name: csm-storage-sa + namespace: huawei-csm + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: topo-service-role + labels: + app: csm-storage-service +rules: + - apiGroups: [ "" ] + resources: [ "secrets", "events", "configmaps" ] + verbs: [ "create", "get", "update", "delete" ] + - apiGroups: [ "coordination.k8s.io" ] + resources: [ "leases" ] + verbs: [ "create", "get", "update", "delete" ] + - apiGroups: [ "xuanwu.huawei.io" ] + resources: [ "resourcetopologies", "resourcetopologies/status" ] + verbs: [ "create", "get", "list", "watch", "update", "delete" ] + - apiGroups: [ "*" ] + resources: [ "*" ] + verbs: [ "get", "list", "watch" ] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cmi-controller-role + labels: + app: csm-storage-service +rules: + - apiGroups: [ "xuanwu.huawei.io" ] + resources: [ "storagebackendclaims" ] + verbs: [ "get" ] + - apiGroups: [ "" ] + resources: [ "secrets" ] + verbs: [ "get" ] + - apiGroups: [ "" ] + resources: [ "configmaps" ] + verbs: [ "create", "get", "update" ] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: topo-service-binding + labels: + app: csm-storage-service +subjects: + - kind: ServiceAccount + name: csm-storage-sa + namespace: huawei-csm +roleRef: + kind: ClusterRole + name: topo-service-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cmi-controller-binding + labels: + app: csm-storage-service +subjects: + - kind: ServiceAccount + name: csm-storage-sa + namespace: huawei-csm +roleRef: + kind: ClusterRole + name: cmi-controller-role + apiGroup: rbac.authorization.k8s.io + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: csm-storage-service + name: csm-storage-service + namespace: huawei-csm +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: csm-storage-service + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + creationTimestamp: null + labels: + app: csm-storage-service + spec: +# uncomment if you wish to configure selection constraints for csm-storage-service pods +# nodeSelector: +# kubernetes.io/hostname: "" + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - csm-prometheus-service + topologyKey: kubernetes.io/hostname + weight: 100 + tolerations: + - key: node.kubernetes.io/not-ready + operator: Exists + effect: NoExecute + tolerationSeconds: 15 + - key: node.kubernetes.io/unreachable + operator: Exists + effect: NoExecute + tolerationSeconds: 15 + serviceAccount: csm-storage-sa + serviceAccountName: csm-storage-sa + containers: + - name: liveness-probe + args: + - --cmi-address=/cmi/cmi.sock + - --ip-address=$(POD_IP) + - --healthz-port=9808 + - --log-file-dir=/var/log/huawei-csm/csm-storage-service + - --log-file=liveness-prob + - --logging-module=file + - --log-level=info + - --log-file-size=20M + - --max-backups=9 + image: csm-liveness-probe:{{version}} + imagePullPolicy: IfNotPresent + env: + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - mountPath: /cmi + name: socket-dir + - mountPath: /var/log + name: log + resources: + requests: + cpu: 10m + memory: 128Mi + limits: + cpu: 100m + memory: 128Mi + - name: cmi-controller + image: csm-cmi:{{version}} + env: + - name: ENDPOINT + value: /cmi/cmi.sock + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + imagePullPolicy: IfNotPresent + args: + - --cmi-address=$(ENDPOINT) + - --cmi-name=cmi.huawei.com + - --page-size=100 + - --backend-namespace=huawei-csi + - --log-file-dir=/var/log/huawei-csm/csm-storage-service + - --log-file=cmi-service + - --logging-module=file + - --log-level=info + - --log-file-size=20M + - --max-backups=9 + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 60 + timeoutSeconds: 3 + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + volumeMounts: + - mountPath: /cmi + name: socket-dir + - mountPath: /var/log/ + name: log + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 300m + memory: 512Mi + - args: + - --cmi-address=$(ENDPOINT) + - --rt-retry-base-delay=5s + - --pv-retry-base-delay=5s + - --pod-retry-base-delay=5s + - --rt-retry-max-delay=5m + - --pv-retry-max-delay=1m + - --pod-retry-max-delay=1m + - --resync-period=15m + - --csi-driver-name=csi.huawei.com + - --enable-leader-election=false + - --leader-lease-duration=8s + - --leader-renew-deadline=6s + - --leader-retry-period=2s + - --log-file-dir=/var/log/huawei-csm/csm-storage-service + - --log-file=topo-service + - --logging-module=file + - --log-level=info + - --log-file-size=20M + - --max-backups=9 + env: + - name: ENDPOINT + value: /cmi/cmi.sock + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CLUSTER_NAME + value: kubernetes + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: csm-topo-service:{{version}} + imagePullPolicy: IfNotPresent + name: topo-service + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 300m + memory: 512Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /cmi + name: socket-dir + - mountPath: /var/log + name: log + - mountPath: /etc/localtime + name: host-time + volumes: + - emptyDir: { } + name: socket-dir + - hostPath: + path: /var/log/ + type: Directory + name: log + - hostPath: + path: /etc/localtime + type: File + name: host-time \ No newline at end of file diff --git a/manual/huawei-csm/templates/prometheus-ssl.yaml b/manual/huawei-csm/templates/prometheus-ssl.yaml new file mode 100644 index 0000000..8843561 --- /dev/null +++ b/manual/huawei-csm/templates/prometheus-ssl.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +type: kubernetes.io/tls +metadata: + name: prometheus-ssl + namespace: huawei-csm +data: + tls.crt: + tls.key: \ No newline at end of file diff --git a/pkg/client/clientset/versioned/clientset.go b/pkg/client/clientset/versioned/clientset.go new file mode 100644 index 0000000..080a27d --- /dev/null +++ b/pkg/client/clientset/versioned/clientset.go @@ -0,0 +1,117 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package versioned + +import ( + "fmt" + xuanwuv1 "github.com/huawei/csm/v2/pkg/client/clientset/versioned/typed/xuanwu/v1" + "net/http" + + discovery "k8s.io/client-go/discovery" + rest "k8s.io/client-go/rest" + flowcontrol "k8s.io/client-go/util/flowcontrol" +) + +type Interface interface { + Discovery() discovery.DiscoveryInterface + XuanwuV1() xuanwuv1.XuanwuV1Interface +} + +// Clientset contains the clients for groups. +type Clientset struct { + *discovery.DiscoveryClient + xuanwuV1 *xuanwuv1.XuanwuV1Client +} + +// XuanwuV1 retrieves the XuanwuV1Client +func (c *Clientset) XuanwuV1() xuanwuv1.XuanwuV1Interface { + return c.xuanwuV1 +} + +// Discovery retrieves the DiscoveryClient +func (c *Clientset) Discovery() discovery.DiscoveryInterface { + if c == nil { + return nil + } + return c.DiscoveryClient +} + +// NewForConfig creates a new Clientset for the given config. +// If config's RateLimiter is not set and QPS and Burst are acceptable, +// NewForConfig will generate a rate-limiter in configShallowCopy. +// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), +// where httpClient was generated with rest.HTTPClientFor(c). +func NewForConfig(c *rest.Config) (*Clientset, error) { + configShallowCopy := *c + + if configShallowCopy.UserAgent == "" { + configShallowCopy.UserAgent = rest.DefaultKubernetesUserAgent() + } + + // share the transport between all clients + httpClient, err := rest.HTTPClientFor(&configShallowCopy) + if err != nil { + return nil, err + } + + return NewForConfigAndClient(&configShallowCopy, httpClient) +} + +// NewForConfigAndClient creates a new Clientset for the given config and http client. +// Note the http client provided takes precedence over the configured transport values. +// If config's RateLimiter is not set and QPS and Burst are acceptable, +// NewForConfigAndClient will generate a rate-limiter in configShallowCopy. +func NewForConfigAndClient(c *rest.Config, httpClient *http.Client) (*Clientset, error) { + configShallowCopy := *c + if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 { + if configShallowCopy.Burst <= 0 { + return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0") + } + configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst) + } + + var cs Clientset + var err error + cs.xuanwuV1, err = xuanwuv1.NewForConfigAndClient(&configShallowCopy, httpClient) + if err != nil { + return nil, err + } + + cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfigAndClient(&configShallowCopy, httpClient) + if err != nil { + return nil, err + } + return &cs, nil +} + +// NewForConfigOrDie creates a new Clientset for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *rest.Config) *Clientset { + cs, err := NewForConfig(c) + if err != nil { + panic(err) + } + return cs +} + +// New creates a new Clientset for the given RESTClient. +func New(c rest.Interface) *Clientset { + var cs Clientset + cs.xuanwuV1 = xuanwuv1.New(c) + + cs.DiscoveryClient = discovery.NewDiscoveryClient(c) + return &cs +} diff --git a/pkg/client/clientset/versioned/doc.go b/pkg/client/clientset/versioned/doc.go new file mode 100644 index 0000000..4f3b5be --- /dev/null +++ b/pkg/client/clientset/versioned/doc.go @@ -0,0 +1,17 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated clientset. +package versioned diff --git a/pkg/client/clientset/versioned/fake/clientset_generated.go b/pkg/client/clientset/versioned/fake/clientset_generated.go new file mode 100644 index 0000000..f285c10 --- /dev/null +++ b/pkg/client/clientset/versioned/fake/clientset_generated.go @@ -0,0 +1,83 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + clientset "github.com/huawei/csm/v2/pkg/client/clientset/versioned" + xuanwuv1 "github.com/huawei/csm/v2/pkg/client/clientset/versioned/typed/xuanwu/v1" + fakexuanwuv1 "github.com/huawei/csm/v2/pkg/client/clientset/versioned/typed/xuanwu/v1/fake" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/discovery" + fakediscovery "k8s.io/client-go/discovery/fake" + "k8s.io/client-go/testing" +) + +// NewSimpleClientset returns a clientset that will respond with the provided objects. +// It's backed by a very simple object tracker that processes creates, updates and deletions as-is, +// without applying any validations and/or defaults. It shouldn't be considered a replacement +// for a real clientset and is mostly useful in simple unit tests. +func NewSimpleClientset(objects ...runtime.Object) *Clientset { + o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder()) + for _, obj := range objects { + if err := o.Add(obj); err != nil { + panic(err) + } + } + + cs := &Clientset{tracker: o} + cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake} + cs.AddReactor("*", "*", testing.ObjectReaction(o)) + cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { + gvr := action.GetResource() + ns := action.GetNamespace() + watch, err := o.Watch(gvr, ns) + if err != nil { + return false, nil, err + } + return true, watch, nil + }) + + return cs +} + +// Clientset implements clientset.Interface. Meant to be embedded into a +// struct to get a default implementation. This makes faking out just the method +// you want to test easier. +type Clientset struct { + testing.Fake + discovery *fakediscovery.FakeDiscovery + tracker testing.ObjectTracker +} + +func (c *Clientset) Discovery() discovery.DiscoveryInterface { + return c.discovery +} + +func (c *Clientset) Tracker() testing.ObjectTracker { + return c.tracker +} + +var ( + _ clientset.Interface = &Clientset{} + _ testing.FakeClient = &Clientset{} +) + +// XuanwuV1 retrieves the XuanwuV1Client +func (c *Clientset) XuanwuV1() xuanwuv1.XuanwuV1Interface { + return &fakexuanwuv1.FakeXuanwuV1{Fake: &c.Fake} +} diff --git a/pkg/client/clientset/versioned/fake/doc.go b/pkg/client/clientset/versioned/fake/doc.go new file mode 100644 index 0000000..9f3a3ff --- /dev/null +++ b/pkg/client/clientset/versioned/fake/doc.go @@ -0,0 +1,17 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated fake clientset. +package fake diff --git a/pkg/client/clientset/versioned/fake/register.go b/pkg/client/clientset/versioned/fake/register.go new file mode 100644 index 0000000..09e15f4 --- /dev/null +++ b/pkg/client/clientset/versioned/fake/register.go @@ -0,0 +1,54 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + xuanwuv1 "github.com/huawei/csm/v2/client/apis/xuanwu/v1" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + serializer "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" +) + +var scheme = runtime.NewScheme() +var codecs = serializer.NewCodecFactory(scheme) + +var localSchemeBuilder = runtime.SchemeBuilder{ + xuanwuv1.AddToScheme, +} + +// AddToScheme adds all types of this clientset into the given scheme. This allows composition +// of clientsets, like in: +// +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) +// +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// +// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types +// correctly. +var AddToScheme = localSchemeBuilder.AddToScheme + +func init() { + v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"}) + utilruntime.Must(AddToScheme(scheme)) +} diff --git a/pkg/client/clientset/versioned/scheme/doc.go b/pkg/client/clientset/versioned/scheme/doc.go new file mode 100644 index 0000000..f337483 --- /dev/null +++ b/pkg/client/clientset/versioned/scheme/doc.go @@ -0,0 +1,17 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// This package contains the scheme of the automatically generated clientset. +package scheme diff --git a/pkg/client/clientset/versioned/scheme/register.go b/pkg/client/clientset/versioned/scheme/register.go new file mode 100644 index 0000000..f74d6b3 --- /dev/null +++ b/pkg/client/clientset/versioned/scheme/register.go @@ -0,0 +1,54 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package scheme + +import ( + xuanwuv1 "github.com/huawei/csm/v2/client/apis/xuanwu/v1" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + serializer "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" +) + +var Scheme = runtime.NewScheme() +var Codecs = serializer.NewCodecFactory(Scheme) +var ParameterCodec = runtime.NewParameterCodec(Scheme) +var localSchemeBuilder = runtime.SchemeBuilder{ + xuanwuv1.AddToScheme, +} + +// AddToScheme adds all types of this clientset into the given scheme. This allows composition +// of clientsets, like in: +// +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) +// +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// +// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types +// correctly. +var AddToScheme = localSchemeBuilder.AddToScheme + +func init() { + v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) + utilruntime.Must(AddToScheme(Scheme)) +} diff --git a/pkg/client/clientset/versioned/typed/xuanwu/v1/doc.go b/pkg/client/clientset/versioned/typed/xuanwu/v1/doc.go new file mode 100644 index 0000000..5c8da48 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/xuanwu/v1/doc.go @@ -0,0 +1,17 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated typed clients. +package v1 diff --git a/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/doc.go b/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/doc.go new file mode 100644 index 0000000..62e5be5 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/doc.go @@ -0,0 +1,17 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// Package fake has the automatically generated clients. +package fake diff --git a/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_resourcetopology.go b/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_resourcetopology.go new file mode 100644 index 0000000..0f324f3 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_resourcetopology.go @@ -0,0 +1,130 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + xuanwuv1 "github.com/huawei/csm/v2/client/apis/xuanwu/v1" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeResourceTopologies implements ResourceTopologyInterface +type FakeResourceTopologies struct { + Fake *FakeXuanwuV1 +} + +var resourcetopologiesResource = schema.GroupVersionResource{Group: "xuanwu.huawei.io", Version: "v1", Resource: "resourcetopologies"} + +var resourcetopologiesKind = schema.GroupVersionKind{Group: "xuanwu.huawei.io", Version: "v1", Kind: "ResourceTopology"} + +// Get takes name of the resourceTopology, and returns the corresponding resourceTopology object, and an error if there is any. +func (c *FakeResourceTopologies) Get(ctx context.Context, name string, options v1.GetOptions) (result *xuanwuv1.ResourceTopology, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootGetAction(resourcetopologiesResource, name), &xuanwuv1.ResourceTopology{}) + if obj == nil { + return nil, err + } + return obj.(*xuanwuv1.ResourceTopology), err +} + +// List takes label and field selectors, and returns the list of ResourceTopologies that match those selectors. +func (c *FakeResourceTopologies) List(ctx context.Context, opts v1.ListOptions) (result *xuanwuv1.ResourceTopologyList, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootListAction(resourcetopologiesResource, resourcetopologiesKind, opts), &xuanwuv1.ResourceTopologyList{}) + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &xuanwuv1.ResourceTopologyList{ListMeta: obj.(*xuanwuv1.ResourceTopologyList).ListMeta} + for _, item := range obj.(*xuanwuv1.ResourceTopologyList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested resourceTopologies. +func (c *FakeResourceTopologies) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewRootWatchAction(resourcetopologiesResource, opts)) +} + +// Create takes the representation of a resourceTopology and creates it. Returns the server's representation of the resourceTopology, and an error, if there is any. +func (c *FakeResourceTopologies) Create(ctx context.Context, resourceTopology *xuanwuv1.ResourceTopology, opts v1.CreateOptions) (result *xuanwuv1.ResourceTopology, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootCreateAction(resourcetopologiesResource, resourceTopology), &xuanwuv1.ResourceTopology{}) + if obj == nil { + return nil, err + } + return obj.(*xuanwuv1.ResourceTopology), err +} + +// Update takes the representation of a resourceTopology and updates it. Returns the server's representation of the resourceTopology, and an error, if there is any. +func (c *FakeResourceTopologies) Update(ctx context.Context, resourceTopology *xuanwuv1.ResourceTopology, opts v1.UpdateOptions) (result *xuanwuv1.ResourceTopology, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateAction(resourcetopologiesResource, resourceTopology), &xuanwuv1.ResourceTopology{}) + if obj == nil { + return nil, err + } + return obj.(*xuanwuv1.ResourceTopology), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeResourceTopologies) UpdateStatus(ctx context.Context, resourceTopology *xuanwuv1.ResourceTopology, opts v1.UpdateOptions) (*xuanwuv1.ResourceTopology, error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateSubresourceAction(resourcetopologiesResource, "status", resourceTopology), &xuanwuv1.ResourceTopology{}) + if obj == nil { + return nil, err + } + return obj.(*xuanwuv1.ResourceTopology), err +} + +// Delete takes name of the resourceTopology and deletes it. Returns an error if one occurs. +func (c *FakeResourceTopologies) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewRootDeleteActionWithOptions(resourcetopologiesResource, name, opts), &xuanwuv1.ResourceTopology{}) + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeResourceTopologies) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewRootDeleteCollectionAction(resourcetopologiesResource, listOpts) + + _, err := c.Fake.Invokes(action, &xuanwuv1.ResourceTopologyList{}) + return err +} + +// Patch applies the patch and returns the patched resourceTopology. +func (c *FakeResourceTopologies) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *xuanwuv1.ResourceTopology, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootPatchSubresourceAction(resourcetopologiesResource, name, pt, data, subresources...), &xuanwuv1.ResourceTopology{}) + if obj == nil { + return nil, err + } + return obj.(*xuanwuv1.ResourceTopology), err +} diff --git a/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_xuanwu_client.go b/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_xuanwu_client.go new file mode 100644 index 0000000..c8bff42 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_xuanwu_client.go @@ -0,0 +1,38 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1 "github.com/huawei/csm/v2/pkg/client/clientset/versioned/typed/xuanwu/v1" + + rest "k8s.io/client-go/rest" + testing "k8s.io/client-go/testing" +) + +type FakeXuanwuV1 struct { + *testing.Fake +} + +func (c *FakeXuanwuV1) ResourceTopologies() v1.ResourceTopologyInterface { + return &FakeResourceTopologies{c} +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *FakeXuanwuV1) RESTClient() rest.Interface { + var ret *rest.RESTClient + return ret +} diff --git a/pkg/client/clientset/versioned/typed/xuanwu/v1/generated_expansion.go b/pkg/client/clientset/versioned/typed/xuanwu/v1/generated_expansion.go new file mode 100644 index 0000000..b3820a2 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/xuanwu/v1/generated_expansion.go @@ -0,0 +1,18 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1 + +type ResourceTopologyExpansion interface{} diff --git a/pkg/client/clientset/versioned/typed/xuanwu/v1/resourcetopology.go b/pkg/client/clientset/versioned/typed/xuanwu/v1/resourcetopology.go new file mode 100644 index 0000000..ca87a53 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/xuanwu/v1/resourcetopology.go @@ -0,0 +1,181 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1 + +import ( + "context" + v1 "github.com/huawei/csm/v2/client/apis/xuanwu/v1" + scheme "github.com/huawei/csm/v2/pkg/client/clientset/versioned/scheme" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// ResourceTopologiesGetter has a method to return a ResourceTopologyInterface. +// A group's client should implement this interface. +type ResourceTopologiesGetter interface { + ResourceTopologies() ResourceTopologyInterface +} + +// ResourceTopologyInterface has methods to work with ResourceTopology resources. +type ResourceTopologyInterface interface { + Create(ctx context.Context, resourceTopology *v1.ResourceTopology, opts metav1.CreateOptions) (*v1.ResourceTopology, error) + Update(ctx context.Context, resourceTopology *v1.ResourceTopology, opts metav1.UpdateOptions) (*v1.ResourceTopology, error) + UpdateStatus(ctx context.Context, resourceTopology *v1.ResourceTopology, opts metav1.UpdateOptions) (*v1.ResourceTopology, error) + Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error + Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.ResourceTopology, error) + List(ctx context.Context, opts metav1.ListOptions) (*v1.ResourceTopologyList, error) + Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.ResourceTopology, err error) + ResourceTopologyExpansion +} + +// resourceTopologies implements ResourceTopologyInterface +type resourceTopologies struct { + client rest.Interface +} + +// newResourceTopologies returns a ResourceTopologies +func newResourceTopologies(c *XuanwuV1Client) *resourceTopologies { + return &resourceTopologies{ + client: c.RESTClient(), + } +} + +// Get takes name of the resourceTopology, and returns the corresponding resourceTopology object, and an error if there is any. +func (c *resourceTopologies) Get(ctx context.Context, name string, options metav1.GetOptions) (result *v1.ResourceTopology, err error) { + result = &v1.ResourceTopology{} + err = c.client.Get(). + Resource("resourcetopologies"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of ResourceTopologies that match those selectors. +func (c *resourceTopologies) List(ctx context.Context, opts metav1.ListOptions) (result *v1.ResourceTopologyList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1.ResourceTopologyList{} + err = c.client.Get(). + Resource("resourcetopologies"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(ctx). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested resourceTopologies. +func (c *resourceTopologies) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Resource("resourcetopologies"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch(ctx) +} + +// Create takes the representation of a resourceTopology and creates it. Returns the server's representation of the resourceTopology, and an error, if there is any. +func (c *resourceTopologies) Create(ctx context.Context, resourceTopology *v1.ResourceTopology, opts metav1.CreateOptions) (result *v1.ResourceTopology, err error) { + result = &v1.ResourceTopology{} + err = c.client.Post(). + Resource("resourcetopologies"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(resourceTopology). + Do(ctx). + Into(result) + return +} + +// Update takes the representation of a resourceTopology and updates it. Returns the server's representation of the resourceTopology, and an error, if there is any. +func (c *resourceTopologies) Update(ctx context.Context, resourceTopology *v1.ResourceTopology, opts metav1.UpdateOptions) (result *v1.ResourceTopology, err error) { + result = &v1.ResourceTopology{} + err = c.client.Put(). + Resource("resourcetopologies"). + Name(resourceTopology.Name). + VersionedParams(&opts, scheme.ParameterCodec). + Body(resourceTopology). + Do(ctx). + Into(result) + return +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *resourceTopologies) UpdateStatus(ctx context.Context, resourceTopology *v1.ResourceTopology, opts metav1.UpdateOptions) (result *v1.ResourceTopology, err error) { + result = &v1.ResourceTopology{} + err = c.client.Put(). + Resource("resourcetopologies"). + Name(resourceTopology.Name). + SubResource("status"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(resourceTopology). + Do(ctx). + Into(result) + return +} + +// Delete takes name of the resourceTopology and deletes it. Returns an error if one occurs. +func (c *resourceTopologies) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error { + return c.client.Delete(). + Resource("resourcetopologies"). + Name(name). + Body(&opts). + Do(ctx). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *resourceTopologies) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error { + var timeout time.Duration + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Resource("resourcetopologies"). + VersionedParams(&listOpts, scheme.ParameterCodec). + Timeout(timeout). + Body(&opts). + Do(ctx). + Error() +} + +// Patch applies the patch and returns the patched resourceTopology. +func (c *resourceTopologies) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.ResourceTopology, err error) { + result = &v1.ResourceTopology{} + err = c.client.Patch(pt). + Resource("resourcetopologies"). + Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} diff --git a/pkg/client/clientset/versioned/typed/xuanwu/v1/xuanwu_client.go b/pkg/client/clientset/versioned/typed/xuanwu/v1/xuanwu_client.go new file mode 100644 index 0000000..8ca1783 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/xuanwu/v1/xuanwu_client.go @@ -0,0 +1,104 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1 + +import ( + v1 "github.com/huawei/csm/v2/client/apis/xuanwu/v1" + "github.com/huawei/csm/v2/pkg/client/clientset/versioned/scheme" + "net/http" + + rest "k8s.io/client-go/rest" +) + +type XuanwuV1Interface interface { + RESTClient() rest.Interface + ResourceTopologiesGetter +} + +// XuanwuV1Client is used to interact with features provided by the xuanwu.huawei.io group. +type XuanwuV1Client struct { + restClient rest.Interface +} + +func (c *XuanwuV1Client) ResourceTopologies() ResourceTopologyInterface { + return newResourceTopologies(c) +} + +// NewForConfig creates a new XuanwuV1Client for the given config. +// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), +// where httpClient was generated with rest.HTTPClientFor(c). +func NewForConfig(c *rest.Config) (*XuanwuV1Client, error) { + config := *c + if err := setConfigDefaults(&config); err != nil { + return nil, err + } + httpClient, err := rest.HTTPClientFor(&config) + if err != nil { + return nil, err + } + return NewForConfigAndClient(&config, httpClient) +} + +// NewForConfigAndClient creates a new XuanwuV1Client for the given config and http client. +// Note the http client provided takes precedence over the configured transport values. +func NewForConfigAndClient(c *rest.Config, h *http.Client) (*XuanwuV1Client, error) { + config := *c + if err := setConfigDefaults(&config); err != nil { + return nil, err + } + client, err := rest.RESTClientForConfigAndClient(&config, h) + if err != nil { + return nil, err + } + return &XuanwuV1Client{client}, nil +} + +// NewForConfigOrDie creates a new XuanwuV1Client for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *rest.Config) *XuanwuV1Client { + client, err := NewForConfig(c) + if err != nil { + panic(err) + } + return client +} + +// New creates a new XuanwuV1Client for the given RESTClient. +func New(c rest.Interface) *XuanwuV1Client { + return &XuanwuV1Client{c} +} + +func setConfigDefaults(config *rest.Config) error { + gv := v1.SchemeGroupVersion + config.GroupVersion = &gv + config.APIPath = "/apis" + config.NegotiatedSerializer = scheme.Codecs.WithoutConversion() + + if config.UserAgent == "" { + config.UserAgent = rest.DefaultKubernetesUserAgent() + } + + return nil +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *XuanwuV1Client) RESTClient() rest.Interface { + if c == nil { + return nil + } + return c.restClient +} diff --git a/pkg/client/informers/externalversions/factory.go b/pkg/client/informers/externalversions/factory.go new file mode 100644 index 0000000..781e7aa --- /dev/null +++ b/pkg/client/informers/externalversions/factory.go @@ -0,0 +1,248 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package externalversions + +import ( + versioned "github.com/huawei/csm/v2/pkg/client/clientset/versioned" + internalinterfaces "github.com/huawei/csm/v2/pkg/client/informers/externalversions/internalinterfaces" + xuanwu "github.com/huawei/csm/v2/pkg/client/informers/externalversions/xuanwu" + reflect "reflect" + sync "sync" + time "time" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + cache "k8s.io/client-go/tools/cache" +) + +// SharedInformerOption defines the functional option type for SharedInformerFactory. +type SharedInformerOption func(*sharedInformerFactory) *sharedInformerFactory + +type sharedInformerFactory struct { + client versioned.Interface + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc + lock sync.Mutex + defaultResync time.Duration + customResync map[reflect.Type]time.Duration + + informers map[reflect.Type]cache.SharedIndexInformer + // startedInformers is used for tracking which informers have been started. + // This allows Start() to be called multiple times safely. + startedInformers map[reflect.Type]bool + // wg tracks how many goroutines were started. + wg sync.WaitGroup + // shuttingDown is true when Shutdown has been called. It may still be running + // because it needs to wait for goroutines. + shuttingDown bool +} + +// WithCustomResyncConfig sets a custom resync period for the specified informer types. +func WithCustomResyncConfig(resyncConfig map[v1.Object]time.Duration) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + for k, v := range resyncConfig { + factory.customResync[reflect.TypeOf(k)] = v + } + return factory + } +} + +// WithTweakListOptions sets a custom filter on all listers of the configured SharedInformerFactory. +func WithTweakListOptions(tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + factory.tweakListOptions = tweakListOptions + return factory + } +} + +// WithNamespace limits the SharedInformerFactory to the specified namespace. +func WithNamespace(namespace string) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + factory.namespace = namespace + return factory + } +} + +// NewSharedInformerFactory constructs a new instance of sharedInformerFactory for all namespaces. +func NewSharedInformerFactory(client versioned.Interface, defaultResync time.Duration) SharedInformerFactory { + return NewSharedInformerFactoryWithOptions(client, defaultResync) +} + +// NewFilteredSharedInformerFactory constructs a new instance of sharedInformerFactory. +// Listers obtained via this SharedInformerFactory will be subject to the same filters +// as specified here. +// Deprecated: Please use NewSharedInformerFactoryWithOptions instead +func NewFilteredSharedInformerFactory(client versioned.Interface, defaultResync time.Duration, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerFactory { + return NewSharedInformerFactoryWithOptions(client, defaultResync, WithNamespace(namespace), WithTweakListOptions(tweakListOptions)) +} + +// NewSharedInformerFactoryWithOptions constructs a new instance of a SharedInformerFactory with additional options. +func NewSharedInformerFactoryWithOptions(client versioned.Interface, defaultResync time.Duration, options ...SharedInformerOption) SharedInformerFactory { + factory := &sharedInformerFactory{ + client: client, + namespace: v1.NamespaceAll, + defaultResync: defaultResync, + informers: make(map[reflect.Type]cache.SharedIndexInformer), + startedInformers: make(map[reflect.Type]bool), + customResync: make(map[reflect.Type]time.Duration), + } + + // Apply all options + for _, opt := range options { + factory = opt(factory) + } + + return factory +} + +func (f *sharedInformerFactory) Start(stopCh <-chan struct{}) { + f.lock.Lock() + defer f.lock.Unlock() + + if f.shuttingDown { + return + } + + for informerType, informer := range f.informers { + if !f.startedInformers[informerType] { + f.wg.Add(1) + // We need a new variable in each loop iteration, + // otherwise the goroutine would use the loop variable + // and that keeps changing. + informer := informer + go func() { + defer f.wg.Done() + informer.Run(stopCh) + }() + f.startedInformers[informerType] = true + } + } +} + +func (f *sharedInformerFactory) Shutdown() { + f.lock.Lock() + f.shuttingDown = true + f.lock.Unlock() + + // Will return immediately if there is nothing to wait for. + f.wg.Wait() +} + +func (f *sharedInformerFactory) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool { + informers := func() map[reflect.Type]cache.SharedIndexInformer { + f.lock.Lock() + defer f.lock.Unlock() + + informers := map[reflect.Type]cache.SharedIndexInformer{} + for informerType, informer := range f.informers { + if f.startedInformers[informerType] { + informers[informerType] = informer + } + } + return informers + }() + + res := map[reflect.Type]bool{} + for informType, informer := range informers { + res[informType] = cache.WaitForCacheSync(stopCh, informer.HasSynced) + } + return res +} + +// InternalInformerFor returns the SharedIndexInformer for obj using an internal +// client. +func (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer { + f.lock.Lock() + defer f.lock.Unlock() + + informerType := reflect.TypeOf(obj) + informer, exists := f.informers[informerType] + if exists { + return informer + } + + resyncPeriod, exists := f.customResync[informerType] + if !exists { + resyncPeriod = f.defaultResync + } + + informer = newFunc(f.client, resyncPeriod) + f.informers[informerType] = informer + + return informer +} + +// SharedInformerFactory provides shared informers for resources in all known +// API group versions. +// +// It is typically used like this: +// +// ctx, cancel := context.Background() +// defer cancel() +// factory := NewSharedInformerFactory(client, resyncPeriod) +// defer factory.WaitForStop() // Returns immediately if nothing was started. +// genericInformer := factory.ForResource(resource) +// typedInformer := factory.SomeAPIGroup().V1().SomeType() +// factory.Start(ctx.Done()) // Start processing these informers. +// synced := factory.WaitForCacheSync(ctx.Done()) +// for v, ok := range synced { +// if !ok { +// fmt.Fprintf(os.Stderr, "caches failed to sync: %v", v) +// return +// } +// } +// +// // Creating informers can also be created after Start, but then +// // Start must be called again: +// anotherGenericInformer := factory.ForResource(resource) +// factory.Start(ctx.Done()) +type SharedInformerFactory interface { + internalinterfaces.SharedInformerFactory + + // Start initializes all requested informers. They are handled in goroutines + // which run until the stop channel gets closed. + Start(stopCh <-chan struct{}) + + // Shutdown marks a factory as shutting down. At that point no new + // informers can be started anymore and Start will return without + // doing anything. + // + // In addition, Shutdown blocks until all goroutines have terminated. For that + // to happen, the close channel(s) that they were started with must be closed, + // either before Shutdown gets called or while it is waiting. + // + // Shutdown may be called multiple times, even concurrently. All such calls will + // block until all goroutines have terminated. + Shutdown() + + // WaitForCacheSync blocks until all started informers' caches were synced + // or the stop channel gets closed. + WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool + + // ForResource gives generic access to a shared informer of the matching type. + ForResource(resource schema.GroupVersionResource) (GenericInformer, error) + + // InternalInformerFor returns the SharedIndexInformer for obj using an internal + // client. + InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer + + Xuanwu() xuanwu.Interface +} + +func (f *sharedInformerFactory) Xuanwu() xuanwu.Interface { + return xuanwu.New(f, f.namespace, f.tweakListOptions) +} diff --git a/pkg/client/informers/externalversions/generic.go b/pkg/client/informers/externalversions/generic.go new file mode 100644 index 0000000..43a54c8 --- /dev/null +++ b/pkg/client/informers/externalversions/generic.go @@ -0,0 +1,59 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package externalversions + +import ( + "fmt" + v1 "github.com/huawei/csm/v2/client/apis/xuanwu/v1" + + schema "k8s.io/apimachinery/pkg/runtime/schema" + cache "k8s.io/client-go/tools/cache" +) + +// GenericInformer is type of SharedIndexInformer which will locate and delegate to other +// sharedInformers based on type +type GenericInformer interface { + Informer() cache.SharedIndexInformer + Lister() cache.GenericLister +} + +type genericInformer struct { + informer cache.SharedIndexInformer + resource schema.GroupResource +} + +// Informer returns the SharedIndexInformer. +func (f *genericInformer) Informer() cache.SharedIndexInformer { + return f.informer +} + +// Lister returns the GenericLister. +func (f *genericInformer) Lister() cache.GenericLister { + return cache.NewGenericLister(f.Informer().GetIndexer(), f.resource) +} + +// ForResource gives generic access to a shared informer of the matching type +// TODO extend this to unknown resources with a client pool +func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) { + switch resource { + // Group=xuanwu.huawei.io, Version=v1 + case v1.SchemeGroupVersion.WithResource("resourcetopologies"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Xuanwu().V1().ResourceTopologies().Informer()}, nil + + } + + return nil, fmt.Errorf("no informer found for %v", resource) +} diff --git a/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go b/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go new file mode 100644 index 0000000..30296af --- /dev/null +++ b/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go @@ -0,0 +1,37 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package internalinterfaces + +import ( + versioned "github.com/huawei/csm/v2/pkg/client/clientset/versioned" + time "time" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + cache "k8s.io/client-go/tools/cache" +) + +// NewInformerFunc takes versioned.Interface and time.Duration to return a SharedIndexInformer. +type NewInformerFunc func(versioned.Interface, time.Duration) cache.SharedIndexInformer + +// SharedInformerFactory a small interface to allow for adding an informer without an import cycle +type SharedInformerFactory interface { + Start(stopCh <-chan struct{}) + InformerFor(obj runtime.Object, newFunc NewInformerFunc) cache.SharedIndexInformer +} + +// TweakListOptionsFunc is a function that transforms a v1.ListOptions. +type TweakListOptionsFunc func(*v1.ListOptions) diff --git a/pkg/client/informers/externalversions/xuanwu/interface.go b/pkg/client/informers/externalversions/xuanwu/interface.go new file mode 100644 index 0000000..dbe6959 --- /dev/null +++ b/pkg/client/informers/externalversions/xuanwu/interface.go @@ -0,0 +1,43 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package xuanwu + +import ( + internalinterfaces "github.com/huawei/csm/v2/pkg/client/informers/externalversions/internalinterfaces" + v1 "github.com/huawei/csm/v2/pkg/client/informers/externalversions/xuanwu/v1" +) + +// Interface provides access to each of this group's versions. +type Interface interface { + // V1 provides access to shared informers for resources in V1. + V1() v1.Interface +} + +type group struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// V1 returns a new v1.Interface. +func (g *group) V1() v1.Interface { + return v1.New(g.factory, g.namespace, g.tweakListOptions) +} diff --git a/pkg/client/informers/externalversions/xuanwu/v1/interface.go b/pkg/client/informers/externalversions/xuanwu/v1/interface.go new file mode 100644 index 0000000..0bd2d8f --- /dev/null +++ b/pkg/client/informers/externalversions/xuanwu/v1/interface.go @@ -0,0 +1,42 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + internalinterfaces "github.com/huawei/csm/v2/pkg/client/informers/externalversions/internalinterfaces" +) + +// Interface provides access to all the informers in this group version. +type Interface interface { + // ResourceTopologies returns a ResourceTopologyInformer. + ResourceTopologies() ResourceTopologyInformer +} + +type version struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// ResourceTopologies returns a ResourceTopologyInformer. +func (v *version) ResourceTopologies() ResourceTopologyInformer { + return &resourceTopologyInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} +} diff --git a/pkg/client/informers/externalversions/xuanwu/v1/resourcetopology.go b/pkg/client/informers/externalversions/xuanwu/v1/resourcetopology.go new file mode 100644 index 0000000..25c894a --- /dev/null +++ b/pkg/client/informers/externalversions/xuanwu/v1/resourcetopology.go @@ -0,0 +1,86 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + "context" + xuanwuv1 "github.com/huawei/csm/v2/client/apis/xuanwu/v1" + versioned "github.com/huawei/csm/v2/pkg/client/clientset/versioned" + internalinterfaces "github.com/huawei/csm/v2/pkg/client/informers/externalversions/internalinterfaces" + v1 "github.com/huawei/csm/v2/pkg/client/listers/xuanwu/v1" + time "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// ResourceTopologyInformer provides access to a shared informer and lister for +// ResourceTopologies. +type ResourceTopologyInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1.ResourceTopologyLister +} + +type resourceTopologyInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// NewResourceTopologyInformer constructs a new informer for ResourceTopology type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewResourceTopologyInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredResourceTopologyInformer(client, resyncPeriod, indexers, nil) +} + +// NewFilteredResourceTopologyInformer constructs a new informer for ResourceTopology type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredResourceTopologyInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.XuanwuV1().ResourceTopologies().List(context.TODO(), options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.XuanwuV1().ResourceTopologies().Watch(context.TODO(), options) + }, + }, + &xuanwuv1.ResourceTopology{}, + resyncPeriod, + indexers, + ) +} + +func (f *resourceTopologyInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredResourceTopologyInformer(client, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *resourceTopologyInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&xuanwuv1.ResourceTopology{}, f.defaultInformer) +} + +func (f *resourceTopologyInformer) Lister() v1.ResourceTopologyLister { + return v1.NewResourceTopologyLister(f.Informer().GetIndexer()) +} diff --git a/pkg/client/listers/xuanwu/v1/expansion_generated.go b/pkg/client/listers/xuanwu/v1/expansion_generated.go new file mode 100644 index 0000000..50a3a87 --- /dev/null +++ b/pkg/client/listers/xuanwu/v1/expansion_generated.go @@ -0,0 +1,20 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +// Code generated by lister-gen. DO NOT EDIT. + +package v1 + +// ResourceTopologyListerExpansion allows custom methods to be added to +// ResourceTopologyLister. +type ResourceTopologyListerExpansion interface{} diff --git a/pkg/client/listers/xuanwu/v1/resourcetopology.go b/pkg/client/listers/xuanwu/v1/resourcetopology.go new file mode 100644 index 0000000..1e1236e --- /dev/null +++ b/pkg/client/listers/xuanwu/v1/resourcetopology.go @@ -0,0 +1,66 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +// Code generated by lister-gen. DO NOT EDIT. + +package v1 + +import ( + v1 "github.com/huawei/csm/v2/client/apis/xuanwu/v1" + + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" +) + +// ResourceTopologyLister helps list ResourceTopologies. +// All objects returned here must be treated as read-only. +type ResourceTopologyLister interface { + // List lists all ResourceTopologies in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1.ResourceTopology, err error) + // Get retrieves the ResourceTopology from the index for a given name. + // Objects returned here must be treated as read-only. + Get(name string) (*v1.ResourceTopology, error) + ResourceTopologyListerExpansion +} + +// resourceTopologyLister implements the ResourceTopologyLister interface. +type resourceTopologyLister struct { + indexer cache.Indexer +} + +// NewResourceTopologyLister returns a new ResourceTopologyLister. +func NewResourceTopologyLister(indexer cache.Indexer) ResourceTopologyLister { + return &resourceTopologyLister{indexer: indexer} +} + +// List lists all ResourceTopologies in the indexer. +func (s *resourceTopologyLister) List(selector labels.Selector) (ret []*v1.ResourceTopology, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v1.ResourceTopology)) + }) + return ret, err +} + +// Get retrieves the ResourceTopology from the index for a given name. +func (s *resourceTopologyLister) Get(name string) (*v1.ResourceTopology, error) { + obj, exists, err := s.indexer.GetByKey(name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(v1.Resource("resourcetopology"), name) + } + return obj.(*v1.ResourceTopology), nil +} diff --git a/provider/backend/backend.go b/provider/backend/backend.go new file mode 100644 index 0000000..d4460a1 --- /dev/null +++ b/provider/backend/backend.go @@ -0,0 +1,40 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package backend is a package that manager storage backend +package backend + +import ( + "context" + + "github.com/huawei/csm/v2/utils/log" +) + +// GetClientByBackendName get Client by backend name +func GetClientByBackendName(ctx context.Context, backendName string) (ClientInfo, error) { + log.AddContext(ctx).Infof("Start to get client, name: %s", backendName) + config, err := NewStorageBackendConfigBuilder(ctx, backendName). + WithSbcInfo(). + WithSecretInfo(). + WithConfigMapInfo().Build() + if err != nil { + log.AddContext(ctx).Errorf("build storage config failed, name: %s, error: %v", backendName, err) + return ClientInfo{}, err + } + + return NewClientInfoBuilder(ctx). + WithVolumeType(config.StorageType).WithClient(config).Build() +} diff --git a/provider/backend/client_info_builder.go b/provider/backend/client_info_builder.go new file mode 100644 index 0000000..dc88b4b --- /dev/null +++ b/provider/backend/client_info_builder.go @@ -0,0 +1,86 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package backend is a package that manager storage backend +package backend + +import ( + "context" + "errors" + + "github.com/huawei/csm/v2/provider/constants" + "github.com/huawei/csm/v2/storage/client/centralizedstorage" + "github.com/huawei/csm/v2/storage/constant" + "github.com/huawei/csm/v2/utils/log" +) + +// ClientInfoBuilder client builder +type ClientInfoBuilder struct { + ctx context.Context + err error + clientInfo *ClientInfo +} + +// NewClientInfoBuilder init an instance of ClientInfoBuilder +func NewClientInfoBuilder(ctx context.Context) *ClientInfoBuilder { + return &ClientInfoBuilder{ctx: ctx, clientInfo: &ClientInfo{}} +} + +// Build init an instance of ClientInfo +func (b *ClientInfoBuilder) Build() (ClientInfo, error) { + return *b.clientInfo, b.err +} + +// WithVolumeType build with volume type +func (b *ClientInfoBuilder) WithVolumeType(storageType string) *ClientInfoBuilder { + if b.err != nil { + return b + } + + volumeType, ok := volumeTypes[storageType] + if !ok { + b.err = errors.New("illegalArgumentError unsupported storage type") + } + b.clientInfo.VolumeType = volumeType + return b +} + +// WithClient build with client +func (b *ClientInfoBuilder) WithClient(config *constant.StorageBackendConfig) *ClientInfoBuilder { + if b.err != nil { + return b + } + + client, err := centralizedstorage.NewCentralizedClient(b.ctx, config) + if err != nil { + log.AddContext(b.ctx).Errorf("init centralized client failed, backendName: %s, error: %v", + config.StorageBackendName, err) + b.err = err + return b + } + + if err := client.Login(b.ctx); err != nil { + log.AddContext(b.ctx).Errorf("login storage failed, backendName: %s, error: %v", + config.StorageBackendName, err) + b.err = err + return b + } + + b.clientInfo.StorageName = config.StorageBackendName + b.clientInfo.StorageType = constants.OceanStorage + b.clientInfo.Client = client + return b +} diff --git a/provider/backend/client_info_builder_test.go b/provider/backend/client_info_builder_test.go new file mode 100644 index 0000000..6cf909d --- /dev/null +++ b/provider/backend/client_info_builder_test.go @@ -0,0 +1,101 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package backend is a package that manager storage backend +package backend + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/huawei/csm/v2/provider/constants" + "github.com/huawei/csm/v2/storage/constant" +) + +func TestClientInfoBuilder_WithVolumeType_Success(t *testing.T) { + // arrange + builder := &ClientInfoBuilder{ + ctx: context.Background(), + clientInfo: &ClientInfo{}, + } + + // act + getRes := builder.WithVolumeType(constants.StorageNas) + + // assert + if getRes.clientInfo.VolumeType != constants.NasVolume { + t.Errorf("TestClientInfoBuilder_WithVolumeType_Success failed, want = %s, got = %s", + constants.NasVolume, getRes.clientInfo.VolumeType) + } +} + +func TestClientInfoBuilder_WithVolumeType_UnsupportedErr(t *testing.T) { + // arrange + wantErr := errors.New("illegalArgumentError unsupported storage type") + builder := &ClientInfoBuilder{ + ctx: context.Background(), + clientInfo: &ClientInfo{}, + } + + // act + getRes := builder.WithVolumeType("noneType") + + // assert + if !reflect.DeepEqual(wantErr, getRes.err) { + t.Errorf("TestClientInfoBuilder_WithVolumeType_UnsupportedErr failed, wantErr = %v, gotErr = %v", + wantErr, getRes.err) + } +} + +func TestClientInfoBuilder_WithVolumeType_ErrExisted(t *testing.T) { + // arrange + wantErr := errors.New("existed err") + builder := &ClientInfoBuilder{ + ctx: context.Background(), + clientInfo: &ClientInfo{}, + err: wantErr, + } + + // act + getRes := builder.WithVolumeType(constants.StorageNas) + + // assert + if !reflect.DeepEqual(wantErr, getRes.err) { + t.Errorf("TestClientInfoBuilder_WithVolumeType_ErrExisted failed, wantErr = %v, gotErr = %v", + wantErr, getRes.err) + } +} + +func TestClientInfoBuilder_WithClient_ErrExisted(t *testing.T) { + // arrange + wantErr := errors.New("existed err") + builder := &ClientInfoBuilder{ + ctx: context.Background(), + clientInfo: &ClientInfo{}, + err: wantErr, + } + + // act + getRes := builder.WithClient(&constant.StorageBackendConfig{}) + + // assert + if !reflect.DeepEqual(wantErr, getRes.err) { + t.Errorf("TestClientInfoBuilder_WithClient_ErrExisted failed, wantErr = %v, gotErr = %v", + wantErr, getRes.err) + } +} diff --git a/provider/backend/storage_config_builder.go b/provider/backend/storage_config_builder.go new file mode 100644 index 0000000..db8e855 --- /dev/null +++ b/provider/backend/storage_config_builder.go @@ -0,0 +1,221 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package backend is a package that manager storage backend +package backend + +import ( + "context" + "fmt" + + xuanwuV1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + v1 "k8s.io/api/core/v1" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/cache" + + cmiConfig "github.com/huawei/csm/v2/config/cmi" + "github.com/huawei/csm/v2/provider/constants" + "github.com/huawei/csm/v2/provider/grpc/helper" + "github.com/huawei/csm/v2/provider/utils" + "github.com/huawei/csm/v2/storage/constant" + "github.com/huawei/csm/v2/utils/log" +) + +// volumeTypes volume type mapping +// Map key is the storage field of the backend. +// Map key is the volume type. +var volumeTypes = map[string]string{ + constants.StorageNas: constants.NasVolume, + constants.StorageSan: constants.LunVolume, +} + +// StorageBackendConfigBuilder storage backend config builder +type StorageBackendConfigBuilder struct { + ctx context.Context + err error + sbc *xuanwuV1.StorageBackendClaim + config *constant.StorageBackendConfig + backendName string +} + +// NewStorageBackendConfigBuilder init an instance of StorageBackendConfigBuilder +func NewStorageBackendConfigBuilder(ctx context.Context, backendName string) *StorageBackendConfigBuilder { + return &StorageBackendConfigBuilder{ctx: ctx, backendName: backendName, config: &constant.StorageBackendConfig{}} +} + +// Build init an instance of StorageBackendConfig +func (b *StorageBackendConfigBuilder) Build() (*constant.StorageBackendConfig, error) { + return b.config, b.err +} + +// WithSbcInfo build with sbc info +func (b *StorageBackendConfigBuilder) WithSbcInfo() *StorageBackendConfigBuilder { + if b.err != nil { + return b + } + + sbc, err := helper.GetClientSet().SbcClient.XuanwuV1().StorageBackendClaims(cmiConfig.GetNamespace()). + Get(b.ctx, b.backendName, metaV1.GetOptions{}) + if err != nil { + log.AddContext(b.ctx).Errorf("Get StorageBackendClaims failed, error: %v", err) + b.err = err + return b + } + + b.sbc = sbc + b.config.StorageBackendNamespace = sbc.Namespace + b.config.StorageBackendName = sbc.Name + b.config.ClientMaxThreads = cmiConfig.GetClientMaxThreads() + return b +} + +// WithSecretInfo build with secret info +func (b *StorageBackendConfigBuilder) WithSecretInfo() *StorageBackendConfigBuilder { + if b.err != nil { + return b + } + + secret, err := getSecretInfo(b.ctx, b.sbc.Status.SecretMeta) + if err != nil { + log.AddContext(b.ctx).Errorf("Get Secret failed, error: %v", err) + b.err = err + return b + } + + if err := parseSecretInfo(secret, b.config); err != nil { + log.AddContext(b.ctx).Errorf("parse Secret failed, error: %v", err) + b.err = err + return b + } + return b +} + +// WithConfigMapInfo build with config map info +func (b *StorageBackendConfigBuilder) WithConfigMapInfo() *StorageBackendConfigBuilder { + if b.err != nil { + return b + } + + configMap, err := getConfigmapInfo(b.ctx, b.sbc.Status.ConfigmapMeta) + if err != nil { + log.AddContext(b.ctx).Errorf("get ConfigMap failed, error: %v", err) + b.err = err + return b + } + + if err := parseConfigmapInfo(b.ctx, configMap, b.config); err != nil { + log.AddContext(b.ctx).Errorf("parse ConfigMap failed, error: %v", err) + b.err = err + return b + } + + return b +} + +func getSecretInfo(ctx context.Context, meta string) (*v1.Secret, error) { + namespace, name, err := cache.SplitMetaNamespaceKey(meta) + if err != nil { + return nil, fmt.Errorf("split secret meta %s namespace failed, error: %v", meta, err) + } + + secret, err := helper.GetClientSet().KubeClient.CoreV1().Secrets(namespace).Get(ctx, name, metaV1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("get secret with name %s and namespace %s failed, error: %v", + name, namespace, err) + } + return secret, nil +} + +func getConfigmapInfo(ctx context.Context, configmapMeta string) (*v1.ConfigMap, error) { + namespace, name, err := cache.SplitMetaNamespaceKey(configmapMeta) + if err != nil { + return nil, fmt.Errorf("split configmap meta %s namespace failed, error: %v", configmapMeta, err) + } + + configmap, err := helper.GetClientSet().KubeClient.CoreV1().ConfigMaps(namespace).Get(ctx, name, metaV1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("get configmap for [%s] failed, error: %v", configmapMeta, err) + } + return configmap, nil +} + +func parseConfigmapInfo(ctx context.Context, configmap *v1.ConfigMap, config *constant.StorageBackendConfig) error { + configDataMap, err := utils.ConvertConfigmapToMap(configmap) + if err != nil { + return fmt.Errorf("convert configmap data to map failed. err is [%v]", err) + } + + err = parseBackendType(configDataMap, config) + if err != nil { + return err + } + + return parseBackendUrls(configDataMap, config) +} + +func parseSecretInfo(secret *v1.Secret, storageConfig *constant.StorageBackendConfig) error { + if secret.Data == nil { + return fmt.Errorf("the Data not exist in secret %s", secret.Name) + } + + if err := parseBackendUser(secret.Data, storageConfig); err != nil { + return err + } + + storageConfig.SecretNamespace = secret.Namespace + storageConfig.SecretName = secret.Name + + return nil +} + +func parseBackendUser(config map[string][]byte, + storageConfig *constant.StorageBackendConfig) error { + user, exist := config["user"] + if !exist { + return fmt.Errorf("the [user] filed not exist in secret") + } + storageConfig.User = string(user) + return nil +} + +func parseBackendUrls(config map[string]interface{}, storageConfig *constant.StorageBackendConfig) error { + configUrls, ok := config["urls"].([]interface{}) + if !ok { + return fmt.Errorf("the urls filed of config %v convert to []interface{} failed, please check", config) + } + + urls := make([]string, len(configUrls)) + for i, arg := range configUrls { + urls[i], ok = arg.(string) + if !ok { + return fmt.Errorf("convert interface{} [%v] to string failed, "+ + "configUrls is %v, please check ", arg, configUrls) + } + } + + storageConfig.Urls = urls + return nil +} + +func parseBackendType(config map[string]interface{}, storageConfig *constant.StorageBackendConfig) error { + storage, exist := config["storage"] + if !exist { + return fmt.Errorf("the storage filed not exist in configmap Data %v", config) + } + + storageConfig.StorageType = fmt.Sprintf("%s", storage) + return nil +} diff --git a/provider/backend/storage_config_builder_test.go b/provider/backend/storage_config_builder_test.go new file mode 100644 index 0000000..4f891e8 --- /dev/null +++ b/provider/backend/storage_config_builder_test.go @@ -0,0 +1,43 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package backend is a package that manager storage backend +package backend + +import ( + "context" + "errors" + "reflect" + "testing" +) + +func TestStorageBackendConfigBuilder_WithSbcInfo_ErrExisted(t *testing.T) { + // arrange + wantErr := errors.New("existed err") + builder := &StorageBackendConfigBuilder{ + ctx: context.Background(), + err: wantErr, + } + + // act + getRes := builder.WithSbcInfo() + + // assert + if !reflect.DeepEqual(wantErr, getRes.err) { + t.Errorf("TestStorageBackendConfigBuilder_WithSbcInfo_ErrExisted failed, wantErr = %v, gotErr = %v", + wantErr, getRes.err) + } +} diff --git a/provider/backend/types.go b/provider/backend/types.go new file mode 100644 index 0000000..c878373 --- /dev/null +++ b/provider/backend/types.go @@ -0,0 +1,30 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package backend is a package that manager storage backend +package backend + +// ClientInfo storage client info +type ClientInfo struct { + // storage name + StorageName string + // storage type, e.g. oceanStorage + StorageType string + // volume type, e.g. nas or lun + VolumeType string + // storage Client + Client interface{} +} diff --git a/provider/collect/backend_updator.go b/provider/collect/backend_updator.go new file mode 100644 index 0000000..5785fef --- /dev/null +++ b/provider/collect/backend_updator.go @@ -0,0 +1,123 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package collect is a package that provides object and performance collect +package collect + +import ( + "context" + "fmt" + "reflect" + + csiV1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + csiInformers "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/informers/externalversions" + "k8s.io/client-go/tools/cache" + + "github.com/huawei/csm/v2/provider/backend" + "github.com/huawei/csm/v2/provider/grpc/helper" + "github.com/huawei/csm/v2/storage/client/centralizedstorage" + "github.com/huawei/csm/v2/utils/log" +) + +// RunBackendInformer run backend informer +func RunBackendInformer(stopCh chan struct{}) { + factory := csiInformers.NewSharedInformerFactory(helper.GetClientSet().SbcClient, 0) + factory.Xuanwu().V1().StorageBackendClaims().Informer().AddEventHandler( + cache.ResourceEventHandlerFuncs{ + UpdateFunc: func(oldObj, newObj interface{}) { updateBackendCache(oldObj, newObj) }, + DeleteFunc: func(obj interface{}) { deleteBackendCache(obj) }, + }, + ) + + factory.Start(stopCh) +} + +func updateBackendCache(oldObj, newObj interface{}) { + if unknown, ok := newObj.(cache.DeletedFinalStateUnknown); ok && unknown.Obj != nil { + newObj = unknown.Obj + } + + if unknown, ok := oldObj.(cache.DeletedFinalStateUnknown); ok && unknown.Obj != nil { + oldObj = unknown.Obj + } + + // Check whether obj is a storageBackendClaim CR. + oldStorageBackendClaim, ok := oldObj.(*csiV1.StorageBackendClaim) + if !ok { + log.Errorf("failed to convert old obj to storageBackendClaim, oldObj is [%v]", oldObj) + return + } + + newStorageBackendClaim, ok := newObj.(*csiV1.StorageBackendClaim) + if !ok { + log.Errorf("failed to convert new obj to storageBackendClaim, newObj is [%v]", newObj) + return + } + + if reflect.DeepEqual(newStorageBackendClaim.Spec, oldStorageBackendClaim.Spec) { + log.Debugf("the spec struct of storageBackendClaim [%s] are not changed, "+ + "do not update backend cache", oldStorageBackendClaim.Name) + return + } + + err := releaseCache(newStorageBackendClaim.Name) + if err != nil { + log.Errorln(err) + return + } + + _, err = GetClient(context.Background(), newStorageBackendClaim.Name, backend.GetClientByBackendName) + if err != nil { + log.Errorf("get Client failed, err is [%v]", err) + return + } +} + +func deleteBackendCache(obj interface{}) { + if unknown, ok := obj.(cache.DeletedFinalStateUnknown); ok && unknown.Obj != nil { + obj = unknown.Obj + } + + storageBackendClaim, ok := obj.(*csiV1.StorageBackendClaim) + if !ok { + log.Errorf("failed to convert obj to storageBackendClaim, obj is [%v]", obj) + return + } + + err := releaseCache(storageBackendClaim.Name) + if err != nil { + log.Errorln(err) + } +} + +func releaseCache(backendName string) error { + log.Infof("start release backend [%s]", backendName) + client, ok := clientCache[backendName] + if !ok { + log.Infof("backend [%s] client does not exist", backendName) + return nil + } + + centralizedClient, ok := client.Client.(*centralizedstorage.CentralizedClient) + if !ok { + return fmt.Errorf("backend [%s] client convert to centralizedClient failed", backendName) + } + + centralizedClient.Logout(context.Background()) + RemoveClient(backendName) + + return nil +} diff --git a/provider/collect/backend_updator_test.go b/provider/collect/backend_updator_test.go new file mode 100644 index 0000000..539b015 --- /dev/null +++ b/provider/collect/backend_updator_test.go @@ -0,0 +1,153 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package collect is a package that provides object and performance collect +package collect + +import ( + "context" + "reflect" + "testing" + + csiV1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/agiledragon/gomonkey/v2" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/huawei/csm/v2/provider/backend" + "github.com/huawei/csm/v2/storage/client/centralizedstorage" +) + +func TestReleaseCache(t *testing.T) { + //arrange + backendName := "backend" + var wantErr error + client := ¢ralizedstorage.CentralizedClient{} + clientInfo := backend.ClientInfo{Client: client} + RegisterClient(backendName, clientInfo) + + //mock + patches := gomonkey.NewPatches() + patches.ApplyMethod(client, "Logout", + func(_ *centralizedstorage.CentralizedClient, ctx context.Context) {}) + + //act + err := releaseCache(backendName) + + //assert + if !reflect.DeepEqual(err, wantErr) { + t.Errorf("releaseCache() error = %v, wantErr %v", err, wantErr) + } + + //clean + t.Cleanup(func() { + patches.Reset() + }) + +} + +func TestUpdateBackendCache_SameSpec(t *testing.T) { + //arrange + oldObj := &csiV1.StorageBackendClaim{} + newObj := &csiV1.StorageBackendClaim{} + backendName := "backend" + client := ¢ralizedstorage.CentralizedClient{} + clientInfo := backend.ClientInfo{StorageName: "storage", Client: client} + RegisterClient(backendName, clientInfo) + + //act + updateBackendCache(oldObj, newObj) + + //assert + if info, ok := clientCache[backendName]; !ok || !reflect.DeepEqual(info, clientInfo) { + t.Errorf("TestUpdateBackendCache_SameSpec() failed") + } + + //clean + t.Cleanup(func() { + RemoveClient(backendName) + }) +} + +func TestUpdateBackendCache_DifferentSpec(t *testing.T) { + //arrange + backendName := "backend" + oldObj := &csiV1.StorageBackendClaim{ + ObjectMeta: metaV1.ObjectMeta{Name: backendName}, + Spec: csiV1.StorageBackendClaimSpec{SecretMeta: "meta1"}, + } + newObj := &csiV1.StorageBackendClaim{ + ObjectMeta: metaV1.ObjectMeta{Name: backendName}, + Spec: csiV1.StorageBackendClaimSpec{SecretMeta: "meta2"}, + } + client := ¢ralizedstorage.CentralizedClient{} + clientInfo := backend.ClientInfo{StorageName: "storage", Client: client} + RegisterClient(backendName, clientInfo) + + //mock + patches := gomonkey.NewPatches() + patches.ApplyMethod(client, "Logout", + func(_ *centralizedstorage.CentralizedClient, ctx context.Context) {}) + patches.ApplyFunc(GetClient, func(ctx context.Context, backendName string, + discoverFunc func(context.Context, string) (backend.ClientInfo, error)) (backend.ClientInfo, error) { + clientInfo := backend.ClientInfo{StorageName: "storage", StorageType: "type1", Client: client} + RegisterClient(backendName, clientInfo) + return backend.ClientInfo{}, nil + }) + + //act + updateBackendCache(oldObj, newObj) + + //assert + if info, ok := clientCache[backendName]; !ok || reflect.DeepEqual(info, clientInfo) { + t.Errorf("TestUpdateBackendCache_DifferentSpec() failed") + } + + //clean + t.Cleanup(func() { + RemoveClient(backendName) + patches.Reset() + }) +} + +func TestDeleteBackendCache(t *testing.T) { + //arrange + backendName := "backend" + obj := &csiV1.StorageBackendClaim{ + ObjectMeta: metaV1.ObjectMeta{Name: backendName}, + Spec: csiV1.StorageBackendClaimSpec{SecretMeta: "meta1"}, + } + client := ¢ralizedstorage.CentralizedClient{} + clientInfo := backend.ClientInfo{StorageName: "storage", Client: client} + RegisterClient(backendName, clientInfo) + + //mock + patches := gomonkey.NewPatches() + patches.ApplyMethod(client, "Logout", + func(_ *centralizedstorage.CentralizedClient, ctx context.Context) {}) + + //act + deleteBackendCache(obj) + + //assert + if _, ok := clientCache[backendName]; ok { + t.Errorf("TestDeleteBackendCache() failed") + } + + //clean + t.Cleanup(func() { + patches.Reset() + }) +} diff --git a/provider/collect/collect_helper.go b/provider/collect/collect_helper.go new file mode 100644 index 0000000..de09ecb --- /dev/null +++ b/provider/collect/collect_helper.go @@ -0,0 +1,149 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package collect is a package that provides object and performance collect +package collect + +import ( + "context" + "sync" + + cmiConfig "github.com/huawei/csm/v2/config/cmi" + "github.com/huawei/csm/v2/grpc/lib/go/cmi" + "github.com/huawei/csm/v2/provider/constants" + "github.com/huawei/csm/v2/provider/utils" +) + +var ( + // IndicatorsMapping storage indicators mapping + IndicatorsMapping = map[string]int{ + constants.Filesystem: 40, + constants.Lun: 11, + constants.Controller: 207, + constants.StoragePool: 216, + } +) + +// CountFunc count function, e.g. query total filesystem number in storage +type CountFunc func(ctx context.Context) (int, error) + +// QueryFunc query function, e.g. query filesystem information +type QueryFunc func(context.Context) ([]map[string]interface{}, error) + +// PageFunc page query function, e.g. page query filesystem information +type PageFunc func(context.Context, int, int) ([]map[string]interface{}, error) + +// BuildResponse build a collect response +func BuildResponse(request *cmi.CollectRequest) *cmi.CollectResponse { + return &cmi.CollectResponse{ + BackendName: request.GetBackendName(), + CollectType: request.GetCollectType(), + MetricsType: request.GetMetricsType(), + Details: []*cmi.CollectDetail{}, + } +} + +// AddCollectDetail add detail to the response +// input type is struct +func AddCollectDetail[T any](t T, response *cmi.CollectResponse) { + AddCollectDetailWithMap(utils.StructToMap(t), response) +} + +// AddCollectDetailWithMap add detail to the response +// input type is a map +func AddCollectDetailWithMap(data map[string]string, response *cmi.CollectResponse) { + detail := &cmi.CollectDetail{Data: data} + response.Details = append(response.Details, detail) +} + +// ConvertToResponse convert input to response +func ConvertToResponse[I, T any](input I, request *cmi.CollectRequest) (*cmi.CollectResponse, error) { + targets, err := utils.MapToStructSlice[I, T](input) + if err != nil { + return nil, err + } + + response := BuildResponse(request) + for _, target := range targets { + AddCollectDetail(target, response) + } + + return response, nil +} + +// BuildFailedPageResult build a failed paginated result +func BuildFailedPageResult(err error) PageResultTuple { + return PageResultTuple{ + Data: []map[string]interface{}{}, + Error: err, + } +} + +// BuildSuccessPageResult build a successful paginated result +func BuildSuccessPageResult(data []map[string]interface{}) PageResultTuple { + return PageResultTuple{Data: data} +} + +// ConcurrentPaginate a universal concurrent paging query function +// Each page will use a goroutine to query +func ConcurrentPaginate(ctx context.Context, count CountFunc, query PageFunc) ([]map[string]interface{}, error) { + total, err := count(ctx) + if err != nil { + return []map[string]interface{}{}, err + } + + var wg sync.WaitGroup + var out = make(chan PageResultTuple) + var start, pageSize = 0, cmiConfig.GetQueryStoragePageSize() + for total > 0 { + end := start + pageSize + wg.Add(1) + go pageQuery(ctx, start, end, &wg, query, out) + start = end + total -= pageSize + } + + go func() { + wg.Wait() + close(out) + }() + + return ReadQueryResult(out) +} + +// pageQuery page query storage data +func pageQuery(ctx context.Context, start, end int, wg *sync.WaitGroup, query PageFunc, ch chan<- PageResultTuple) { + defer wg.Done() + var result PageResultTuple + pageData, err := query(ctx, start, end) + if err != nil { + result = BuildFailedPageResult(err) + } + result = BuildSuccessPageResult(pageData) + ch <- result +} + +// ReadQueryResult read query result form channel +func ReadQueryResult(input <-chan PageResultTuple) ([]map[string]interface{}, error) { + var result []map[string]interface{} + for tuple := range input { + if tuple.Error != nil { + return nil, tuple.Error + } + result = append(result, tuple.Data...) + } + return result, nil +} diff --git a/provider/collect/collect_helper_test.go b/provider/collect/collect_helper_test.go new file mode 100644 index 0000000..fcf2071 --- /dev/null +++ b/provider/collect/collect_helper_test.go @@ -0,0 +1,103 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package collect is a package that provides object and performance collect +package collect + +import ( + "reflect" + "testing" + + "github.com/huawei/csm/v2/grpc/lib/go/cmi" +) + +func Test_AddCollectDetail_Success(t *testing.T) { + // arrange + response := &cmi.CollectResponse{} + data := map[string]string{ + "Id": "test-id", + "Name": "test-name", + } + detail := struct { + Id string `json:"Id" metrics:"Id"` + Name string `json:"Name" metrics:"Name"` + }{Id: "test-id", Name: "test-name"} + + // action + AddCollectDetail(detail, response) + + // assert + if len(response.GetDetails()) != 1 { + t.Errorf("Test_AddCollectDetail_Success() failed, want deltails = 1, but got = %d", + len(response.GetDetails())) + return + } + + got := response.GetDetails()[0].GetData() + if !reflect.DeepEqual(got, data) { + t.Errorf("Test_AddCollectDetail_Success() failed, want data = %v, but got = %v", data, got) + } +} + +func Test_AddCollectDetailWithMap_Success(t *testing.T) { + // arrange + response := &cmi.CollectResponse{} + data := map[string]string{ + "Id": "test-id", + "Name": "test-name", + } + // action + AddCollectDetailWithMap(data, response) + + // assert + if len(response.GetDetails()) != 1 { + t.Errorf("Test_AddCollectDetailWithMap_Success() failed, want deltails = 1, but got = %d", + len(response.GetDetails())) + return + } + + got := response.GetDetails()[0].GetData() + if !reflect.DeepEqual(got, data) { + t.Errorf("Test_AddCollectDetailWithMap_Success() failed, want data = %v, but got = %v", data, got) + } +} + +func TestConvertToResponse(t *testing.T) { + // arrange + request := &cmi.CollectRequest{ + BackendName: "test-backend", + CollectType: "test-collect", + MetricsType: "test-metrics", + } + input := []map[string]interface{}{ + { + "ID": "1", + "NAME": "TEST-1", + }, + { + "ID": "2", + "NAME": "TEST-2", + }, + } + + // action + _, err := ConvertToResponse[[]map[string]interface{}, LunObject](input, request) + + // assert + if err != nil { + t.Errorf("TestConvertToResponse() failed, error = %v", err) + } +} diff --git a/provider/collect/collect_object.go b/provider/collect/collect_object.go new file mode 100644 index 0000000..b41b2d4 --- /dev/null +++ b/provider/collect/collect_object.go @@ -0,0 +1,111 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package collect is a package that provides object and performance collect +package collect + +import ( + "context" + + "github.com/huawei/csm/v2/grpc/lib/go/cmi" + "github.com/huawei/csm/v2/provider/backend" + "github.com/huawei/csm/v2/provider/constants" + "github.com/huawei/csm/v2/storage/client/centralizedstorage" + "github.com/huawei/csm/v2/utils/log" +) + +// This function will register all objectHandlers +// If a handler is not registered here, an error will be reported when calling DoCollect +// These objectHandlers will be saved in the Global variable register +func init() { + RegisterObjectHandler(constants.OceanStorage, constants.Lun, CollectLun) + RegisterObjectHandler(constants.OceanStorage, constants.Array, CollectArray) + RegisterObjectHandler(constants.OceanStorage, constants.Controller, CollectController) + RegisterObjectHandler(constants.OceanStorage, constants.Filesystem, CollectFilesystem) + RegisterObjectHandler(constants.OceanStorage, constants.StoragePool, CollectStoragePool) +} + +// ObjectCollector object data collector +type ObjectCollector struct{} + +// Collect this purpose of this function is to find a handler and invoke it +func (o *ObjectCollector) Collect(ctx context.Context, request *cmi.CollectRequest) (*cmi.CollectResponse, error) { + clientInfo, err := GetClient(ctx, request.GetBackendName(), backend.GetClientByBackendName) + if err != nil { + log.AddContext(ctx).Errorf("objectCollector get client failed, error: [%v]", err) + return nil, err + } + + handler, err := GetObjectHandler(clientInfo.StorageType, request.GetCollectType()) + if err != nil { + log.AddContext(ctx).Errorf("objectCollector get handler function failed, error: [%v]", err) + return nil, err + } + + return handler(ctx, clientInfo.Client, request) +} + +// CollectArray collect object data of array in storage +func CollectArray(ctx context.Context, client *centralizedstorage.CentralizedClient, + request *cmi.CollectRequest) (*cmi.CollectResponse, error) { + return DoCollect[map[string]interface{}, ArrayObject](ctx, request, client.GetSystemInfo) +} + +// CollectController collect object data of array in storage +func CollectController(ctx context.Context, client *centralizedstorage.CentralizedClient, + request *cmi.CollectRequest) (*cmi.CollectResponse, error) { + return DoCollect[[]map[string]interface{}, ControllerObject](ctx, request, client.GetControllers) +} + +// CollectStoragePool collect object data of storage pool in storage +func CollectStoragePool(ctx context.Context, client *centralizedstorage.CentralizedClient, + request *cmi.CollectRequest) (*cmi.CollectResponse, error) { + return DoCollect[[]map[string]interface{}, StoragePoolObject](ctx, request, client.GetStoragePools) +} + +// CollectLun collect object data of lun in storage +func CollectLun(ctx context.Context, client *centralizedstorage.CentralizedClient, + request *cmi.CollectRequest) (*cmi.CollectResponse, error) { + return DoPageCollect[LunObject](ctx, request, client.GetLunCount, client.GetLuns) +} + +// CollectFilesystem collect object data of filesystem in storage +func CollectFilesystem(ctx context.Context, client *centralizedstorage.CentralizedClient, + request *cmi.CollectRequest) (*cmi.CollectResponse, error) { + return DoPageCollect[FileSystemObject](ctx, request, client.GetFilesystemCount, client.GetFilesystem) +} + +// DoCollect collect data in storage +func DoCollect[I, T any](ctx context.Context, request *cmi.CollectRequest, + fn func(context.Context) (I, error)) (*cmi.CollectResponse, error) { + data, err := fn(ctx) + if err != nil { + log.AddContext(ctx).Errorf("do collect failed, error: %v", err) + return nil, err + } + return ConvertToResponse[I, T](data, request) +} + +// DoPageCollect page collect data in storage +func DoPageCollect[T any](ctx context.Context, request *cmi.CollectRequest, + countFunc CountFunc, pageFunc PageFunc) (*cmi.CollectResponse, error) { + data, err := ConcurrentPaginate(ctx, countFunc, pageFunc) + if err != nil { + log.AddContext(ctx).Errorf("do page collect failed, error: %v", err) + return nil, err + } + return ConvertToResponse[[]map[string]interface{}, T](data, request) +} diff --git a/provider/collect/collect_object_test.go b/provider/collect/collect_object_test.go new file mode 100644 index 0000000..478c683 --- /dev/null +++ b/provider/collect/collect_object_test.go @@ -0,0 +1,163 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package collect is a package that provides object and performance collect +package collect + +import ( + "context" + "errors" + "testing" + + "github.com/agiledragon/gomonkey/v2" + + "github.com/huawei/csm/v2/grpc/lib/go/cmi" + "github.com/huawei/csm/v2/provider/backend" +) + +func TestObjectCollector_Collect_with_client_not_exist(t *testing.T) { + // arrange + var mockCollector = &ObjectCollector{} + + // mock + patches := gomonkey. + ApplyFunc(GetClient, func(context.Context, string, + func(context.Context, string) (backend.ClientInfo, error)) (backend.ClientInfo, error) { + return backend.ClientInfo{}, errors.New("client not exist") + }) + defer patches.Reset() + + // action + _, err := mockCollector.Collect(context.Background(), &cmi.CollectRequest{}) + + // assert + if err == nil || err.Error() != "client not exist" { + t.Errorf("testObjectCollector_Collect_client_not_exist() want an error with client not exist,"+ + " but got error = %s", err.Error()) + } +} + +func TestObjectCollector_Collect_with_handler_not_exist(t *testing.T) { + // arrange + var mockCollector = &ObjectCollector{} + + // mock + patches := gomonkey. + ApplyFunc(GetClient, func(context.Context, string, + func(context.Context, string) (backend.ClientInfo, error)) (backend.ClientInfo, error) { + return backend.ClientInfo{}, nil + }). + ApplyFunc(GetObjectHandler, func(storageType, collectType string) (ObjectHandler, error) { + return nil, errors.New("handler not exist") + }) + defer patches.Reset() + + // action + _, err := mockCollector.Collect(context.Background(), &cmi.CollectRequest{}) + + // assert + if err == nil || err.Error() != "handler not exist" { + t.Errorf("testObjectCollector_Collect_with_handler_not_exist() want an error with handler not exist,"+ + " but got error = %s", err.Error()) + } +} + +func TestObjectCollector_Collect_with_success(t *testing.T) { + // arrange + var mockCollector = &ObjectCollector{} + type mockCorrectClient struct{} + var mockHandler = func(context.Context, interface{}, *cmi.CollectRequest) (*cmi.CollectResponse, error) { + return &cmi.CollectResponse{}, nil + } + + //mock + patches := gomonkey. + ApplyFunc(GetClient, func(context.Context, string, + func(context.Context, string) (backend.ClientInfo, error)) (backend.ClientInfo, error) { + return backend.ClientInfo{Client: &mockCorrectClient{}}, nil + }). + ApplyFunc(GetObjectHandler, func(storageType, collectType string) (ObjectHandler, error) { + return mockHandler, nil + }) + defer patches.Reset() + + // action + _, err := mockCollector.Collect(context.Background(), &cmi.CollectRequest{}) + + // assert + if err != nil { + t.Errorf("testObjectCollector_Collect_with_success() error = %v", err.Error()) + } +} + +func TestDoCollect(t *testing.T) { + // arrange + request := &cmi.CollectRequest{ + BackendName: "test-backend", + CollectType: "test-collect", + MetricsType: "test-metrics", + } + queryFunc := func(context.Context) ([]map[string]interface{}, error) { + var result []map[string]interface{} + for i := 0; i < 1000; i++ { + result = append(result, map[string]interface{}{ + "ID": "123", + "NAME": "name-1", + }) + } + return result, nil + } + + // action + _, err := DoCollect[[]map[string]interface{}, LunObject](context.Background(), request, queryFunc) + + // assert + if err != nil { + t.Errorf("TestDoPageCollect() failed, error = %v", err) + } +} + +func TestDoPageCollect(t *testing.T) { + // arrange + request := &cmi.CollectRequest{ + BackendName: "test-backend", + CollectType: "test-collect", + MetricsType: "test-metrics", + } + + countFunc := func(ctx context.Context) (int, error) { + return 1000, nil + } + pageFunc := func(ctx context.Context, start, end int) ([]map[string]interface{}, error) { + var result []map[string]interface{} + total := end - start + for i := 0; i < total; i++ { + result = append(result, map[string]interface{}{ + "ID": "123", + "NAME": "name-1", + }) + } + return result, nil + } + + // action + _, err := DoPageCollect[LunObject](context.Background(), request, countFunc, pageFunc) + + // assert + if err != nil { + t.Errorf("TestDoPageCollect() failed, error = %v", err) + } +} diff --git a/provider/collect/collect_performance.go b/provider/collect/collect_performance.go new file mode 100644 index 0000000..95894d6 --- /dev/null +++ b/provider/collect/collect_performance.go @@ -0,0 +1,260 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package collect is a package that provides object and performance collect +package collect + +import ( + "context" + "errors" + "strconv" + "strings" + "time" + + "github.com/huawei/csm/v2/grpc/lib/go/cmi" + "github.com/huawei/csm/v2/provider/backend" + "github.com/huawei/csm/v2/provider/constants" + "github.com/huawei/csm/v2/provider/utils" + "github.com/huawei/csm/v2/storage/client/centralizedstorage" + "github.com/huawei/csm/v2/utils/log" +) + +func init() { + RegisterPerformanceHandler(constants.OceanStorage, constants.Lun, GetLunNameMapping) + RegisterPerformanceHandler(constants.OceanStorage, constants.Filesystem, GetFilesystemNameMapping) + RegisterPerformanceHandler(constants.OceanStorage, constants.Controller, GetControllerNameMapping) + RegisterPerformanceHandler(constants.OceanStorage, constants.StoragePool, GetStoragePoolNameMapping) +} + +// PerformanceCollector performance data collector +type PerformanceCollector struct{} + +// Collect performance data +func (p *PerformanceCollector) Collect(ctx context.Context, request *cmi.CollectRequest) (*cmi.CollectResponse, error) { + clientInfo, err := GetClient(ctx, request.GetBackendName(), backend.GetClientByBackendName) + if err != nil { + log.AddContext(ctx).Errorf("objectCollector get Client failed, error: %v", err) + return nil, err + } + + client, ok := clientInfo.Client.(*centralizedstorage.CentralizedClient) + if !ok { + return nil, errors.New("convert Client to centralizedClient failed") + } + + return CollectPerformance(ctx, client, request) +} + +// CollectPerformance collect performance data +func CollectPerformance(ctx context.Context, client *centralizedstorage.CentralizedClient, + request *cmi.CollectRequest) (*cmi.CollectResponse, error) { + // get all performance data. + performances, err := GetPerformanceData(ctx, client, request) + if err != nil { + log.AddContext(ctx).Errorf("collect performance data failed, error: %v", err) + return nil, err + } + + if len(performances) == 0 { + return BuildResponse(request), nil + } + + // get all objects id and name. + nameMapping, err := GetMapping(ctx, request.GetCollectType(), client) + if err != nil { + log.AddContext(ctx).Errorf("collect object name mapping data failed, error: %v", err) + return nil, err + } + + // merge performance data and object name. + return MergePerformance(performances, nameMapping, request), nil +} + +// GetPerformanceData query performance data +func GetPerformanceData(ctx context.Context, client *centralizedstorage.CentralizedClient, + request *cmi.CollectRequest) ([]PerformanceIndicators, error) { + objectType, ok := IndicatorsMapping[request.CollectType] + if !ok { + return nil, errors.New("illegalArgumentErrorunsupported collect type") + } + + storageInfo, err := client.GetSystemInfo(ctx) + if err != nil { + log.AddContext(ctx).Errorf("get storage system info failed, error: %v", err) + return nil, err + } + + var mapData []map[string]interface{} + var postEnable bool + indicators := utils.MapStringToInt(request.Indicators) + version, ok := storageInfo["pointRelease"].(string) + if !ok { + // storage of V3 or V5 not has the pointRelease field + postEnable = false + } else if !strings.HasPrefix(version, constants.StorageV6PointReleasePrefix) { + // only storage of V6 pointRelease is started with number, + // storage with V7 or later version supports the Post request + postEnable = true + } else if version >= constants.MinVersionSupportPost { + // 6.1.2 and later versions in V6 storage support the Post request + postEnable = true + } + + if postEnable { + mapData, err = client.GetPerformanceByPost(ctx, objectType, indicators) + } else { + for i := 0; i < 5; i++ { + mapData, err = client.GetPerformance(ctx, objectType, indicators) + // For storage v6 earlier 6.1.2, if it can not return the performance data caused by concurrency, + // both the mapData and err are nil. But in the same conditions for storage v3 or v5, the mapData + // is nil while the err is not nil. + if err != nil { + break + } + if len(mapData) != 0 { + break + } + time.Sleep(5 * time.Second) + } + } + if err != nil { + log.AddContext(ctx).Errorf("invoke the get performance method of storage client failed, error: %v", err) + return nil, err + } + + // For storage v6 earlier 6.1.2, the storage may return empty data even after 5 time retries. + if len(mapData) == 0 { + log.AddContext(ctx).Warningln("get empty data by the get performance method of storage client") + } + + return utils.MapToStruct[[]map[string]interface{}, []PerformanceIndicators](mapData) +} + +// GetMapping get object mapping +// result map key is object id. +// +// result map value is object name. +func GetMapping(ctx context.Context, collectType string, + client *centralizedstorage.CentralizedClient) (map[string]string, error) { + handler, err := GetPerformanceHandler(constants.OceanStorage, collectType) + if err != nil { + log.AddContext(ctx).Errorf("get performance handler failed, error: %v", err) + return nil, err + } + + return handler(ctx, client) +} + +// MergePerformance merge performance data +func MergePerformance(performances []PerformanceIndicators, nameMapping map[string]string, + request *cmi.CollectRequest) *cmi.CollectResponse { + + response := BuildResponse(request) + for _, performance := range performances { + mapData := performance.ToMap() + objectName, ok := nameMapping[performance.ObjectId] + if !ok { + continue + } + mapData[constants.ObjectName] = objectName + mapData[constants.ObjectId] = performance.ObjectId + AddCollectDetailWithMap(mapData, response) + } + + return response +} + +// ToMap Parse performance data and convert it into a map +func (p PerformanceIndicators) ToMap() map[string]string { + if len(p.Indicators) == 0 || len(p.Indicators) != len(p.IndicatorValues) { + return map[string]string{} + } + + var dataMap = map[string]string{} + for i, indicator := range p.Indicators { + key := strconv.Itoa(indicator) + dataMap[key] = strconv.FormatFloat(p.IndicatorValues[i], 'f', 4, 64) + } + return dataMap +} + +// GetNameMapping A universal function for obtaining name mapping +func GetNameMapping(ctx context.Context, queryFunc QueryFunc) (map[string]string, error) { + data, err := queryFunc(ctx) + if err != nil { + log.AddContext(ctx).Errorf("query storage to get name mapping failed, error: %v", err) + return map[string]string{}, nil + } + return DoNameMapping(data), nil +} + +// GetNameMappingWithPage A universal function for obtaining name mapping with page query +func GetNameMappingWithPage(ctx context.Context, countFunc CountFunc, pageFunc PageFunc) (map[string]string, error) { + data, err := ConcurrentPaginate(ctx, countFunc, pageFunc) + if err != nil { + log.AddContext(ctx).Errorf("concurrent Paginate failed, error: %v", err) + return nil, err + } + return DoNameMapping(data), nil +} + +// DoNameMapping A universal function for parsing name mapping +func DoNameMapping(data []map[string]interface{}) map[string]string { + var nameMapping = map[string]string{} + for _, item := range data { + id, ok := item["ID"].(string) + if !ok { + continue + } + name, ok := item["NAME"].(string) + if !ok { + continue + } + nameMapping[id] = name + } + return nameMapping +} + +// GetLunNameMapping get lun name mapping +// Key is lun id. +// Value is lun name. +func GetLunNameMapping(ctx context.Context, client *centralizedstorage.CentralizedClient) (map[string]string, error) { + return GetNameMappingWithPage(ctx, client.GetLunCount, client.GetLuns) +} + +// GetFilesystemNameMapping get filesystem name mapping +// Key is filesystem id. +// Value is filesystem name. +func GetFilesystemNameMapping(ctx context.Context, + client *centralizedstorage.CentralizedClient) (map[string]string, error) { + return GetNameMappingWithPage(ctx, client.GetFilesystemCount, client.GetFilesystem) +} + +// GetControllerNameMapping get controller name mapping +// Key is controller id. +// Value is controller name. +func GetControllerNameMapping(ctx context.Context, + client *centralizedstorage.CentralizedClient) (map[string]string, error) { + return GetNameMapping(ctx, client.GetControllers) +} + +// GetStoragePoolNameMapping get storage pool name mapping +// Key is storage pool id. +// Value is storage pool name. +func GetStoragePoolNameMapping(ctx context.Context, + client *centralizedstorage.CentralizedClient) (map[string]string, error) { + return GetNameMapping(ctx, client.GetStoragePools) +} diff --git a/provider/collect/collect_performance_test.go b/provider/collect/collect_performance_test.go new file mode 100644 index 0000000..d32f446 --- /dev/null +++ b/provider/collect/collect_performance_test.go @@ -0,0 +1,436 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package collect is a package that provides object and performance collect +package collect + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + + "github.com/huawei/csm/v2/grpc/lib/go/cmi" + "github.com/huawei/csm/v2/provider/constants" + "github.com/huawei/csm/v2/provider/utils" + "github.com/huawei/csm/v2/storage/client/centralizedstorage" +) + +func TestDoNameMapping(t *testing.T) { + // arrange + data := []map[string]interface{}{ + {"ID": "ID-1", "NAME": "NAME-1"}, + {"ID": "ID-2", "NAME": "NAME-2"}, + {"ID": "ID-3", "NAME": "NAME-3"}, + {"ID": "ID-3", "NAME-NOT-EXIST": "NAME--NOT-EXIST"}, + {"ID-NOT-EXIST": "ID-NOT-EXIST", "NAME-NOT-EXIST": "NAME-NOT-EXIST"}, + } + want := map[string]string{ + "ID-1": "NAME-1", + "ID-2": "NAME-2", + "ID-3": "NAME-3", + } + + // action + got := DoNameMapping(data) + + // assert + if !reflect.DeepEqual(want, got) { + t.Errorf("TestDoNameMapping() want = %v, but got = %v", want, got) + } +} + +func TestGetNameMappingWithPage(t *testing.T) { + // arrange + want := map[string]string{ + "ID-1": "NAME-1", + "ID-2": "NAME-2", + "ID-3": "NAME-3", + } + + countFunc := func(ctx context.Context) (int, error) { + return 1000, nil + } + + pageFunc := func(ctx context.Context, start, end int) ([]map[string]interface{}, error) { + return []map[string]interface{}{}, nil + } + + // mock + applyFunc := gomonkey.ApplyFunc(ConcurrentPaginate, func(context.Context, CountFunc, + PageFunc) ([]map[string]interface{}, error) { + return []map[string]interface{}{ + {"ID": "ID-1", "NAME": "NAME-1"}, + {"ID": "ID-2", "NAME": "NAME-2"}, + {"ID": "ID-3", "NAME": "NAME-3"}, + }, nil + }) + defer applyFunc.Reset() + + // action + got, err := GetNameMappingWithPage(context.Background(), countFunc, pageFunc) + + // assert + if err != nil { + t.Errorf("TestGetNameMappingWithPage() error = %v", err) + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("TestGetNameMappingWithPage() want = %v, but got = %v", want, got) + } +} + +func TestGetNameMapping(t *testing.T) { + // arrange + want := map[string]string{ + "ID-1": "NAME-1", + "ID-2": "NAME-2", + "ID-3": "NAME-3", + } + + queryFunc := func(ctx context.Context) ([]map[string]interface{}, error) { + return []map[string]interface{}{ + {"ID": "ID-1", "NAME": "NAME-1"}, + {"ID": "ID-2", "NAME": "NAME-2"}, + {"ID": "ID-3", "NAME": "NAME-3"}, + }, nil + } + + // action + got, err := GetNameMapping(context.Background(), queryFunc) + + // assert + if err != nil { + t.Errorf("TestGetNameMapping() error = %v", err) + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("TestGetNameMapping() want = %v, but got = %v", want, got) + } +} + +func TestCollectPerformance_with_get_performance_data_error(t *testing.T) { + // arrange + request := &cmi.CollectRequest{} + client := ¢ralizedstorage.CentralizedClient{} + + // mock + applyFunc := gomonkey.ApplyFunc(GetPerformanceData, func(ctx context.Context, + client *centralizedstorage.CentralizedClient, request *cmi.CollectRequest) ([]PerformanceIndicators, error) { + return []PerformanceIndicators{}, errors.New("GetPerformanceData error") + }) + defer applyFunc.Reset() + + // action + _, err := CollectPerformance(context.Background(), client, request) + + // assert + if err == nil { + t.Error("TestGetNameMapping() want an GetPerformanceData error, but got nil") + } +} + +func TestCollectPerformance_with_get_mapping_error(t *testing.T) { + // arrange + request := &cmi.CollectRequest{} + client := ¢ralizedstorage.CentralizedClient{} + + // mock + applyFunc := gomonkey. + ApplyFunc(GetPerformanceData, func(ctx context.Context, + client *centralizedstorage.CentralizedClient, request *cmi.CollectRequest) ([]PerformanceIndicators, error) { + return []PerformanceIndicators{{}}, nil + }). + ApplyFunc(GetMapping, func(ctx context.Context, collectType string, + client *centralizedstorage.CentralizedClient) (map[string]string, error) { + return nil, errors.New("GetMapping error") + }) + defer applyFunc.Reset() + + // action + _, err := CollectPerformance(context.Background(), client, request) + + // assert + if err == nil { + t.Error("TestGetNameMapping() want an GetMapping error, but got nil") + } +} + +func TestCollectPerformance_with_success(t *testing.T) { + // arrange + request := &cmi.CollectRequest{} + client := ¢ralizedstorage.CentralizedClient{} + + // mock + applyFunc := gomonkey. + ApplyFunc(GetPerformanceData, func(ctx context.Context, + client *centralizedstorage.CentralizedClient, request *cmi.CollectRequest) ([]PerformanceIndicators, error) { + return []PerformanceIndicators{}, nil + }). + ApplyFunc(GetMapping, func(ctx context.Context, collectType string, + client *centralizedstorage.CentralizedClient) (map[string]string, error) { + return map[string]string{}, nil + }). + ApplyFunc(MergePerformance, func(performances []PerformanceIndicators, nameMapping map[string]string, + request *cmi.CollectRequest) *cmi.CollectResponse { + return &cmi.CollectResponse{} + }) + defer applyFunc.Reset() + + // action + _, err := CollectPerformance(context.Background(), client, request) + + // assert + if err != nil { + t.Errorf("TestGetNameMapping() error = %v", err) + } +} + +func TestGetPerformanceData_with_storage_V7_success(t *testing.T) { + // arrange + request := &cmi.CollectRequest{CollectType: constants.Filesystem} + client := ¢ralizedstorage.CentralizedClient{} + + // mock + applyFunc := gomonkey. + ApplyFunc(utils.MapStringToInt, func(sources []string) []int { + return []int{1, 2} + }). + ApplyMethodFunc(client, "GetSystemInfo", func(ctx context.Context) ( + map[string]interface{}, error) { + return map[string]interface{}{ + "pointRelease": "V700R001C00", + }, nil + }). + ApplyMethodFunc(client, "GetPerformanceByPost", func(ctx context.Context, + objectType int, indicators []int) ([]map[string]interface{}, error) { + return []map[string]interface{}{ + { + "indicators": []int{1, 2}, + "indicator_values": []float64{0.0, 1.0}, + "object_id": "1", + }, + }, nil + }) + defer applyFunc.Reset() + + // action + _, err := GetPerformanceData(context.Background(), client, request) + + // assert + if err != nil { + t.Errorf("TestGetPerformanceData_with_storage_V7_success() error = %v", err) + } +} + +func TestGetPerformanceData_with_storage_greater_V612_success(t *testing.T) { + // arrange + request := &cmi.CollectRequest{CollectType: constants.Filesystem} + client := ¢ralizedstorage.CentralizedClient{} + + // mock + applyFunc := gomonkey. + ApplyFunc(utils.MapStringToInt, func(sources []string) []int { + return []int{1, 2} + }). + ApplyMethodFunc(client, "GetSystemInfo", func(ctx context.Context) ( + map[string]interface{}, error) { + return map[string]interface{}{ + "pointRelease": "6.1.7", + }, nil + }). + ApplyMethodFunc(client, "GetPerformanceByPost", func(ctx context.Context, + objectType int, indicators []int) ([]map[string]interface{}, error) { + return []map[string]interface{}{ + { + "indicators": []int{1, 2}, + "indicator_values": []float64{0.0, 1.0}, + "object_id": "1", + }, + }, nil + }) + defer applyFunc.Reset() + + // action + _, err := GetPerformanceData(context.Background(), client, request) + + // assert + if err != nil { + t.Errorf("TestGetPerformanceData_with_storage_greater_V612_success() error = %v", err) + } +} + +func TestGetPerformanceData_with_storage_V610_success(t *testing.T) { + // arrange + request := &cmi.CollectRequest{CollectType: constants.Filesystem} + client := ¢ralizedstorage.CentralizedClient{} + + // mock + applyFunc := gomonkey. + ApplyFunc(utils.MapStringToInt, func(sources []string) []int { + return []int{1, 2} + }). + ApplyMethodFunc(client, "GetSystemInfo", func(ctx context.Context) ( + map[string]interface{}, error) { + return map[string]interface{}{ + "pointRelease": "6.1.0", + }, nil + }). + ApplyMethodFunc(client, "GetPerformance", func(ctx context.Context, + objectType int, indicators []int) ([]map[string]interface{}, error) { + return []map[string]interface{}{ + { + "indicators": []int{1, 2}, + "indicator_values": []float64{0.0, 1.0}, + "object_id": "1", + }, + }, nil + }) + defer applyFunc.Reset() + + // action + _, err := GetPerformanceData(context.Background(), client, request) + + // assert + if err != nil { + t.Errorf("TestGetPerformanceData_with_storage_V610_success() error = %v", err) + } +} + +func TestGetPerformanceData_with_storage_V610_empty_return_success(t *testing.T) { + // arrange + request := &cmi.CollectRequest{CollectType: constants.Filesystem} + client := ¢ralizedstorage.CentralizedClient{} + + // mock + applyFunc := gomonkey. + ApplyFunc(utils.MapStringToInt, func(sources []string) []int { + return []int{1, 2} + }). + ApplyMethodFunc(client, "GetSystemInfo", func(ctx context.Context) ( + map[string]interface{}, error) { + return map[string]interface{}{ + "pointRelease": "6.1.0", + }, nil + }). + ApplyMethodFunc(client, "GetPerformance", func(ctx context.Context, + objectType int, indicators []int) ([]map[string]interface{}, error) { + return nil, nil + }) + defer applyFunc.Reset() + + // action + _, err := GetPerformanceData(context.Background(), client, request) + + // assert + if err != nil { + t.Errorf("TestGetPerformanceData_with_storage_v610_empty_return_success() error = %v", err) + } +} + +func TestGetPerformanceData_with_storage_V3orV5_success(t *testing.T) { + // arrange + request := &cmi.CollectRequest{CollectType: constants.Filesystem} + client := ¢ralizedstorage.CentralizedClient{} + + // mock + applyFunc := gomonkey. + ApplyFunc(utils.MapStringToInt, func(sources []string) []int { + return []int{1, 2} + }). + ApplyMethodFunc(client, "GetSystemInfo", func(ctx context.Context) ( + map[string]interface{}, error) { + return map[string]interface{}{}, nil + }). + ApplyMethodFunc(client, "GetPerformance", func(ctx context.Context, + objectType int, indicators []int) ([]map[string]interface{}, error) { + return []map[string]interface{}{ + { + "indicators": []int{1, 2}, + "indicator_values": []float64{0.0, 1.0}, + "object_id": "1", + }, + }, nil + }) + defer applyFunc.Reset() + + // action + _, err := GetPerformanceData(context.Background(), client, request) + + // assert + if err != nil { + t.Errorf("TestGetPerformanceData_with_storage_v610_empty_return_success() error = %v", err) + } +} + +func TestGetPerformanceData_with_collect_type_not_exist_error(t *testing.T) { + // arrange + request := &cmi.CollectRequest{CollectType: "not-exist"} + client := ¢ralizedstorage.CentralizedClient{} + + // action + _, err := GetPerformanceData(context.Background(), client, request) + + // assert + if err == nil { + t.Error("TestGetPerformanceData() want an error, but got nil") + } +} + +func TestMergePerformance(t *testing.T) { + // arrange + request := &cmi.CollectRequest{CollectType: "not-exist"} + nameMapping := map[string]string{"1": "test-object"} + performances := []PerformanceIndicators{ + { + Indicators: []int{1, 2}, + IndicatorValues: []float64{0.0, 1.1}, + ObjectId: "1", + }, + } + + // action + response := MergePerformance(performances, nameMapping, request) + + // assert + if len(response.GetDetails()) != 1 { + t.Error("TestGetPerformanceData() want an error, but got nil") + } +} + +func TestPerformanceIndicators_ToMap(t *testing.T) { + // arrange + performances := PerformanceIndicators{ + Indicators: []int{1, 2}, + IndicatorValues: []float64{0.0, 1.1}, + ObjectId: "1", + } + + want := map[string]string{ + "1": "0.0000", + "2": "1.1000", + } + + // action + got := performances.ToMap() + + // assert + if !reflect.DeepEqual(got, want) { + t.Errorf("TestGetPerformanceData() want = %v, but got = %v", want, got) + } +} diff --git a/provider/collect/collector_factory.go b/provider/collect/collector_factory.go new file mode 100644 index 0000000..7becb41 --- /dev/null +++ b/provider/collect/collector_factory.go @@ -0,0 +1,41 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package collect is a package that provides object and performance collect +package collect + +import ( + "errors" + "fmt" + + "github.com/huawei/csm/v2/grpc/lib/go/cmi" + "github.com/huawei/csm/v2/provider/constants" +) + +var collectorMap = map[string]cmi.CollectorServer{ + constants.Object: &ObjectCollector{}, + constants.Performance: &PerformanceCollector{}, +} + +// GetCollector get collector by metrics type +func GetCollector(metricsType string) (cmi.CollectorServer, error) { + collector, ok := collectorMap[metricsType] + if ok { + return collector, nil + } + errMsg := fmt.Sprintf("not found collector, metricsType type is [%s] ", metricsType) + return nil, errors.New(errMsg) +} diff --git a/provider/collect/collector_factory_test.go b/provider/collect/collector_factory_test.go new file mode 100644 index 0000000..6b98fb5 --- /dev/null +++ b/provider/collect/collector_factory_test.go @@ -0,0 +1,76 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package collect is a package that provides object and performance collect +package collect + +import ( + "reflect" + "testing" + + "github.com/huawei/csm/v2/provider/constants" +) + +func TestGetCollector_with_object_collector(t *testing.T) { + // arrange + var testMetricsType = constants.Object + + // action + collector, err := GetCollector(testMetricsType) + + // assert + if err != nil { + t.Errorf("TestGetCollector_with_object_collector() error = %v", err) + return + } + + if !reflect.DeepEqual(collector, &ObjectCollector{}) { + t.Errorf("GetCollector() got = %v, want %v", collector, &ObjectCollector{}) + } +} + +func TestGetCollector_with_performance_collector(t *testing.T) { + // arrange + var testMetricsType = constants.Performance + + // action + collector, err := GetCollector(testMetricsType) + + // assert + if err != nil { + t.Errorf("TestGetCollector_with_performance_collector() error = %v", err) + return + } + + if !reflect.DeepEqual(collector, &PerformanceCollector{}) { + t.Errorf("TestGetCollector_with_performance_collector() got = %v, want %v", + collector, &PerformanceCollector{}) + } +} + +func TestGetCollector_with_collector_not_exist(t *testing.T) { + // arrange + var testMetricsType = "not-exist-type" + + // action + _, err := GetCollector(testMetricsType) + + // assert + if err == nil { + t.Errorf("TestGetCollector_with_collector_not_exist() want an error but error is nil") + return + } +} diff --git a/provider/collect/register.go b/provider/collect/register.go new file mode 100644 index 0000000..cedc492 --- /dev/null +++ b/provider/collect/register.go @@ -0,0 +1,203 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package collect is a package that provides object and performance collect +package collect + +import ( + "context" + "errors" + "fmt" + "reflect" + "sync" + + "github.com/huawei/csm/v2/grpc/lib/go/cmi" + "github.com/huawei/csm/v2/provider/backend" + "github.com/huawei/csm/v2/utils/log" +) + +var mutex sync.Mutex + +// clientCache +// key is backend name +// values is a storage client, +// e.g. +// +// |-----------------|---------------------------------------| +// | backendName | client | +// |-----------------|---------------------------------------| +// | test-backend | centralizedstorage.CentralizedClient | +// |---------------------------------------------------------| +var clientCache = map[string]backend.ClientInfo{} + +// objectHandlerCache is routing table with three-layer routing +// e.g. +// +// |-------------------|-----------------|-------------------| +// | StorageType | collectType | handler | +// |-------------------|-----------------|-------------------| +// | oceanStorage | controller | CollectController| +// |-------------------|-----------------|-------------------| +// +// The above table indicates that the object data of the controllers in the ocean storage will be collected using +// the CollectController function +var objectHandlerCache = &HandlerMap[ObjectHandler]{} + +// performanceHandlerCache is routing table with three-layer routing +// e.g. +// +// |-------------------|----------------|--------------------| +// | StorageType | collectType | handler | +// |-------------------|----------------|--------------------| +// | oceanStorage | controller | GetLunNameMapping | +// |-------------------|----------------|--------------------| +// +// The above table indicates that GetLunNameMapping is specified to obtain the name mapping of the lun volume +var performanceHandlerCache = &HandlerMap[PerformanceHandler]{} + +// HandlerMap cache format +type HandlerMap[T any] map[string]map[string]T + +// ObjectHandler object handler format +type ObjectHandler TObjectHandler[interface{}] + +// TObjectHandler When clients are different, we must use generics to represent different clients +type TObjectHandler[T any] func(context.Context, T, *cmi.CollectRequest) (*cmi.CollectResponse, error) + +// PerformanceHandler performance handler format +type PerformanceHandler TPerformanceHandler[interface{}] + +// TPerformanceHandler When clients are different, we must use generics to represent different clients +type TPerformanceHandler[T any] func(context.Context, T) (map[string]string, error) + +// RegisterObjectHandler register a function to handle object data +func RegisterObjectHandler[T any](storageType, collectType string, tHandler TObjectHandler[T]) { + registerHandler(objectHandlerCache, storageType, collectType, tHandler.ToObjectHandler()) +} + +// RegisterPerformanceHandler register a function to handle performance data +func RegisterPerformanceHandler[T any](storageType, collectType string, tHandler TPerformanceHandler[T]) { + registerHandler(performanceHandlerCache, storageType, collectType, tHandler.ToPerformanceHandler()) +} + +// RegisterClient key is backend name, value is ClientInfo +func RegisterClient(backendName string, info backend.ClientInfo) { + mutex.Lock() + defer mutex.Unlock() + + clientCache[backendName] = info +} + +// RemoveClient remove the client from cache +func RemoveClient(backendName string) { + mutex.Lock() + defer mutex.Unlock() + + delete(clientCache, backendName) +} + +// GetObjectHandler get collect object data handler +func GetObjectHandler(storageType, collectType string) (ObjectHandler, error) { + return getHandler(objectHandlerCache, storageType, collectType) +} + +// GetPerformanceHandler get collect performance data handler +func GetPerformanceHandler(storageType, collectType string) (PerformanceHandler, error) { + return getHandler(performanceHandlerCache, storageType, collectType) +} + +// registerHandler register a handler with the specified key to the cache +func registerHandler[T any](cache *HandlerMap[T], storageType, collectType string, handler T) { + mutex.Lock() + defer mutex.Unlock() + + handlerMap, ok := (*cache)[storageType] + if !ok { + handlerMap = map[string]T{} + (*cache)[storageType] = handlerMap + } + + handlerMap[collectType] = handler + (*cache)[storageType] = handlerMap +} + +// getHandler query whether there is a handler in the specified cache based on the specified key. +// If so, return the handler. If not, return an error +func getHandler[T any](cache *HandlerMap[T], storageType, collectType string) (T, error) { + handlers, ok := (*cache)[storageType] + var t T + if !ok { + errMsg := fmt.Sprintf("not found handlers, storageType type is [%s] ", collectType) + return t, errors.New(errMsg) + } + + handler, ok := handlers[collectType] + if ok { + return handler, nil + } + + errMsg := fmt.Sprintf("not found handlers, collect type is [%s] ", collectType) + return t, errors.New(errMsg) +} + +// GetClient get or register client +// This function needs two parameter: backendName and discover function. +// discover function should return an instance of client. +func GetClient(ctx context.Context, backendName string, + discoverFunc func(context.Context, string) (backend.ClientInfo, error)) (backend.ClientInfo, error) { + client, ok := clientCache[backendName] + if ok { + return client, nil + } + + client, err := discoverFunc(ctx, backendName) + if err != nil { + log.AddContext(ctx).Errorf("discover client failed, backend name: [%s], error: [%v]", backendName, err) + return backend.ClientInfo{}, err + } + RegisterClient(backendName, client) + return client, nil +} + +// ToObjectHandler convert TObjectHandler to ObjectHandler +func (receiver TObjectHandler[T]) ToObjectHandler() ObjectHandler { + return func(ctx context.Context, param interface{}, request *cmi.CollectRequest) (*cmi.CollectResponse, error) { + if param == nil { + return nil, errors.New("ToObjectHandler IllegalArgumentError, handler function argument is nil") + } + if t, ok := param.(T); ok { + return receiver(ctx, t, request) + } + errMsg := fmt.Sprintf("ToObjectHandler IllegalArgumentError, current param is [%s], "+ + "want is [%s]", reflect.TypeOf(param).Kind().String(), reflect.TypeOf((*T)(nil)).Kind().String()) + return nil, errors.New(errMsg) + } +} + +// ToPerformanceHandler convert TPerformanceHandler to PerformanceHandler +func (receiver TPerformanceHandler[T]) ToPerformanceHandler() PerformanceHandler { + return func(ctx context.Context, param interface{}) (map[string]string, error) { + if param == nil { + return nil, errors.New("ToPerformanceHandler IllegalArgumentError, handler function argument is nil") + } + if t, ok := param.(T); ok { + return receiver(ctx, t) + } + errMsg := fmt.Sprintf("ToPerformanceHandler IllegalArgumentError, current param is [%s], "+ + "want is [%s]", reflect.TypeOf(param).Kind().String(), reflect.TypeOf((*T)(nil)).Kind().String()) + return nil, errors.New(errMsg) + } +} diff --git a/provider/collect/register_test.go b/provider/collect/register_test.go new file mode 100644 index 0000000..91a9a3e --- /dev/null +++ b/provider/collect/register_test.go @@ -0,0 +1,206 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package collect is a package that provides object and performance collect +package collect + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/huawei/csm/v2/grpc/lib/go/cmi" + "github.com/huawei/csm/v2/provider/backend" +) + +func TestRegisterClient(t *testing.T) { + // arrange + var backendName = "mock-backend-name=register-client" + var mockClient = backend.ClientInfo{StorageType: "test-collect"} + + // action + RegisterClient(backendName, mockClient) + + // assert + gotClient, ok := clientCache[backendName] + if !ok { + t.Errorf("RegisterClient() want = %v, but got = %v", mockClient, nil) + } + if !reflect.DeepEqual(mockClient, gotClient) { + t.Errorf("RegisterClient() want = %v, but got = %v", mockClient, gotClient) + } +} + +func TestRemoveClient_success(t *testing.T) { + //arrange + var backendName = "mock-backend-name=register-client" + var mockClient = backend.ClientInfo{StorageType: "test-collect"} + RegisterClient(backendName, mockClient) + + // action + RemoveClient(backendName) + + // assert + _, ok := clientCache[backendName] + if ok { + t.Errorf("RemoveClient() failed") + } +} + +func TestGetClient_success(t *testing.T) { + // arrange + var backendName = "mock-backend-name-with-get-client-success" + var mockClient = backend.ClientInfo{StorageType: "test-collect"} + var discoverFunc = func(ctx context.Context, name string) (backend.ClientInfo, error) { + return mockClient, nil + } + + // action + got, err := GetClient(context.Background(), backendName, discoverFunc) + + // assert + if err != nil { + t.Errorf("TestGetClient_success() error = %v", err) + return + } + if !reflect.DeepEqual(got, mockClient) { + t.Errorf("TestGetClient_success() got = %v, want %v", got, mockClient) + } +} + +func TestGetClient_with_discover_return_error(t *testing.T) { + // arrange + var backendName = "mock-backend-name-with-get-client" + var discoverFunc = func(ctx context.Context, name string) (backend.ClientInfo, error) { + return backend.ClientInfo{}, errors.New("discover error") + } + + // action + _, err := GetClient(context.Background(), backendName, discoverFunc) + + // assert + if err == nil { + t.Error("GetClient() want an error, but error is nil") + } +} + +func TestRegisterObjectHandler_success(t *testing.T) { + // arrange + var mockStorageType, mockCollectType = "test-storage", "test-collect" + var mockObjectFunc = func(context.Context, interface{}, *cmi.CollectRequest) (*cmi.CollectResponse, error) { + return nil, nil + } + + // action + RegisterObjectHandler(mockStorageType, mockCollectType, mockObjectFunc) + + // assert + handlerMap, ok := (*objectHandlerCache)[mockStorageType] + if !ok || len(handlerMap) == 0 { + t.Error("RegisterObjectHandler() failed, want handlerMap is not empty") + return + } + _, ok = handlerMap[mockCollectType] + if !ok { + t.Error("RegisterObjectHandler() failed, want handler is not nil") + } +} + +func TestRegisterPerformanceHandler_success(t *testing.T) { + // arrange + var mockStorageType, mockCollectType = "test-storage", "test-collect" + var mockPerformanceFunc = func(context.Context, interface{}) (map[string]string, error) { + return map[string]string{}, nil + } + + // action + RegisterPerformanceHandler(mockStorageType, mockCollectType, mockPerformanceFunc) + + // assert + handlerMap, ok := (*performanceHandlerCache)[mockStorageType] + if !ok || len(handlerMap) == 0 { + t.Error("RegisterPerformanceHandler() failed, want handlerMap is not empty") + return + } + _, ok = handlerMap[mockCollectType] + if !ok { + t.Error("RegisterPerformanceHandler() failed, want handler is not nil") + } +} + +func TestGetHandler_when_storage_type_is_not_exist(t *testing.T) { + // arrange + var storageType, collectType = "not_exist_storage_type", "collect" + + // action + _, err := GetObjectHandler(storageType, collectType) + + // assert + if err == nil { + t.Error("TestGetHandler_when_storage_type_is_not_exist() want an error, but error is nil") + } +} + +func TestGetHandler_when_collect_type_is_not_exist(t *testing.T) { + // arrange + var storageType, collectType = "storage_type", "storage_type" + var objectFunc = func(context.Context, interface{}, *cmi.CollectRequest) (*cmi.CollectResponse, error) { + return nil, nil + } + + // mock + objectHandlerCache = &HandlerMap[ObjectHandler]{ + storageType: { + collectType: objectFunc, + }, + } + + // action + _, err := GetObjectHandler(storageType, "not_exist_storage_type") + + // assert + if err == nil { + t.Error("TestGetHandler_when_collect_type_is_not_exist() want an error, but error is nil") + } +} + +func TestGetHandler_success(t *testing.T) { + // arrange + var storageType, collectType = "storage_type", "storage_type" + var objectFunc = func(context.Context, interface{}, *cmi.CollectRequest) (*cmi.CollectResponse, error) { + return nil, nil + } + + // mock + objectHandlerCache = &HandlerMap[ObjectHandler]{ + storageType: { + collectType: objectFunc, + }, + } + + // action + got, err := GetObjectHandler(storageType, collectType) + + // assert + if err != nil { + t.Errorf("TestGetHandler_success() error = %v", err) + } + if !reflect.DeepEqual(reflect.ValueOf(got).Pointer(), reflect.ValueOf(objectFunc).Pointer()) { + t.Errorf("TestGetHandler_success() got = %v, want %v", + reflect.ValueOf(got), reflect.ValueOf(objectFunc)) + } +} diff --git a/provider/collect/types.go b/provider/collect/types.go new file mode 100644 index 0000000..d48fc1e --- /dev/null +++ b/provider/collect/types.go @@ -0,0 +1,78 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package collect is a package that provides object and performance collect +package collect + +// PageResultTuple page query result +type PageResultTuple struct { + Error error + Data []map[string]interface{} +} + +// PerformanceIndicators performance information +type PerformanceIndicators struct { + Indicators []int `json:"indicators"` + IndicatorValues []float64 `json:"indicator_values"` + ObjectId string `json:"object_id"` +} + +// ArrayObject array object information +type ArrayObject struct { + Id string `json:"ID" metrics:"ID"` + ProductModeString string `json:"productModeString" metrics:"productModeString"` + ProductMode string `json:"PRODUCTMODE" metrics:"PRODUCTMODE"` + ProductVersion string `json:"PRODUCTVERSION" metrics:"PRODUCTVERSION"` + HealthStatus string `json:"HEALTHSTATUS" metrics:"HEALTHSTATUS"` + RunningStatus string `json:"RUNNINGSTATUS" metrics:"RUNNINGSTATUS"` +} + +// LunObject lun object information +type LunObject struct { + Id string `json:"ID" metrics:"ID"` + Name string `json:"NAME" metrics:"NAME"` + Capacity string `json:"CAPACITY" metrics:"CAPACITY"` + AllocCapacity string `json:"ALLOCCAPACITY" metrics:"ALLOCCAPACITY"` +} + +// ControllerObject controller object information +type ControllerObject struct { + Id string `json:"ID" metrics:"ID"` + Name string `json:"NAME" metrics:"NAME"` + CpuUsage string `json:"CPUUSAGE" metrics:"CPUUSAGE"` + MemoryUsage string `json:"MEMORYUSAGE" metrics:"MEMORYUSAGE"` + RunningStatus string `json:"RUNNINGSTATUS" metrics:"RUNNINGSTATUS"` + HealthStatus string `json:"HEALTHSTATUS" metrics:"HEALTHSTATUS"` +} + +// StoragePoolObject storage pool object information +type StoragePoolObject struct { + Id string `json:"ID" metrics:"ID"` + Name string `json:"NAME" metrics:"NAME"` + FreeCapacity string `json:"USERFREECAPACITY" metrics:"USERFREECAPACITY"` + UsedCapacity string `json:"USERCONSUMEDCAPACITY" metrics:"USERCONSUMEDCAPACITY"` + TotalCapacity string `json:"USERTOTALCAPACITY" metrics:"USERTOTALCAPACITY"` + CapacityUsage string `json:"USERCONSUMEDCAPACITYPERCENTAGE" metrics:"USERCONSUMEDCAPACITYPERCENTAGE"` +} + +// FileSystemObject filesystem object information +type FileSystemObject struct { + Id string `json:"ID" metrics:"ID"` + Name string `json:"NAME" metrics:"NAME"` + Capacity string `json:"CAPACITY" metrics:"CAPACITY"` + AllocCapacity string `json:"ALLOCCAPACITY" metrics:"ALLOCCAPACITY"` + AvailableAndAllocCapacityRatio string `json:"AVAILABLEANDALLOCCAPACITYRATIO" metrics:"AVAILABLEANDALLOCCAPACITYRATIO"` +} diff --git a/provider/constants/constants.go b/provider/constants/constants.go new file mode 100644 index 0000000..d6e0893 --- /dev/null +++ b/provider/constants/constants.go @@ -0,0 +1,83 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package constants is a package that provide global variable +package constants + +const ( + // OceanStorage is a storage type oceanStorage. + OceanStorage = "oceanStorage" + + // Object is a metrics type object. + Object = "object" + + // Performance is a metric type performance. + Performance = "performance" + + // Array is a collect type array. + Array = "array" + + // Controller is a collect type controller. + Controller = "controller" + + // StoragePool is a collect type storagePool. + StoragePool = "storagepool" + + // Lun is a collect type lun. + Lun = "lun" + + // Filesystem is a collect type filesystem. + Filesystem = "filesystem" + + // NasVolume is a volume type nas + NasVolume = "nas" + + // LunVolume is a volume type lun + LunVolume = "lun" + + // ResourceTypeLun is a resource type means lun + ResourceTypeLun = "11" + + // ResourceTypeFilesystem is a resource type means filesystem + ResourceTypeFilesystem = "40" + + // PersistentVolumeKind is a resource kind PersistentVolume + PersistentVolumeKind = "PersistentVolume" + + // PodKind is a resource kind Pod + PodKind = "Pod" + + // DefaultNameSpace default namespace + DefaultNameSpace = "default" + + // StorageNas is a storage volume type oceanstor-nas + StorageNas = "oceanstor-nas" + + // StorageSan is a storage volume type oceanstor-san + StorageSan = "oceanstor-san" + + // ObjectId is the field objectId + ObjectId = "ObjectId" + + // ObjectName is the field ObjectName + ObjectName = "ObjectName" + + // MinVersionSupportPost post request to get performance data is supported since version 6.1.2 + MinVersionSupportPost = "6.1.2" + + // StorageV6PointReleasePrefix defines the number of storage version which supported point version + StorageV6PointReleasePrefix = "6" +) diff --git a/provider/grpc/helper/client_helper.go b/provider/grpc/helper/client_helper.go new file mode 100644 index 0000000..a55d663 --- /dev/null +++ b/provider/grpc/helper/client_helper.go @@ -0,0 +1,73 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package helper is a package that helper function +package helper + +import ( + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + + sbcXuanwuClient "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned" + "github.com/huawei/csm/v2/config/client" + "github.com/huawei/csm/v2/utils/log" +) + +var clientSet = &ClientSet{} + +// ClientSet client set +// contains kubeClient and SbcClient +type ClientSet struct { + KubeClient *kubernetes.Clientset + SbcClient *sbcXuanwuClient.Clientset +} + +// InitClientSet init client set +func InitClientSet() error { + var kubeConfig *rest.Config + var err error + if client.GetKubeConfig() != "" { + kubeConfig, err = clientcmd.BuildConfigFromFlags("", client.GetKubeConfig()) + } else { + kubeConfig, err = rest.InClusterConfig() + } + + if err != nil { + log.Errorf("getting kubeConfig [%s] err: [%v]", client.GetKubeConfig(), err) + return err + } + + kubeClient, err := kubernetes.NewForConfig(kubeConfig) + if err != nil { + log.Errorf("init kube client failed, err: [%v]", err) + return err + } + + sbcClient, err := sbcXuanwuClient.NewForConfig(kubeConfig) + if err != nil { + log.Errorf("init sbc client failed, err: [%v]", err) + return err + } + + clientSet = &ClientSet{KubeClient: kubeClient, SbcClient: sbcClient} + return nil +} + +// GetClientSet get client set +func GetClientSet() *ClientSet { + return clientSet +} diff --git a/provider/grpc/helper/validator_helper.go b/provider/grpc/helper/validator_helper.go new file mode 100644 index 0000000..20c8f27 --- /dev/null +++ b/provider/grpc/helper/validator_helper.go @@ -0,0 +1,46 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package helper is a package that helper function +package helper + +// Validator are validate functions +type Validator[T any] struct { + functions []validateFunc[T] +} + +// validateFunc validate function format +type validateFunc[T any] func(t T) error + +// NewValidator get a instance of Validator +func NewValidator[T any](functions ...validateFunc[T]) *Validator[T] { + return &Validator[T]{ + functions: functions, + } +} + +// Validate validate object +func (receiver *Validator[T]) Validate(t T) error { + if len(receiver.functions) == 0 { + return nil + } + for _, function := range receiver.functions { + if err := function(t); err != nil { + return err + } + } + return nil +} diff --git a/provider/grpc/server/collector.go b/provider/grpc/server/collector.go new file mode 100644 index 0000000..b4a882d --- /dev/null +++ b/provider/grpc/server/collector.go @@ -0,0 +1,82 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package server is a package that implement grpc interface +package server + +import ( + "context" + "errors" + + "github.com/huawei/csm/v2/grpc/lib/go/cmi" + "github.com/huawei/csm/v2/provider/collect" + "github.com/huawei/csm/v2/provider/constants" + "github.com/huawei/csm/v2/provider/grpc/helper" + "github.com/huawei/csm/v2/utils/log" +) + +var collectValidator = helper.NewValidator[*cmi.CollectRequest](validateBackendName, validateCollectType, + validateMetricsType) + +// Collector This object implements the cmi.CollectorServer service. +type Collector struct{} + +// Collect This method is the entry point for collecting data. +// The purpose is to find an adapter and call its collect method. +func (c *Collector) Collect(ctx context.Context, request *cmi.CollectRequest) (*cmi.CollectResponse, error) { + log.AddContext(ctx).Infof("Start to collect, request: %v", request) + defer log.AddContext(ctx).Infof("Finish to collect, backend name %s", request.BackendName) + + if err := collectValidator.Validate(request); err != nil { + return nil, err + } + + collector, err := collect.GetCollector(request.GetMetricsType()) + if err != nil { + log.AddContext(ctx).Errorf("Get collector failed, error: %v", err) + return nil, err + } + log.AddContext(ctx).Infof("Get collector success, collector: %v", collector) + + return collector.Collect(ctx, request) +} + +// validateBackendName validate if the backend name is blank +func validateBackendName(request *cmi.CollectRequest) error { + if request.GetBackendName() == "" { + return errors.New("illegalArgumentError backend name is blank") + } + return nil +} + +// validateCollectType validate if the collect type is blank +func validateCollectType(request *cmi.CollectRequest) error { + if request.GetCollectType() == "" { + return errors.New("illegalArgumentError collect type is blank") + } + return nil +} + +// validateMetricsType validate if the metrics type is blank +func validateMetricsType(request *cmi.CollectRequest) error { + if request.GetMetricsType() == "" { + return errors.New("illegalArgumentError metrics type is blank") + } + if request.GetMetricsType() != constants.Object && request.GetMetricsType() != constants.Performance { + return errors.New("illegalArgumentError unsupported metrics type") + } + return nil +} diff --git a/provider/grpc/server/identity.go b/provider/grpc/server/identity.go new file mode 100644 index 0000000..5ddac03 --- /dev/null +++ b/provider/grpc/server/identity.go @@ -0,0 +1,63 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package server is a package that implement grpc interface +package server + +import ( + "context" + + cmiConfig "github.com/huawei/csm/v2/config/cmi" + "github.com/huawei/csm/v2/grpc/lib/go/cmi" + "github.com/huawei/csm/v2/utils/log" +) + +// Identity this object implements the cmi.IdentityServer service. +type Identity struct{} + +// Probe return running status. +func (i *Identity) Probe(ctx context.Context, request *cmi.ProbeRequest) (*cmi.ProbeResponse, error) { + log.AddContext(ctx).Debugln("Start probe") + return &cmi.ProbeResponse{}, nil +} + +// GetProvisionerInfo get provider info +func (i *Identity) GetProvisionerInfo(ctx context.Context, + request *cmi.GetProviderInfoRequest) (*cmi.GetProviderInfoResponse, error) { + log.AddContext(ctx).Infoln("Start get provider information") + + return &cmi.GetProviderInfoResponse{ + Provider: cmiConfig.GetProviderName(), + }, nil + +} + +// GetProviderCapabilities get provider capabilities +func (i *Identity) GetProviderCapabilities(ctx context.Context, + request *cmi.GetProviderCapabilitiesRequest) (*cmi.GetProviderCapabilitiesResponse, error) { + log.AddContext(ctx).Infoln("Start get provider Capabilities") + + return &cmi.GetProviderCapabilitiesResponse{ + Capabilities: []*cmi.ProviderCapability{ + { + Type: cmi.ProviderCapability_ProviderCapability_Label_Service, + }, + { + Type: cmi.ProviderCapability_ProviderCapability_Collect_Service, + }, + }, + }, nil +} diff --git a/provider/grpc/server/label.go b/provider/grpc/server/label.go new file mode 100644 index 0000000..213badf --- /dev/null +++ b/provider/grpc/server/label.go @@ -0,0 +1,102 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package server is a package that implement grpc interface +package server + +import ( + "context" + "errors" + + "github.com/huawei/csm/v2/grpc/lib/go/cmi" + "github.com/huawei/csm/v2/provider/constants" + "github.com/huawei/csm/v2/provider/grpc/helper" + "github.com/huawei/csm/v2/provider/label" + "github.com/huawei/csm/v2/utils/log" +) + +// createLabelValidator verify the parameters when creating labels, e.g. volumeId, clusterName... +var createLabelValidator = helper.NewValidator[label.Validator](validateVolumeId, validateLabelName, validateKind, + validateClusterName) + +// deleteLabelValidator verify the parameters when deleting labels, e.g. volumeId, labelName... +var deleteLabelValidator = helper.NewValidator[label.Validator](validateVolumeId, validateLabelName, validateKind) + +// Label implement cmi.LabelServiceServer +type Label struct{} + +// CreateLabel create label in storage +func (l *Label) CreateLabel(ctx context.Context, request *cmi.CreateLabelRequest) (*cmi.CreateLabelResponse, error) { + log.AddContext(ctx).Infof("Start to create label, request: %v", request) + + labelRequest := label.ConvertCreateRequest(request) + if err := createLabelValidator.Validate(labelRequest); err != nil { + return nil, err + } + + service := label.GetLabelService() + + return service.CreateLabel(ctx, request) +} + +// DeleteLabel delete label in storage +func (l *Label) DeleteLabel(ctx context.Context, request *cmi.DeleteLabelRequest) (*cmi.DeleteLabelResponse, error) { + log.AddContext(ctx).Infof("Start to delete label, request: %v", request) + + labelRequest := label.ConvertDeleteRequest(request) + if err := deleteLabelValidator.Validate(labelRequest); err != nil { + return nil, err + } + + service := label.GetLabelService() + return service.DeleteLabel(ctx, request) +} + +// validateLabelName validate if the label name is blank +func validateVolumeId(request label.Validator) error { + if request.VolumeId == "" { + return errors.New("illegalArgumentError volume id is blank") + } + return nil +} + +// validateLabelName validate if the label name is blank +func validateLabelName(request label.Validator) error { + if request.LabelName == "" { + return errors.New("illegalArgumentError label name is blank") + } + return nil +} + +// validateLabelName validate if the label name is blank +func validateKind(request label.Validator) error { + if request.Kind == "" { + return errors.New("illegalArgumentError kind is blank") + } + + if request.Kind != constants.PodKind && request.Kind != constants.PersistentVolumeKind { + return errors.New("illegalArgumentError unsupported kind") + } + return nil +} + +// validateLabelName validate if the label name is blank +func validateClusterName(request label.Validator) error { + if request.Kind == constants.PersistentVolumeKind && request.ClusterName == "" { + return errors.New("illegalArgumentError cluster name is blank") + } + return nil +} diff --git a/provider/label/label_helper.go b/provider/label/label_helper.go new file mode 100644 index 0000000..8a3de09 --- /dev/null +++ b/provider/label/label_helper.go @@ -0,0 +1,109 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package label is a package that provide operation storage label +package label + +import ( + "context" + "errors" + + "github.com/huawei/csm/v2/grpc/lib/go/cmi" + "github.com/huawei/csm/v2/provider/backend" + "github.com/huawei/csm/v2/provider/collect" + "github.com/huawei/csm/v2/provider/constants" + "github.com/huawei/csm/v2/provider/utils" + "github.com/huawei/csm/v2/storage/client/centralizedstorage" + "github.com/huawei/csm/v2/utils/log" +) + +// Validator label validator contains all fields to be verified +type Validator struct { + VolumeId string + LabelName string + Kind string + Namespace string + ClusterName string +} + +// OceanStorageLabelRequest operation ocean storage label +type OceanStorageLabelRequest struct { + resourceId string + resourceType string + client *centralizedstorage.CentralizedClient +} + +// ConvertCreateRequest convert CreateLabelRequest to LabelValidator +func ConvertCreateRequest(request *cmi.CreateLabelRequest) Validator { + return Validator{ + VolumeId: request.GetVolumeId(), + LabelName: request.GetLabelName(), + Kind: request.GetKind(), + Namespace: request.GetNamespace(), + ClusterName: request.GetClusterName(), + } +} + +// ConvertDeleteRequest convert DeleteLabelRequest to LabelValidator +func ConvertDeleteRequest(request *cmi.DeleteLabelRequest) Validator { + return Validator{ + VolumeId: request.GetVolumeId(), + LabelName: request.GetLabelName(), + Kind: request.GetKind(), + Namespace: request.GetNamespace(), + } +} + +// PrepareLabelRequest get client and resource object information +func PrepareLabelRequest(ctx context.Context, volumeId string) (OceanStorageLabelRequest, error) { + backendName, volumeName := utils.SplitVolumeId(volumeId) + clientInfo, err := collect.GetClient(ctx, backendName, backend.GetClientByBackendName) + if err != nil { + log.AddContext(ctx).Errorf("delete label get client failed, error: %v", err) + return OceanStorageLabelRequest{}, err + } + + client, ok := clientInfo.Client.(*centralizedstorage.CentralizedClient) + if !ok { + return OceanStorageLabelRequest{}, errors.New("convert storage client failed") + } + + resourceType := getResourceType(clientInfo.VolumeType) + resourceId, err := getResourceId(ctx, volumeName, clientInfo.VolumeType, client) + if err != nil { + log.AddContext(ctx).Errorf("delete label get resource id failed, error: %v", err) + return OceanStorageLabelRequest{}, err + } + + return OceanStorageLabelRequest{resourceId: resourceId, resourceType: resourceType, client: client}, nil +} + +func getResourceId(ctx context.Context, volumeName, volumeType string, + client *centralizedstorage.CentralizedClient) (string, error) { + + if volumeType == constants.NasVolume { + return client.GetFileSystemIdByName(ctx, volumeName) + } + + return client.GetLunIdByName(ctx, volumeName) +} + +func getResourceType(volumeType string) string { + if volumeType == constants.NasVolume { + return constants.ResourceTypeFilesystem + } + return constants.ResourceTypeLun +} diff --git a/provider/label/label_service.go b/provider/label/label_service.go new file mode 100644 index 0000000..6d7150b --- /dev/null +++ b/provider/label/label_service.go @@ -0,0 +1,176 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package label is a package that provide operation storage label +package label + +import ( + "context" + "errors" + "fmt" + + "github.com/huawei/csm/v2/grpc/lib/go/cmi" + "github.com/huawei/csm/v2/provider/constants" + "github.com/huawei/csm/v2/storage/client/centralizedstorage" + "github.com/huawei/csm/v2/utils/log" +) + +// createLabelFunctions create label functions +// including creating pod and pv +var createLabelFunctions = map[string]createLabelFunction{ + constants.PersistentVolumeKind: createPvLabel, + constants.PodKind: createPodLabel, +} + +// deleteLabelFunctions delete label functions +// including deleting pod and pv +var deleteLabelFunctions = map[string]deleteLabelFunction{ + constants.PersistentVolumeKind: deletePvLabel, + constants.PodKind: deletePodLabel, +} + +// createLabelFunction create label function format +type createLabelFunction func(ctx context.Context, resourceId, resourceType string, + client *centralizedstorage.CentralizedClient, request *cmi.CreateLabelRequest) (*cmi.CreateLabelResponse, error) + +// deleteLabelFunction delete label function format +type deleteLabelFunction func(ctx context.Context, resourceId, resourceType string, + client *centralizedstorage.CentralizedClient, request *cmi.DeleteLabelRequest) (*cmi.DeleteLabelResponse, error) + +// OceanStorageLabelService ocean storage label service +type OceanStorageLabelService struct{} + +// CreateLabel create label in ocean storage +func (o *OceanStorageLabelService) CreateLabel(ctx context.Context, + request *cmi.CreateLabelRequest) (*cmi.CreateLabelResponse, error) { + + param, err := PrepareLabelRequest(ctx, request.GetVolumeId()) + if err != nil { + log.AddContext(ctx).Errorf("create label failed, volumeId: %s, error: %v", request.GetVolumeId(), err) + return nil, err + } + + if param.resourceId == "" { + log.AddContext(ctx).Errorln("not found resource id, perhaps the volume does not exist, " + + "so returning failed") + return nil, errors.New("not found resource id") + } + + if request.GetNamespace() == "" { + request.Namespace = constants.DefaultNameSpace + } + + fun, ok := createLabelFunctions[request.Kind] + if !ok { + return nil, errors.New(fmt.Sprintf("illegalArgumentError unsupported resource kind [%s]", request.Kind)) + } + + return fun(ctx, param.resourceId, param.resourceType, param.client, request) +} + +// DeleteLabel delete label in ocean storage +func (o *OceanStorageLabelService) DeleteLabel(ctx context.Context, + request *cmi.DeleteLabelRequest) (*cmi.DeleteLabelResponse, error) { + + param, err := PrepareLabelRequest(ctx, request.GetVolumeId()) + if err != nil { + log.AddContext(ctx).Errorf("delete label failed, volumeId: %s, error: %v", request.GetVolumeId(), err) + return nil, err + } + + if param.resourceId == "" { + log.AddContext(ctx).Infoln("not found resource id, perhaps the volume does not exist, " + + "so returning success") + return &cmi.DeleteLabelResponse{}, nil + } + + if request.GetNamespace() == "" { + request.Namespace = constants.DefaultNameSpace + } + + fun, ok := deleteLabelFunctions[request.Kind] + if !ok { + return nil, errors.New(fmt.Sprintf("illegalArgumentError unsupported resource kind [%s]", request.Kind)) + } + + return fun(ctx, param.resourceId, param.resourceType, param.client, request) +} + +// createPvLabel create pv label +func createPvLabel(ctx context.Context, resourceId, resourceType string, client *centralizedstorage.CentralizedClient, + request *cmi.CreateLabelRequest) (*cmi.CreateLabelResponse, error) { + + var data = centralizedstorage.PvLabelRequest{ + ResourceId: resourceId, + ResourceType: resourceType, + PvName: request.GetLabelName(), + ClusterName: request.GetClusterName(), + } + _, err := client.CreatePvLabel(ctx, data) + if err != nil { + log.AddContext(ctx).Errorf("create pv label failed, volumeId: %s, error: %v", request.GetVolumeId(), err) + return nil, err + } + return &cmi.CreateLabelResponse{}, nil +} + +// createPodLabel create pod label +func createPodLabel(ctx context.Context, resourceId, resourceType string, client *centralizedstorage.CentralizedClient, + request *cmi.CreateLabelRequest) (*cmi.CreateLabelResponse, error) { + + var data = centralizedstorage.PodLabelRequest{ + ResourceId: resourceId, + ResourceType: resourceType, + PodName: request.GetLabelName(), + NameSpace: request.GetNamespace(), + } + _, err := client.CreatePodLabel(ctx, data) + if err != nil { + log.AddContext(ctx).Errorf("create pod label failed, volumeId: %s, error: %v", request.VolumeId, err) + return nil, err + } + return &cmi.CreateLabelResponse{}, nil +} + +// deletePvLabel delete pod label +func deletePvLabel(ctx context.Context, resourceId, resourceType string, client *centralizedstorage.CentralizedClient, + request *cmi.DeleteLabelRequest) (*cmi.DeleteLabelResponse, error) { + + _, err := client.DeletePvLabel(ctx, resourceId, resourceType) + if err != nil { + log.AddContext(ctx).Errorf("delete pv label failed, volumeId: %s, error: %v", request.VolumeId, err) + return nil, err + } + return &cmi.DeleteLabelResponse{}, nil +} + +// deletePodLabel delete pod label +func deletePodLabel(ctx context.Context, resourceId, resourceType string, client *centralizedstorage.CentralizedClient, + request *cmi.DeleteLabelRequest) (*cmi.DeleteLabelResponse, error) { + + var data = centralizedstorage.PodLabelRequest{ + ResourceId: resourceId, + ResourceType: resourceType, + PodName: request.GetLabelName(), + NameSpace: request.GetNamespace(), + } + _, err := client.DeletePodLabel(ctx, data) + if err != nil { + log.AddContext(ctx).Errorf("delete pod label failed, volumeId: %s, error: %v", request.VolumeId, err) + return nil, err + } + return &cmi.DeleteLabelResponse{}, nil +} diff --git a/provider/label/label_service_factory.go b/provider/label/label_service_factory.go new file mode 100644 index 0000000..56fb732 --- /dev/null +++ b/provider/label/label_service_factory.go @@ -0,0 +1,27 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package label is a package that provide operation storage label +package label + +import ( + "github.com/huawei/csm/v2/grpc/lib/go/cmi" +) + +// GetLabelService get label service +func GetLabelService() cmi.LabelServiceServer { + return &OceanStorageLabelService{} +} diff --git a/provider/label/label_service_test.go b/provider/label/label_service_test.go new file mode 100644 index 0000000..4e4b85f --- /dev/null +++ b/provider/label/label_service_test.go @@ -0,0 +1,169 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package label is a package that provide operation storage label +package label + +import ( + "context" + "testing" + + "github.com/agiledragon/gomonkey/v2" + + "github.com/huawei/csm/v2/grpc/lib/go/cmi" + "github.com/huawei/csm/v2/provider/constants" + "github.com/huawei/csm/v2/storage/client/centralizedstorage" +) + +func Test_CreatePvLabel_Success(t *testing.T) { + // arrange + resourceId, resourceType := "", "" + client := ¢ralizedstorage.CentralizedClient{} + request := &cmi.CreateLabelRequest{} + + // mock + methodFunc := gomonkey.ApplyMethodFunc(client, "CreatePvLabel", func(context.Context, + centralizedstorage.PvLabelRequest) (map[string]interface{}, error) { + return map[string]interface{}{}, nil + }) + defer methodFunc.Reset() + + // action + _, err := createPvLabel(context.Background(), resourceId, resourceType, client, request) + + // assert + if err != nil { + t.Errorf("createPvLabel() error = %v", err) + } +} + +func Test_CreatePodLabel_Success(t *testing.T) { + // arrange + resourceId, resourceType := "", "" + client := ¢ralizedstorage.CentralizedClient{} + request := &cmi.CreateLabelRequest{} + + // mock + methodFunc := gomonkey.ApplyMethodFunc(client, "CreatePodLabel", func(context.Context, + centralizedstorage.PodLabelRequest) (map[string]interface{}, error) { + return map[string]interface{}{}, nil + }) + defer methodFunc.Reset() + + // action + _, err := createPodLabel(context.Background(), resourceId, resourceType, client, request) + + // assert + if err != nil { + t.Errorf("createPodLabel() error = %v", err) + } +} + +func Test_DeletePvLabel_Success(t *testing.T) { + // arrange + resourceId, resourceType := "", "" + client := ¢ralizedstorage.CentralizedClient{} + request := &cmi.DeleteLabelRequest{} + + // mock + methodFunc := gomonkey.ApplyMethodFunc(client, "DeletePvLabel", func(context.Context, + string, string) (map[string]interface{}, error) { + return map[string]interface{}{}, nil + }) + defer methodFunc.Reset() + + // action + _, err := deletePvLabel(context.Background(), resourceId, resourceType, client, request) + + // assert + if err != nil { + t.Errorf("deletePvLabel() error = %v", err) + } +} + +func Test_DeletePodLabel_Success(t *testing.T) { + // arrange + resourceId, resourceType := "", "" + client := ¢ralizedstorage.CentralizedClient{} + request := &cmi.DeleteLabelRequest{} + + // mock + methodFunc := gomonkey.ApplyMethodFunc(client, "DeletePodLabel", func(context.Context, + centralizedstorage.PodLabelRequest) (map[string]interface{}, error) { + return map[string]interface{}{}, nil + }) + defer methodFunc.Reset() + + // action + _, err := deletePodLabel(context.Background(), resourceId, resourceType, client, request) + + // assert + if err != nil { + t.Errorf("deletePodLabel() error = %v", err) + } +} + +func Test_OceanStorageLabelService_CreateLabel_Success(t *testing.T) { + // arrange + request := &cmi.CreateLabelRequest{Kind: constants.PodKind} + service := &OceanStorageLabelService{} + + // mock + methodFunc := gomonkey. + ApplyFunc(PrepareLabelRequest, func(context.Context, string) (OceanStorageLabelRequest, + error) { + return OceanStorageLabelRequest{resourceId: "fakeResourceId"}, nil + }). + ApplyFunc(createPodLabel, func(context.Context, string, string, *centralizedstorage.CentralizedClient, + *cmi.CreateLabelRequest) (*cmi.CreateLabelResponse, error) { + return &cmi.CreateLabelResponse{}, nil + }) + defer methodFunc.Reset() + + // action + _, err := service.CreateLabel(context.Background(), request) + + // assert + if err != nil { + t.Errorf("CreateLabel() error = %v", err) + } +} + +func Test_OceanStorageLabelService_DeleteLabel_Success(t *testing.T) { + // arrange + request := &cmi.DeleteLabelRequest{Kind: constants.PodKind} + service := &OceanStorageLabelService{} + + // mock + methodFunc := gomonkey. + ApplyFunc(PrepareLabelRequest, func(context.Context, string) (OceanStorageLabelRequest, + error) { + return OceanStorageLabelRequest{}, nil + }). + ApplyFunc(deletePodLabel, func(context.Context, string, string, *centralizedstorage.CentralizedClient, + *cmi.DeleteLabelRequest) (*cmi.DeleteLabelResponse, error) { + return &cmi.DeleteLabelResponse{}, nil + }) + defer methodFunc.Reset() + + // action + _, err := service.DeleteLabel(context.Background(), request) + + // assert + if err != nil { + t.Errorf("CreateLabel() error = %v", err) + } +} diff --git a/provider/utils/utils.go b/provider/utils/utils.go new file mode 100644 index 0000000..5df05d7 --- /dev/null +++ b/provider/utils/utils.go @@ -0,0 +1,209 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package utils is a package that provide util functions +package utils + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "os" + "reflect" + "regexp" + "strconv" + "strings" + + coreV1 "k8s.io/api/core/v1" +) + +const ( + // BackendNameMaxLength is the max length of backend name + BackendNameMaxLength = 63 + + // BackendNameUidMaxLength is the max length of backend name uid + BackendNameUidMaxLength = 5 + + dns1123LabelFmt = "[a-z0-9]([-a-z0-9]*[a-z0-9])?" + dns1123SubdomainFmt = dns1123LabelFmt + "(" + dns1123LabelFmt + ")*" +) + +var dns1123SubdomainRegexp = regexp.MustCompile("^" + dns1123SubdomainFmt + "$") + +// MapToStructSlice map a struct to slice +func MapToStructSlice[I, O any](input I) ([]O, error) { + var targets []O + valueType := reflect.ValueOf(input) + if valueType.Kind() == reflect.Slice { + return MapToStruct[I, []O](input) + } + + target, err := MapToStruct[I, O](input) + if err != nil { + return []O{}, err + } + targets = append(targets, target) + return targets, nil +} + +// MapToStruct map to struct +func MapToStruct[I, O any](input I) (O, error) { + var o O + marshal, err := json.Marshal(input) + if err != nil { + return o, err + } + err = json.Unmarshal(marshal, &o) + if err != nil { + return o, err + } + return o, nil +} + +// MapStringToInt map a string slice to an int slice +func MapStringToInt(sources []string) []int { + var result []int + for _, source := range sources { + intVal, err := strconv.Atoi(source) + if err != nil { + continue + } + result = append(result, intVal) + } + return result +} + +// StructToMap convert struct to map. +// input t is a struct with metrics tag +// return map key is metrics tag. +// return map value is filed value. +func StructToMap[T any](t T) map[string]string { + mapping := map[string]string{} + filedType := reflect.TypeOf(t) + filedValue := reflect.ValueOf(t) + for i := 0; i < filedType.NumField(); i++ { + value, ok := filedValue.Field(i).Interface().(string) + if !ok || value == "" { + continue + } + mapping[filedType.Field(i).Tag.Get("metrics")] = value + } + return mapping +} + +// CleanupSocketFile clean socket file +func CleanupSocketFile(filePath string) error { + fileExists, err := DoesSocketExist(filePath) + if err != nil { + return err + } + + if fileExists { + if err := os.Remove(filePath); err != nil { + return fmt.Errorf("failed to remove stale file=%s with error: %+v", filePath, err) + } + } + return nil +} + +// DoesSocketExist determine if the socket file exists +func DoesSocketExist(socketPath string) (bool, error) { + if _, err := os.Lstat(socketPath); err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, fmt.Errorf("failed to lstat the socket %s with error: %+v", socketPath, err) + } + return true, nil +} + +// SplitVolumeId splits the volumeId to backend name and pv name +func SplitVolumeId(volumeId string) (string, string) { + splits := strings.SplitN(volumeId, ".", 2) + var backendName, pvName string + if len(splits) == 2 { + backendName, pvName = splits[0], splits[1] + } else { + backendName, pvName = splits[0], "" + } + return GetBackendName(backendName), pvName +} + +// GetBackendName format the name of backend +func GetBackendName(name string) string { + if IsDNSFormat(name) { + return name + } + return BuildBackendName(name) +} + +// IsDNSFormat Determine if the DNS format is met +func IsDNSFormat(source string) bool { + if len(source) > BackendNameMaxLength { + return false + } + return dns1123SubdomainRegexp.MatchString(source) +} + +// BuildBackendName build backend name +func BuildBackendName(name string) string { + nameLen := BackendNameMaxLength - BackendNameUidMaxLength - 1 + if len(name) > nameLen { + name = name[:nameLen] + } + hashCode := GenerateHashCode(name, BackendNameUidMaxLength) + mappingName := BackendNameMapping(name) + return fmt.Sprintf("%s-%s", mappingName, hashCode) +} + +// GenerateHashCode generate hash code +func GenerateHashCode(txt string, max int) string { + hashInstance := sha256.New() + hashInstance.Write([]byte(txt)) + sum := hashInstance.Sum(nil) + result := fmt.Sprintf("%x", sum) + if len(result) < max { + return result + } + return result[:max] +} + +// BackendNameMapping mapping backend name +func BackendNameMapping(name string) string { + removeUnderline := strings.ReplaceAll(name, "_", "-") + removePoint := strings.ReplaceAll(removeUnderline, ".", "-") + return strings.ToLower(removePoint) +} + +// CSIConfig holds the CSI config of backend resources +type CSIConfig struct { + Backends map[string]interface{} `json:"backends"` +} + +// ConvertConfigmapToMap formats configmap data to map struct +func ConvertConfigmapToMap(configmap *coreV1.ConfigMap) (map[string]interface{}, error) { + if configmap.Data == nil { + return nil, fmt.Errorf("configmap: [%s] the configmap.Data is nil", configmap.Name) + } + + var csiConfig CSIConfig + err := json.Unmarshal([]byte(configmap.Data["csi.json"]), &csiConfig) + if err != nil { + return nil, fmt.Errorf("json.Unmarshal configmap.Data[\"csi.json\"] failed. err is [%v]", err) + } + + return csiConfig.Backends, nil +} diff --git a/provider/utils/utils_test.go b/provider/utils/utils_test.go new file mode 100644 index 0000000..5eab964 --- /dev/null +++ b/provider/utils/utils_test.go @@ -0,0 +1,118 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "reflect" + "testing" +) + +type testStruct struct { + Id string `json:"Id" metrics:"Id"` + Name string `json:"Name" metrics:"Name"` +} + +func Test_MapStringToInt_Success(t *testing.T) { + // arrange + var want = []int{1, 2, 3} + var stringSlice = []string{"1", "2", "3"} + + // action + got := MapStringToInt(stringSlice) + + // assert + if !reflect.DeepEqual(got, want) { + t.Errorf("Test_MapStringToInt_Success() failed, want data = %v, but got = %v", want, got) + } +} + +func Test_MapToStruct_Success(t *testing.T) { + // arrange + input := map[string]string{ + "Id": "test-id", + "Name": "test-name", + } + + // action + _, err := MapToStruct[map[string]string, testStruct](input) + + // assert + if err != nil { + t.Errorf("Test_MapToStruct_Success() failed, error = %v", err) + } +} + +func Test_MapToStructSlice_Output_Is_Struct(t *testing.T) { + // arrange + input := map[string]interface{}{ + "Id": "test-id", + "Name": "test-name", + } + + // action + slice, err := MapToStructSlice[map[string]interface{}, testStruct](input) + + // assert + if err != nil { + t.Errorf("Test_MapToStructSlice_Output_Is_Struct() failed, error = %v", err) + } + + if len(slice) != 1 { + t.Errorf("Test_MapToStructSlice_Output_Is_Struct() failed, want len = %d, got = %d", 1, len(slice)) + } +} + +func Test_MapToStructSlice_Output_Is_StructSlice(t *testing.T) { + // arrange + single := map[string]interface{}{ + "Id": "test-id", + "Name": "test-name", + } + input := []map[string]interface{}{single, single} + + // action + slice, err := MapToStructSlice[[]map[string]interface{}, testStruct](input) + + // assert + if err != nil { + t.Errorf("Test_MapToStructSlice_Output_Is_StructSlice() failed, error = %v", err) + } + + if len(slice) != 2 { + t.Errorf("Test_MapToStructSlice_Output_Is_StructSlice() failed, want len = %d, got = %d", 1, len(slice)) + } +} + +func Test_StructToMap_Success(t *testing.T) { + // arrange + object := testStruct{ + Id: "1", + Name: "NAME-1", + } + want := map[string]string{ + "Id": "1", + "Name": "NAME-1", + } + + // action + got := StructToMap(object) + + // assert + if !reflect.DeepEqual(got, want) { + t.Errorf("Test_StructToMap_Success() = %v, want %v", got, want) + } +} diff --git a/server/.keepdir b/server/.keepdir new file mode 100644 index 0000000..e69de29 diff --git a/server/prometheus-exporter/clientset/clinet_set.go b/server/prometheus-exporter/clientset/clinet_set.go new file mode 100644 index 0000000..b20ed1c --- /dev/null +++ b/server/prometheus-exporter/clientset/clinet_set.go @@ -0,0 +1,129 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package clientset provide all client use by prometheus exporter +package clientset + +import ( + "sync" + + sbcXuanwuClient "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + + clientConfig "github.com/huawei/csm/v2/config/client" + storageGRPC "github.com/huawei/csm/v2/grpc/lib/go/cmi" + "github.com/huawei/csm/v2/utils/log" +) + +var once sync.Once + +// ClientsSet contains all clients needed by prometheus exporter +type ClientsSet struct { + // KubeClient get the Kubernetes client to get Kubernetes resource + KubeClient *kubernetes.Clientset + // SbcClient get backend client + SbcClient *sbcXuanwuClient.Clientset + // StorageGRPCClientSet From grpc get storage data client + StorageGRPCClientSet *storageGRPC.ClientSet + // InitError when init error this will set the reason + InitError error +} + +// exporterClientSet all client needed by prometheus exporter +var exporterClientSet *ClientsSet + +// GetExporterClientSet get the exporterClientSet +func GetExporterClientSet() *ClientsSet { + return exporterClientSet +} + +func initKubeClientAndSbcClient() { + if exporterClientSet == nil { + return + } + var kubeConfig *rest.Config + var err error + + if clientConfig.GetKubeConfig() != "" { + kubeConfig, err = clientcmd.BuildConfigFromFlags("", clientConfig.GetKubeConfig()) + } else { + kubeConfig, err = rest.InClusterConfig() + } + + if err != nil { + log.Errorf("getting kubeConfig [%s] err: [%v]", clientConfig.GetKubeConfig(), err) + exporterClientSet.InitError = err + return + } + + kubeClient, err := kubernetes.NewForConfig(kubeConfig) + if err != nil { + log.Errorf("init kube client failed, err: [%v]", err) + exporterClientSet.InitError = err + return + } + + sbcClient, err := sbcXuanwuClient.NewForConfig(kubeConfig) + if err != nil { + log.Errorf("init sbc client failed, err: [%v]", err) + exporterClientSet.InitError = err + return + } + exporterClientSet.KubeClient = kubeClient + exporterClientSet.SbcClient = sbcClient + return +} + +// InitExporterClientSet return exporterClientSet. if it not init we will do it +func InitExporterClientSet(grpcSock string) *ClientsSet { + if exporterClientSet == nil { + log.Infoln("start to initExporterClientSet") + once.Do(func() { + exporterClientSet = &ClientsSet{} + grpcClientSet, err := storageGRPC.GetClientSet(grpcSock) + if err != nil { + log.Errorln("can not get Client") + exporterClientSet.InitError = err + return + } + exporterClientSet.StorageGRPCClientSet = grpcClientSet + initKubeClientAndSbcClient() + }) + } else { + log.Debugln("initExporterClientSet is already call") + } + return exporterClientSet +} + +// DeleteExporterClientSet Release the exporterClientSet +func DeleteExporterClientSet() { + if exporterClientSet == nil { + return + } + if exporterClientSet.StorageGRPCClientSet == nil { + return + } + if exporterClientSet.StorageGRPCClientSet.Conn == nil { + return + } + err := exporterClientSet.StorageGRPCClientSet.Conn.Close() + if err != nil { + log.Errorln("can not delete storage grpc Client") + return + } +} diff --git a/server/prometheus-exporter/clientset/clinet_set_test.go b/server/prometheus-exporter/clientset/clinet_set_test.go new file mode 100644 index 0000000..36fb171 --- /dev/null +++ b/server/prometheus-exporter/clientset/clinet_set_test.go @@ -0,0 +1,70 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package clientset + +import ( + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "google.golang.org/grpc" + + storageGRPC "github.com/huawei/csm/v2/grpc/lib/go/cmi" +) + +func TestInitExporterClientSet(t *testing.T) { + // arrange + want := &ClientsSet{} + + // mock + patches := gomonkey. + ApplyFunc(storageGRPC.GetClientSet, func(address string) (*ClientsSet, error) { + return nil, nil + }).ApplyFunc(initKubeClientAndSbcClient, func() { return }) + defer patches.Reset() + + // action + got := InitExporterClientSet("fake_data") + + // assert + if !reflect.DeepEqual(got, want) { + t.Errorf("GetExporterClientSet() got = %v, want %v", got, want) + } +} + +func TestDeleteExporterClientSet(t *testing.T) { + // array + called := false + + // mock + patches := gomonkey. + ApplyGlobalVar(&exporterClientSet, &ClientsSet{ + StorageGRPCClientSet: &storageGRPC.ClientSet{Conn: &grpc.ClientConn{}}}). + ApplyMethodFunc(exporterClientSet.StorageGRPCClientSet.Conn, "Close", func() error { + called = true + return nil + }) + defer patches.Reset() + + // action + DeleteExporterClientSet() + + // assert + if called != true { + t.Errorf("DeleteExporterClientSet() called = %v, want true", called) + } +} diff --git a/server/prometheus-exporter/collector/array_collector.go b/server/prometheus-exporter/collector/array_collector.go new file mode 100644 index 0000000..94ea9d7 --- /dev/null +++ b/server/prometheus-exporter/collector/array_collector.go @@ -0,0 +1,94 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package collector includes all huawei storage collectors to gather and export huawei storage metrics. +package collector + +import ( + "fmt" + + "github.com/prometheus/client_golang/prometheus" + + metricsCache "github.com/huawei/csm/v2/server/prometheus-exporter/metricscache" +) + +func init() { + RegisterCollector("array", NewArrayCollector) +} + +const ( + productModeString = "productModeString" + productMode = "PRODUCTMODE" +) + +var arrayObjectMetricsLabelMap = map[string][]string{ + "basic_info": {"endpoint", "sn", "model", "version", "object"}, + "health_status": {"endpoint", "sn", "status", "object"}, + "running_status": {"endpoint", "sn", "status", "object"}, +} +var arrayObjectMetricsHelpMap = map[string]string{ + "basic_info": "Huawei Storage Array Basic Info", + "health_status": "Huawei Storage Array Health Status", + "running_status": "Huawei Storage Array Running Status", +} + +var arrayObjectMetricsParseMap = map[string]parseRelation{ + "basic_info": {"", parseStorageReturnZero}, + "health_status": {"HEALTHSTATUS", parseStorageData}, + "running_status": {"RUNNINGSTATUS", parseStorageData}, +} +var arrayObjectLabelParseMap = map[string]parseRelation{ + "endpoint": {"backendName", parseStorageData}, + "sn": {"ID", parseStorageData}, + "model": {"", parseArrayModel}, + "version": {"PRODUCTVERSION", parseStorageData}, + "status": {"", parseStorageStatus}, + "object": {"collectorName", parseStorageData}, +} + +func parseArrayModel(inDataKey, metricsName string, inData map[string]string) string { + var modelName string + if inData[productModeString] != "" { + modelName = inData[productModeString] + } else if inData[productMode] != "" { + modelName = StorageProductMode[inData[productMode]] + } + return modelName +} + +// ArrayCollector implements the prometheus.Collector interface and build storage array info +type ArrayCollector struct { + *BaseCollector +} + +func NewArrayCollector(backendName string, monitorType string, metricsIndicators []string, + metricsDataCache *metricsCache.MetricsDataCache) (prometheus.Collector, error) { + if monitorType == "object" { + return &ArrayCollector{ + BaseCollector: (&BaseCollector{}).SetBackendName(backendName). + SetMonitorType(monitorType). + SetCollectorName("array"). + SetMetricsHelpMap(arrayObjectMetricsHelpMap). + SetMetricsLabelMap(arrayObjectMetricsLabelMap). + SetLabelParseMap(arrayObjectLabelParseMap). + SetMetricsParseMap(arrayObjectMetricsParseMap). + SetMetricsDataCache(metricsDataCache). + SetMetrics(make(map[string]*prometheus.Desc)), + }, nil + } + + return nil, fmt.Errorf("can not create array collector, the monitor type is not object") +} diff --git a/server/prometheus-exporter/collector/array_collector_test.go b/server/prometheus-exporter/collector/array_collector_test.go new file mode 100644 index 0000000..3c7764e --- /dev/null +++ b/server/prometheus-exporter/collector/array_collector_test.go @@ -0,0 +1,89 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package collector + +import ( + "reflect" + "testing" + + "github.com/prometheus/client_golang/prometheus" +) + +func Test_parseArrayModel_GetProductModeString(t *testing.T) { + // arrange + mockInDataKey := "fake_key" + mockMetricsName := "fake_metrics" + mockInData := map[string]string{ + "productModeString": "fake_product_string", + "PRODUCTMODE": "fake_product_mode", + } + + // action + got := parseArrayModel(mockInDataKey, mockMetricsName, mockInData) + + // assert + if !reflect.DeepEqual(got, "fake_product_string") { + t.Errorf("parseStorageData() got = %v, want %v", got, "fake_data") + } +} + +func Test_parseArrayModel_GetProductMode(t *testing.T) { + // arrange + mockInDataKey := "fake_key" + mockMetricsName := "fake_metrics" + mockInData := map[string]string{ + "PRODUCTMODE": "61", + } + + // action + got := parseArrayModel(mockInDataKey, mockMetricsName, mockInData) + + // assert + if !reflect.DeepEqual(got, "6800 V3") { + t.Errorf("parseStorageData() got = %v, want %v", got, "fake_data") + } +} + +func TestNewArrayCollector(t *testing.T) { + // arrange + var wantCollector = &ArrayCollector{ + BaseCollector: &BaseCollector{ + backendName: "fake_backend", + monitorType: "object", + collectorName: "array", + metricsHelpMap: arrayObjectMetricsHelpMap, + metricsLabelMap: arrayObjectMetricsLabelMap, + labelParseMap: arrayObjectLabelParseMap, + metricsParseMap: arrayObjectMetricsParseMap, + metricsDataCache: nil, + metrics: make(map[string]*prometheus.Desc), + }, + } + + // action + got, err := NewArrayCollector("fake_backend", "object", []string{""}, + nil) + + // assert + if (err != nil) != false { + t.Errorf("NewArrayCollector() error = %v, wantErr %v", err, true) + return + } + if !reflect.DeepEqual(got, wantCollector) { + t.Errorf("NewArrayCollector() got = %v, want %v", got, nil) + } +} diff --git a/server/prometheus-exporter/collector/collector.go b/server/prometheus-exporter/collector/collector.go new file mode 100644 index 0000000..3b2175d --- /dev/null +++ b/server/prometheus-exporter/collector/collector.go @@ -0,0 +1,275 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package collector includes all huawei storage collectors to gather and export huawei storage metrics. +package collector + +import ( + "context" + "fmt" + "strconv" + "strings" + + "github.com/prometheus/client_golang/prometheus" + + metricsCache "github.com/huawei/csm/v2/server/prometheus-exporter/metricscache" + "github.com/huawei/csm/v2/utils/log" +) + +const MetricsNamespace = "huawei_storage" + +// a collector constructor +type collectorInitFunc = func(backendName, monitorType string, metricsIndicators []string, + metricsDataCache *metricsCache.MetricsDataCache) (prometheus.Collector, error) + +// factories are routing table with collector factory routing +// key is collector name +// value is a collector constructor +// e.g. +// |---------------|--------------------| +// | collectorName | collectorInitFunc | +// |---------------|--------------------| +// | array | NewArrayCollector | +// |---------------|--------------------| +var factories = make(map[string]collectorInitFunc) + +// RegisterCollector register a collector constructor to factories +func RegisterCollector(collectorName string, + factory collectorInitFunc) { + factories[collectorName] = factory +} + +// BaseCollector implements the prometheus.Collector interface. +type BaseCollector struct { + backendName string + monitorType string + collectorName string + metricsHelpMap map[string]string + metricsLabelMap map[string][]string + labelParseMap map[string]parseRelation + metricsParseMap map[string]parseRelation + metricsDataCache *metricsCache.MetricsDataCache + metrics map[string]*prometheus.Desc +} + +// Describe implements the prometheus.Collector interface. +// Use BuildDesc to build prometheus.Desc then send to prometheus. +func (baseCollector *BaseCollector) Describe(ch chan<- *prometheus.Desc) { + baseCollector.BuildDesc() + for _, i := range baseCollector.metrics { + ch <- i + } +} + +// NewPerformanceBaseCollector build a performance BaseCollector to other collector +func NewPerformanceBaseCollector(backendName, monitorType, collectorName string, metricsIndicators []string, + metricsDataCache *metricsCache.MetricsDataCache) (*BaseCollector, error) { + if len(metricsIndicators) == 0 || metricsIndicators[0] == "" { + return nil, fmt.Errorf("can not create [%s] collector, "+ + "the metricsIndicators is empty or error", collectorName) + } + metricsData := strings.Split(metricsIndicators[0], ",") + return (&BaseCollector{}).SetBackendName(backendName). + SetMonitorType(monitorType). + SetCollectorName(collectorName). + SetMetricsHelpMap(pickPerformanceParsMap[string](metricsData, performanceMetricsHelpMap)). + SetMetricsLabelMap(pickPerformanceParsMap[[]string](metricsData, performanceMetricsLabelMap)). + SetLabelParseMap(performanceLabelParseMap). + SetMetricsParseMap(pickPerformanceParsMap[parseRelation](metricsData, performanceMetricsParseMap)). + SetMetricsDataCache(metricsDataCache). + SetMetrics(make(map[string]*prometheus.Desc)), nil +} + +func (baseCollector *BaseCollector) setPrometheusMetric(ctx context.Context, ch chan<- prometheus.Metric, + metricsName string, detailData map[string]string) { + metricsParseRelation, ok := baseCollector.metricsParseMap[metricsName] + if !ok { + log.AddContext(ctx).Warningln("can not get the metricsParseRelation") + return + } + // parse metricsValue + metricsValue := metricsParseRelation.parseFunc( + metricsParseRelation.parseKey, metricsName, detailData) + metricsValueFloat, err := strconv.ParseFloat(metricsValue, bitSize) + if err != nil { + log.AddContext(ctx).Debugf("can not get the metricsValueFloat the metricsName is [%v]", metricsName) + return + } + // parse metricsLabel, from label key get label value + labelKeys, ok := baseCollector.metricsLabelMap[metricsName] + if !ok { + log.AddContext(ctx).Warningln("can not get the labelKeys") + return + } + labelValueSlice := parseLabelListToLabelValueSlice( + labelKeys, baseCollector.labelParseMap, metricsName, detailData) + if len(labelValueSlice) != len(labelKeys) { + log.AddContext(ctx).Warningln("can not get the labelValueSlice") + return + } + ch <- prometheus.MustNewConstMetric( + baseCollector.metrics[metricsName], + prometheus.GaugeValue, + metricsValueFloat, + labelValueSlice..., + ) +} + +// Collect implements the prometheus.Collector interface. +// Parse the data cached in MetricsDataCache and generate the return required by Prometheus. +// Use metricsParseMap to obtain the metric data parseRelation to parse the metric. +// Use metricsLabelMap to obtain the metric label parseRelation to parse tag information. +func (baseCollector *BaseCollector) Collect(ch chan<- prometheus.Metric) { + collectorCacheData := baseCollector.metricsDataCache.GetMetricsData(baseCollector.collectorName) + ctx := context.Background() + if collectorCacheData == nil || len(collectorCacheData.Details) == 0 { + log.AddContext(ctx).Warningln("can not get the collectorCacheData") + return + } + + for _, storageCollectDetail := range collectorCacheData.Details { + detailData := storageCollectDetail.Data + if len(detailData) == 0 { + log.AddContext(ctx).Warningln("can not get the detailData") + continue + } + + // Set backendName and collectorName to one cacheData, used by parseRelation.parseFunc + detailData["backendName"] = collectorCacheData.BackendName + detailData["collectorName"] = collectorCacheData.CollectType + for metricsName := range baseCollector.metrics { + baseCollector.setPrometheusMetric(ctx, ch, metricsName, detailData) + } + } +} + +// BuildDesc use BaseCollector.metricsDescMap create different Collector prometheus.Desc +func (baseCollector *BaseCollector) BuildDesc() { + if baseCollector.metrics == nil { + baseCollector.metrics = make(map[string]*prometheus.Desc) + } + for metricsName, helpInfo := range baseCollector.metricsHelpMap { + baseCollector.metrics[metricsName] = + prometheus.NewDesc( + prometheus.BuildFQName( + MetricsNamespace, baseCollector.collectorName, metricsName), + helpInfo, + baseCollector.metricsLabelMap[metricsName], + nil) + } +} + +// SetBackendName set backendName +func (baseCollector *BaseCollector) SetBackendName(backendName string) *BaseCollector { + baseCollector.backendName = backendName + return baseCollector +} + +// SetMonitorType set monitorType +func (baseCollector *BaseCollector) SetMonitorType(monitorType string) *BaseCollector { + baseCollector.monitorType = monitorType + return baseCollector +} + +// SetCollectorName set collectorName +func (baseCollector *BaseCollector) SetCollectorName(collectorName string) *BaseCollector { + baseCollector.collectorName = collectorName + return baseCollector +} + +// SetMetricsHelpMap set metricsHelpMap +func (baseCollector *BaseCollector) SetMetricsHelpMap(metricsHelpMap map[string]string) *BaseCollector { + baseCollector.metricsHelpMap = metricsHelpMap + return baseCollector +} + +// SetMetricsLabelMap set metricsLabelMap +func (baseCollector *BaseCollector) SetMetricsLabelMap(metricsLabelMap map[string][]string) *BaseCollector { + baseCollector.metricsLabelMap = metricsLabelMap + return baseCollector +} + +// SetLabelParseMap set labelParseMap +func (baseCollector *BaseCollector) SetLabelParseMap(labelParseMap map[string]parseRelation) *BaseCollector { + baseCollector.labelParseMap = labelParseMap + return baseCollector +} + +// SetMetricsParseMap set metricsParseMap +func (baseCollector *BaseCollector) SetMetricsParseMap(metricsParseMap map[string]parseRelation) *BaseCollector { + baseCollector.metricsParseMap = metricsParseMap + return baseCollector +} + +// SetMetricsDataCache set metricsDataCache +func (baseCollector *BaseCollector) SetMetricsDataCache( + metricsDataCache *metricsCache.MetricsDataCache) *BaseCollector { + baseCollector.metricsDataCache = metricsDataCache + return baseCollector +} + +// SetMetrics set metrics +func (baseCollector *BaseCollector) SetMetrics(metrics map[string]*prometheus.Desc) *BaseCollector { + baseCollector.metrics = metrics + return baseCollector +} + +// CollectorSet implements the prometheus.Collector interface. +// Save Multi BaseCollector +type CollectorSet struct { + collectors []prometheus.Collector +} + +// NewCollectorSet create all objects that need to be collected in this batch +func NewCollectorSet(ctx context.Context, params map[string][]string, backendName, monitorType string, + metricsDataCache *metricsCache.MetricsDataCache) (*CollectorSet, error) { + var collectors []prometheus.Collector + + for collectorName, metricsIndicators := range params { + collectorFunc, ok := factories[collectorName] + if !ok { + log.AddContext(ctx).Errorf("New collector error, the factories not have %s", collectorName) + continue + } + collector, err := collectorFunc(backendName, monitorType, metricsIndicators, metricsDataCache) + if err != nil { + log.AddContext(ctx).Errorf("New collector for %s, the monitorType : %s, error: %v", + collectorName, monitorType, err) + continue + } + collectors = append(collectors, collector) + } + + if len(collectors) == 0 { + return nil, fmt.Errorf("can not get the collector") + } + + return &CollectorSet{ + collectors: collectors, + }, nil +} + +func (collectorSet *CollectorSet) Describe(ch chan<- *prometheus.Desc) { + for _, collector := range collectorSet.collectors { + collector.Describe(ch) + } +} + +func (collectorSet *CollectorSet) Collect(ch chan<- prometheus.Metric) { + for _, collector := range collectorSet.collectors { + collector.Collect(ch) + } +} diff --git a/server/prometheus-exporter/collector/collector_parse_func.go b/server/prometheus-exporter/collector/collector_parse_func.go new file mode 100644 index 0000000..ae41990 --- /dev/null +++ b/server/prometheus-exporter/collector/collector_parse_func.go @@ -0,0 +1,108 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package collector includes all huawei storage collectors to gather and export huawei storage metrics. +// In this file we write shared parsing methods +package collector + +import "strconv" + +const ( + healthStatusToPrometheus = "health_status" + healthStatusFromStorage = "HEALTHSTATUS" + runningStatusToPrometheus = "running_status" + runningStatusFromStorage = "RUNNINGSTATUS" + sectorsTOGb = 1024 * 1024 * 2 + capacityKey = "CAPACITY" + allocCapacityKey = "ALLOCCAPACITY" + calculatePercentage = 100 + bitSize = 64 + precisionOfTwo = 2 + precisionOfFour = 4 +) + +type metricsParseFunc func(inDataKey, metricsName string, inData map[string]string) string + +type parseRelation struct { + parseKey string + parseFunc metricsParseFunc +} + +func parseStorageData(inDataKey, metricsName string, inData map[string]string) string { + if len(inData) == 0 { + return "" + } + return inData[inDataKey] +} + +func parseStorageReturnZero(inDataKey, metricsName string, inData map[string]string) string { + return "0.0" +} + +func parseStorageStatus(inDataKey, metricsName string, inData map[string]string) string { + if len(inData) == 0 { + return "" + } + if metricsName == healthStatusToPrometheus { + return StorageHealthStatus[inData[healthStatusFromStorage]] + } + if metricsName == runningStatusToPrometheus { + return StorageRunningStatus[inData[runningStatusFromStorage]] + } + return "" +} + +func parseStorageSectorsToGB(inDataKey, metricsName string, inData map[string]string) string { + if len(inData) == 0 { + return "" + } + sectorsData, err := strconv.ParseFloat(inData[inDataKey], bitSize) + if err != nil { + return "" + } + return strconv.FormatFloat(sectorsData/sectorsTOGb, 'f', precisionOfFour, bitSize) +} + +func parseLabelListToLabelValueSlice(labelKeys []string, + labelParseRelation map[string]parseRelation, metricsName string, inData map[string]string) []string { + var labelValueSlice []string + for _, labelName := range labelKeys { + parseRelationData, exist := labelParseRelation[labelName] + if !exist { + labelValueSlice = append(labelValueSlice, "") + continue + } + labelValue := parseRelationData.parseFunc( + parseRelationData.parseKey, metricsName, inData) + labelValueSlice = append(labelValueSlice, labelValue) + } + return labelValueSlice +} + +func parseCapacityUsage(inDataKey, metricsName string, inData map[string]string) string { + if len(inData) == 0 { + return "" + } + capacity, err := strconv.ParseFloat(inData[capacityKey], bitSize) + if err != nil || capacity == 0 { + return "" + } + allocCapacity, err := strconv.ParseFloat(inData[allocCapacityKey], bitSize) + if err != nil { + return "" + } + return strconv.FormatFloat(allocCapacity/capacity*calculatePercentage, 'f', precisionOfTwo, bitSize) +} diff --git a/server/prometheus-exporter/collector/collector_parse_func_test.go b/server/prometheus-exporter/collector/collector_parse_func_test.go new file mode 100644 index 0000000..ad6beb6 --- /dev/null +++ b/server/prometheus-exporter/collector/collector_parse_func_test.go @@ -0,0 +1,129 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package collector + +import ( + "reflect" + "testing" +) + +func Test_parseStorageData_GetDataSuccess(t *testing.T) { + // arrange + mockInDataKey := "fake_key" + mockMetricsName := "fake_metrics" + mockInData := map[string]string{ + "fake_key": "fake_data", + } + + // action + got := parseStorageData(mockInDataKey, mockMetricsName, mockInData) + + // assert + if !reflect.DeepEqual(got, "fake_data") { + t.Errorf("parseStorageData() got = %v, want %v", got, "fake_data") + } +} + +func Test_parseStorageData_GetDataEmpty(t *testing.T) { + // arrange + mockInDataKey := "fake_key1" + mockMetricsName := "fake_metrics" + mockInData := map[string]string{ + "fake_key": "fake_data", + } + + // action + got := parseStorageData(mockInDataKey, mockMetricsName, mockInData) + + // assert + if !reflect.DeepEqual(got, "") { + t.Errorf("parseStorageData() got = %v, want %v", got, "fake_data") + } +} + +func Test_parseStorageStatus_GetHealthStatus(t *testing.T) { + // arrange + mockInDataKey := "" + mockMetricsName := "health_status" + mockInData := map[string]string{ + "HEALTHSTATUS": "1", + } + + // action + got := parseStorageStatus(mockInDataKey, mockMetricsName, mockInData) + + // assert + if !reflect.DeepEqual(got, "Normal") { + t.Errorf("parseStorageStatus() got = %v, want %v", got, "fake_data") + } +} + +func Test_parseStorageStatus_GetRunningStatus(t *testing.T) { + // arrange + mockInDataKey := "" + mockMetricsName := "running_status" + mockInData := map[string]string{ + "RUNNINGSTATUS": "1", + } + + // action + got := parseStorageStatus(mockInDataKey, mockMetricsName, mockInData) + + // assert + if !reflect.DeepEqual(got, "Normal") { + t.Errorf("parseStorageStatus() got = %v, want %v", got, "fake_data") + } +} + +func Test_parseLabelListToLabelValueSlice_GetLabelValueSuccess(t *testing.T) { + // arrange + mockLabelKeys := []string{"fake_label_key1", "fake_label_key2"} + mockLabelParseRelation := map[string]parseRelation{ + "fake_label_key1": {"fake_key1", parseStorageData}, + "fake_label_key2": {"fake_key2", parseStorageData}, + } + mockInData := map[string]string{ + "fake_key1": "fake_data1", + "fake_key2": "fake_data2", + } + wantlabelValueSlice := []string{"fake_data1", "fake_data2"} + + // action + got := parseLabelListToLabelValueSlice(mockLabelKeys, mockLabelParseRelation, "", mockInData) + + // assert + if !reflect.DeepEqual(got, wantlabelValueSlice) { + t.Errorf("parseLabelListToLabelValueSlice() got = %v, want %v", + got, wantlabelValueSlice) + } +} + +func Test_parseStorageSectorsToGB(t *testing.T) { + // arrange + mockInDataKey := "fake_key" + mockInData := map[string]string{ + "fake_key": "209715200", + } + + // action + got := parseStorageSectorsToGB(mockInDataKey, "", mockInData) + + // assert + if !reflect.DeepEqual(got, "100.0000") { + t.Errorf("parseStorageStatus() got = %v, want %v", got, "100.0000") + } +} diff --git a/server/prometheus-exporter/collector/collector_test.go b/server/prometheus-exporter/collector/collector_test.go new file mode 100644 index 0000000..8682011 --- /dev/null +++ b/server/prometheus-exporter/collector/collector_test.go @@ -0,0 +1,393 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package collector + +import ( + "context" + "reflect" + "testing" + + "github.com/prometheus/client_golang/prometheus" + + storageGRPC "github.com/huawei/csm/v2/grpc/lib/go/cmi" + metricsCache "github.com/huawei/csm/v2/server/prometheus-exporter/metricscache" +) + +func TestRegisterCollector(t *testing.T) { + // arrange + var CollectorName = "fake_collector" + var NewCollectorFunc = func(backendName, monitorType string, metricsIndicators []string, + pMetricsDataCache *metricsCache.MetricsDataCache) (prometheus.Collector, error) { + return nil, nil + } + + // mock + RegisterCollector(CollectorName, NewCollectorFunc) + + // assert + newCollectorFunc, ok := factories[CollectorName] + if !ok { + t.Errorf("RegisterCollector() want func, but got = %v", nil) + } + + if reflect.ValueOf(NewCollectorFunc).Pointer() != + reflect.ValueOf(newCollectorFunc).Pointer() { + t.Error("RegisterCollector get func error") + } +} + +func TestBaseCollector_BuildDesc(t *testing.T) { + // arrange + var mockMetricsLabelMap = map[string][]string{ + "fake_key1": {"fake_label1", "fake_label2"}, + "fake_key2": {"fake_label2", "fake_label3"}} + var mockMetricsHelpMap = map[string]string{ + "fake_key1": "fake help info1", + "fake_key2": "fake help info2", + } + var mockCollector = BaseCollector{ + backendName: "fake_name", + monitorType: "fake_monitortype", + collectorName: "fake_name", + metricsHelpMap: mockMetricsHelpMap, + metricsLabelMap: mockMetricsLabelMap, + metrics: make(map[string]*prometheus.Desc), + } + var mockMetrics = map[string]*prometheus.Desc{ + "fake_key1": prometheus.NewDesc( + prometheus.BuildFQName( + MetricsNamespace, mockCollector.collectorName, "fake_key1"), + "fake help info1", + mockCollector.metricsLabelMap["fake_key1"], + nil), + "fake_key2": prometheus.NewDesc( + prometheus.BuildFQName( + MetricsNamespace, mockCollector.collectorName, "fake_key2"), + "fake help info2", + mockCollector.metricsLabelMap["fake_key2"], + nil), + } + + // action + mockCollector.BuildDesc() + + // assert + if !reflect.DeepEqual(mockMetrics, mockCollector.metrics) { + t.Errorf("BuildDesc() want = %v, but got = %v", mockMetrics, mockCollector) + } +} + +func TestNewCollectorSet_FactoriesNotHaveCollector(t *testing.T) { + // arrange + var ( + ctx = context.TODO() + CollectorName = "fake_collector" + NewCollectorFunc = func(backendName, monitorType string, metricsIndicators []string, + pMetricsDataCache *metricsCache.MetricsDataCache) (prometheus.Collector, error) { + return nil, nil + } + ) + factories[CollectorName] = NewCollectorFunc + var wantCollector *CollectorSet = nil + + // action + got, err := NewCollectorSet(ctx, nil, "", "", + nil) + + // assert + if (err != nil) != true { + t.Errorf("NewCollectorSet() error = %v, wantErr %v", err, true) + return + } + if !reflect.DeepEqual(got, wantCollector) { + t.Errorf("NewCollectorSet() got = %v, want %v", got, nil) + } +} + +func TestNewCollectorSet_GetCollectorSetSuccess(t *testing.T) { + // arrange + var ( + ctx = context.TODO() + CollectorName = "fake_collector" + NewCollectorFunc = func(backendName, monitorType string, metricsIndicators []string, + pMetricsDataCache *metricsCache.MetricsDataCache) (prometheus.Collector, error) { + return nil, nil + } + ) + factories[CollectorName] = NewCollectorFunc + mockparams := map[string][]string{"fake_collector": {"fake_data"}} + var wantCollector = &CollectorSet{ + collectors: []prometheus.Collector{nil}, + } + + // action + got, err := NewCollectorSet(ctx, mockparams, + "fake_backend_name", "fake_type", + nil) + + // assert + if (err != nil) != false { + t.Errorf("NewCollectorSet() error = %v, wantErr %v", err, true) + return + } + if !reflect.DeepEqual(got, wantCollector) { + t.Errorf("NewCollectorSet() got = %v, want %v", got, nil) + } +} + +func buildMockBaseCollector() *BaseCollector { + var mockMetricsLabelMap = map[string][]string{"fake_key1": {"fake_label1", "fake_label2", "fake_label3"}} + var mockMetricsHelpMap = map[string]string{"fake_key1": "fake help info1"} + var mockLabelParseMap = map[string]parseRelation{ + "fake_label1": {"backendName", parseStorageData}, + "fake_label2": {"collectorName", parseStorageData}, + "fake_label3": {"fake_data", parseStorageData}, + } + var mockMetricsParseMap = map[string]parseRelation{ + "fake_key1": {"", parseStorageReturnZero}, + } + var mockMetrics = map[string]*prometheus.Desc{ + "fake_key1": prometheus.NewDesc( + prometheus.BuildFQName( + MetricsNamespace, "fake_name", "fake_key1"), + "fake help info1", + mockMetricsLabelMap["fake_key1"], + nil), + } + mockCollectDetail := storageGRPC.CollectDetail{ + Data: map[string]string{"fake_data": "test_data"}, + } + mockCollectResponse := storageGRPC.CollectResponse{ + BackendName: "fake_backend_name", + CollectType: "fake_collector_name", + MetricsType: "fake_type", + Details: []*storageGRPC.CollectDetail{&mockCollectDetail}, + } + mockMetricsData := metricsCache.BaseMetricsData{ + MetricsType: "fake_collector_name", + MetricsDataResponse: &mockCollectResponse, + } + mockMetricsDataCache := &metricsCache.MetricsDataCache{ + BackendName: "fake_name", + CacheDataMap: map[string]metricsCache.MetricsData{ + "fake_collector_name": &mockMetricsData}, + } + var mockCollector = BaseCollector{ + backendName: "fake_backend_name", + monitorType: "fake_monitor_type", + collectorName: "fake_collector_name", + metricsHelpMap: mockMetricsHelpMap, + metricsLabelMap: mockMetricsLabelMap, + labelParseMap: mockLabelParseMap, + metricsParseMap: mockMetricsParseMap, + metricsDataCache: mockMetricsDataCache, + metrics: mockMetrics, + } + return &mockCollector +} + +func TestBaseCollector_Collect_ParseSuccess(t *testing.T) { + // arrange + mockCollector := buildMockBaseCollector() + mockMetricChan := make(chan prometheus.Metric, 2) + mockLabelValueSlice := []string{ + "fake_backend_name", "fake_collector_name", "test_data"} + wantPrometheusMetric := prometheus.MustNewConstMetric( + mockCollector.metrics["fake_key1"], + prometheus.GaugeValue, + 0.0, + mockLabelValueSlice..., + ) + + // action + mockCollector.Collect(mockMetricChan) + defer close(mockMetricChan) + + // assert + got, ok := <-mockMetricChan + if !ok { + t.Errorf("Collect() got = %v", got) + } + if !reflect.DeepEqual(got, wantPrometheusMetric) { + t.Errorf("Collect() got = %v, want %v", got, wantPrometheusMetric) + } +} + +func TestBaseCollector_SetBackendName(t *testing.T) { + // arrange + mockBackendName := "fake_backend_name" + mockCollector := &BaseCollector{} + + // action + mockCollector.SetBackendName(mockBackendName) + + // assert + if !reflect.DeepEqual(mockCollector.backendName, mockBackendName) { + t.Errorf("SetBackendName() got = %v, want %v", mockCollector.backendName, mockBackendName) + } +} + +func TestBaseCollector_SetMonitorType(t *testing.T) { + // arrange + mockMonitorType := "fake_monitor_type" + mockCollector := &BaseCollector{} + + // action + mockCollector.SetMonitorType(mockMonitorType) + + // assert + if !reflect.DeepEqual(mockCollector.monitorType, mockMonitorType) { + t.Errorf("SetBackendName() got = %v, want %v", mockCollector.monitorType, mockMonitorType) + } +} + +func TestBaseCollector_SetCollectorName(t *testing.T) { + // arrange + mockCollectorName := "fake_collector_name" + mockCollector := &BaseCollector{} + + // action + mockCollector.SetCollectorName(mockCollectorName) + + // assert + if !reflect.DeepEqual(mockCollector.collectorName, mockCollectorName) { + t.Errorf("SetBackendName() got = %v, want %v", mockCollector.collectorName, mockCollectorName) + } +} + +func TestBaseCollector_SetMetricsHelpMap(t *testing.T) { + // arrange + metricsHelpMap := map[string]string{ + "fake_key": "fake_help_info", + } + mockCollector := &BaseCollector{} + want := map[string][]string{ + "fake_key": {"fake_help_info"}, + } + equalMaps := func(a, b map[string][]string) bool { + if len(a) != len(b) { + return false + } + for k, v := range a { + if vb, ok := b[k]; !ok || !reflect.DeepEqual(vb, v) { + return false + } + } + return true + } + + // action + mockCollector.SetMetricsHelpMap(metricsHelpMap) + + // assert + if equalMaps(mockCollector.metricsLabelMap, want) { + t.Errorf("SetBackendName() got = %v, want %v", mockCollector.metricsHelpMap, want) + } +} + +func TestBaseCollector_SetMetricsLabelMap(t *testing.T) { + // arrange + metricsLabelMap := map[string][]string{ + "fake_key": {"fake_label1", "fake_label2"}, + } + mockCollector := &BaseCollector{} + + // action + mockCollector.SetMetricsLabelMap(metricsLabelMap) + + // assert + if !reflect.DeepEqual(mockCollector.metricsLabelMap, metricsLabelMap) { + t.Errorf("SetBackendName() got = %v, want %v", mockCollector.metricsLabelMap, metricsLabelMap) + } +} + +func TestBaseCollector_SetLabelParseMap(t *testing.T) { + // arrange + labelParseMap := map[string]parseRelation{ + "fake_label": {"fake_key", parseStorageData}, + } + mockCollector := &BaseCollector{} + + // action + mockCollector.SetLabelParseMap(labelParseMap) + + // assert + if !reflect.DeepEqual(mockCollector.labelParseMap, labelParseMap) { + t.Errorf("SetBackendName() got = %v, want %v", mockCollector.labelParseMap, labelParseMap) + } +} + +func TestBaseCollector_SetMetricsParseMap(t *testing.T) { + // arrange + metricsParseMap := map[string]parseRelation{ + "fake_key": {"", parseStorageReturnZero}, + } + mockCollector := &BaseCollector{} + + // action + mockCollector.SetMetricsParseMap(metricsParseMap) + + // assert + if !reflect.DeepEqual(mockCollector.metricsParseMap, metricsParseMap) { + t.Errorf("SetBackendName() got = %v, want %v", mockCollector.metricsParseMap, metricsParseMap) + } +} + +func TestBaseCollector_SetMetricsDataCache(t *testing.T) { + // arrange + metricsDataCache := &metricsCache.MetricsDataCache{ + BackendName: "fake_backend_name", + CacheDataMap: map[string]metricsCache.MetricsData{}} + mockCollector := &BaseCollector{} + + // action + mockCollector.SetMetricsDataCache(metricsDataCache) + + // assert + if !reflect.DeepEqual(mockCollector.metricsDataCache, metricsDataCache) { + t.Errorf("SetBackendName() got = %v, want %v", mockCollector.metricsDataCache, metricsDataCache) + } +} + +func TestBaseCollector_SetMetrics(t *testing.T) { + // arrange + metrics := make(map[string]*prometheus.Desc) + mockCollector := &BaseCollector{} + + // action + mockCollector.SetMetrics(metrics) + + // assert + if !reflect.DeepEqual(mockCollector.metrics, metrics) { + t.Errorf("SetBackendName() got = %v, want %v", mockCollector.metrics, metrics) + } +} + +func TestNewPerformanceBaseCollector(t *testing.T) { + // arrange + var mockMetricsIndicators []string + + // action + _, err := NewPerformanceBaseCollector("fake_backend", "performance", + "fake_collector", mockMetricsIndicators, nil) + + // assert + if err == nil { + t.Errorf("NewPerformanceBaseCollector() error = %v, wantErr %v", err, true) + return + } +} diff --git a/server/prometheus-exporter/collector/controller_collector.go b/server/prometheus-exporter/collector/controller_collector.go new file mode 100644 index 0000000..fadda75 --- /dev/null +++ b/server/prometheus-exporter/collector/controller_collector.go @@ -0,0 +1,91 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package collector includes all huawei storage collectors to gather and export huawei storage metrics. +package collector + +import ( + "fmt" + + "github.com/prometheus/client_golang/prometheus" + + metricsCache "github.com/huawei/csm/v2/server/prometheus-exporter/metricscache" +) + +func init() { + RegisterCollector("controller", NewControllerCollector) +} + +var controllerObjectMetricsLabelMap = map[string][]string{ + "cpu_usage": {"endpoint", "id", "name", "object"}, + "memory_usage": {"endpoint", "id", "name", "object"}, + "health_status": {"endpoint", "id", "status", "name", "object"}, + "running_status": {"endpoint", "id", "status", "name", "object"}, +} +var controllerObjectMetricsHelpMap = map[string]string{ + "cpu_usage": "CPU utilization(%)", + "memory_usage": "Memory utilization(%)", + "health_status": "Health Status", + "running_status": "Running Status", +} + +var controllerObjectMetricsParseMap = map[string]parseRelation{ + "cpu_usage": {"CPUUSAGE", parseStorageData}, + "memory_usage": {"MEMORYUSAGE", parseStorageData}, + "health_status": {"HEALTHSTATUS", parseStorageData}, + "running_status": {"RUNNINGSTATUS", parseStorageData}, +} +var controllerObjectLabelParseMap = map[string]parseRelation{ + "endpoint": {"backendName", parseStorageData}, + "id": {"ID", parseStorageData}, + "name": {"NAME", parseStorageData}, + "status": {"", parseStorageStatus}, + "object": {"collectorName", parseStorageData}, +} + +// ControllerCollector implements the prometheus.Collector interface and build storage Controller info +type ControllerCollector struct { + *BaseCollector +} + +func NewControllerCollector(backendName, monitorType string, metricsIndicators []string, + metricsDataCache *metricsCache.MetricsDataCache) (prometheus.Collector, error) { + if monitorType == "object" { + return &ControllerCollector{ + BaseCollector: (&BaseCollector{}).SetBackendName(backendName). + SetMonitorType(monitorType). + SetCollectorName("controller"). + SetMetricsHelpMap(controllerObjectMetricsHelpMap). + SetMetricsLabelMap(controllerObjectMetricsLabelMap). + SetLabelParseMap(controllerObjectLabelParseMap). + SetMetricsParseMap(controllerObjectMetricsParseMap). + SetMetricsDataCache(metricsDataCache). + SetMetrics(make(map[string]*prometheus.Desc)), + }, nil + } else if monitorType == "performance" { + performanceBaseCollector, err := NewPerformanceBaseCollector( + backendName, monitorType, "controller", metricsIndicators, metricsDataCache) + if err != nil { + return nil, err + } + return &ControllerCollector{ + BaseCollector: performanceBaseCollector, + }, nil + } + + return nil, fmt.Errorf("can not create controller collector, " + + "the monitor type not in object or performance") +} diff --git a/server/prometheus-exporter/collector/controller_collector_test.go b/server/prometheus-exporter/collector/controller_collector_test.go new file mode 100644 index 0000000..7fe14af --- /dev/null +++ b/server/prometheus-exporter/collector/controller_collector_test.go @@ -0,0 +1,69 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package collector + +import ( + "reflect" + "testing" + + "github.com/prometheus/client_golang/prometheus" +) + +func TestNewControllerCollector_GetObjectCollector(t *testing.T) { + // arrange + var wantCollector = &ControllerCollector{ + BaseCollector: &BaseCollector{ + backendName: "fake_backend", + monitorType: "object", + collectorName: "controller", + metricsHelpMap: controllerObjectMetricsHelpMap, + metricsLabelMap: controllerObjectMetricsLabelMap, + labelParseMap: controllerObjectLabelParseMap, + metricsParseMap: controllerObjectMetricsParseMap, + metricsDataCache: nil, + metrics: make(map[string]*prometheus.Desc), + }, + } + + // action + got, err := NewControllerCollector("fake_backend", "object", []string{""}, + nil) + + // assert + if err != nil { + t.Errorf("NewControllerCollector() error = %v", err) + return + } + if !reflect.DeepEqual(got, wantCollector) { + t.Errorf("NewControllerCollector() got = %v, want %v", got, nil) + } +} + +func TestNewControllerCollector_GetPerformanceCollector(t *testing.T) { + // arrange + var mockMetricsIndicators []string + + // action + _, err := NewControllerCollector("fake_backend", "performance", mockMetricsIndicators, + nil) + + // assert + if err == nil { + t.Errorf("NewControllerCollector() error = %v, wantErr %v", err, true) + return + } +} diff --git a/server/prometheus-exporter/collector/filesystem_collector.go b/server/prometheus-exporter/collector/filesystem_collector.go new file mode 100644 index 0000000..246e04a --- /dev/null +++ b/server/prometheus-exporter/collector/filesystem_collector.go @@ -0,0 +1,96 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package collector includes all huawei storage collectors to gather and export huawei storage metrics. +package collector + +import ( + "fmt" + + "github.com/prometheus/client_golang/prometheus" + + metricsCache "github.com/huawei/csm/v2/server/prometheus-exporter/metricscache" +) + +func init() { + RegisterCollector("filesystem", NewFilesystemCollector) +} + +var filesystemBuildMap = map[string]collectorInitFunc{ + "object": buildObjectFilesystemCollector, + "performance": buildPerformanceFilesystemCollector, +} + +var filesystemObjectMetricsLabelMap = map[string][]string{ + "capacity": {"endpoint", "id", "name", "object"}, + "capacity_usage": {"endpoint", "id", "name", "object"}, +} +var filesystemObjectMetricsHelpMap = map[string]string{ + "capacity": "filesystem capacity(GB)", + "capacity_usage": "filesystem capacity usage(%)", +} + +var filesystemObjectMetricsParseMap = map[string]parseRelation{ + "capacity": {"CAPACITY", parseStorageSectorsToGB}, + "capacity_usage": {"", parseCapacityUsage}, +} +var filesystemObjectLabelParseMap = map[string]parseRelation{ + "endpoint": {"backendName", parseStorageData}, + "id": {"ID", parseStorageData}, + "name": {"NAME", parseStorageData}, + "object": {"collectorName", parseStorageData}, +} + +type FilesystemCollector struct { + *BaseCollector +} + +func buildObjectFilesystemCollector(backendName string, monitorType string, metricsIndicators []string, + metricsDataCache *metricsCache.MetricsDataCache) (prometheus.Collector, error) { + return &FilesystemCollector{ + BaseCollector: (&BaseCollector{}).SetBackendName(backendName). + SetMonitorType(monitorType). + SetCollectorName("filesystem"). + SetMetricsHelpMap(filesystemObjectMetricsHelpMap). + SetMetricsLabelMap(filesystemObjectMetricsLabelMap). + SetLabelParseMap(filesystemObjectLabelParseMap). + SetMetricsParseMap(filesystemObjectMetricsParseMap). + SetMetricsDataCache(metricsDataCache). + SetMetrics(make(map[string]*prometheus.Desc)), + }, nil +} + +func buildPerformanceFilesystemCollector(backendName string, monitorType string, metricsIndicators []string, + metricsDataCache *metricsCache.MetricsDataCache) (prometheus.Collector, error) { + performanceBaseCollector, err := NewPerformanceBaseCollector( + backendName, monitorType, "filesystem", metricsIndicators, metricsDataCache) + if err != nil { + return nil, err + } + return &FilesystemCollector{ + BaseCollector: performanceBaseCollector, + }, nil +} + +func NewFilesystemCollector(backendName string, monitorType string, metricsIndicators []string, + metricsDataCache *metricsCache.MetricsDataCache) (prometheus.Collector, error) { + buildFunc, ok := filesystemBuildMap[monitorType] + if !ok { + return nil, fmt.Errorf("can not create filesystem collector, " + + "the monitor type not in object or performance") + } + return buildFunc(backendName, monitorType, metricsIndicators, metricsDataCache) +} diff --git a/server/prometheus-exporter/collector/filesystem_collector_test.go b/server/prometheus-exporter/collector/filesystem_collector_test.go new file mode 100644 index 0000000..af0f84f --- /dev/null +++ b/server/prometheus-exporter/collector/filesystem_collector_test.go @@ -0,0 +1,54 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package collector + +import ( + "reflect" + "testing" + + "github.com/prometheus/client_golang/prometheus" +) + +func Test_buildObjectFilesystemCollector(t *testing.T) { + // arrange + var wantCollector = &FilesystemCollector{ + BaseCollector: &BaseCollector{ + backendName: "fake_backend", + monitorType: "object", + collectorName: "filesystem", + metricsHelpMap: filesystemObjectMetricsHelpMap, + metricsLabelMap: filesystemObjectMetricsLabelMap, + labelParseMap: filesystemObjectLabelParseMap, + metricsParseMap: filesystemObjectMetricsParseMap, + metricsDataCache: nil, + metrics: make(map[string]*prometheus.Desc), + }, + } + + // action + got, err := NewFilesystemCollector("fake_backend", "object", []string{""}, + nil) + + // assert + if err != nil { + t.Errorf("NewFilesystemCollector() error = %v", err) + return + } + if !reflect.DeepEqual(got, wantCollector) { + t.Errorf("NewFilesystemCollector() got = %v, want %v", got, nil) + } +} diff --git a/server/prometheus-exporter/collector/lun_collector.go b/server/prometheus-exporter/collector/lun_collector.go new file mode 100644 index 0000000..5f40756 --- /dev/null +++ b/server/prometheus-exporter/collector/lun_collector.go @@ -0,0 +1,96 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package collector includes all huawei storage collectors to gather and export huawei storage metrics. +package collector + +import ( + "fmt" + + "github.com/prometheus/client_golang/prometheus" + + metricsCache "github.com/huawei/csm/v2/server/prometheus-exporter/metricscache" +) + +func init() { + RegisterCollector("lun", NewLunCollector) +} + +var lunBuildMap = map[string]collectorInitFunc{ + "object": buildObjectLunCollector, + "performance": buildPerformanceLunCollector, +} + +var lunObjectMetricsLabelMap = map[string][]string{ + "capacity": {"endpoint", "id", "name", "object"}, + "capacity_usage": {"endpoint", "id", "name", "object"}, +} +var lunObjectMetricsHelpMap = map[string]string{ + "capacity": "LUN capacity(GB)", + "capacity_usage": "LUN capacity usage(%)", +} + +var lunObjectMetricsParseMap = map[string]parseRelation{ + "capacity": {"CAPACITY", parseStorageSectorsToGB}, + "capacity_usage": {"", parseCapacityUsage}, +} +var lunObjectLabelParseMap = map[string]parseRelation{ + "endpoint": {"backendName", parseStorageData}, + "id": {"ID", parseStorageData}, + "name": {"NAME", parseStorageData}, + "object": {"collectorName", parseStorageData}, +} + +type LunCollector struct { + *BaseCollector +} + +func buildObjectLunCollector(backendName string, monitorType string, metricsIndicators []string, + metricsDataCache *metricsCache.MetricsDataCache) (prometheus.Collector, error) { + return &LunCollector{ + BaseCollector: (&BaseCollector{}).SetBackendName(backendName). + SetMonitorType(monitorType). + SetCollectorName("lun"). + SetMetricsHelpMap(lunObjectMetricsHelpMap). + SetMetricsLabelMap(lunObjectMetricsLabelMap). + SetLabelParseMap(lunObjectLabelParseMap). + SetMetricsParseMap(lunObjectMetricsParseMap). + SetMetricsDataCache(metricsDataCache). + SetMetrics(make(map[string]*prometheus.Desc)), + }, nil +} + +func buildPerformanceLunCollector(backendName string, monitorType string, metricsIndicators []string, + metricsDataCache *metricsCache.MetricsDataCache) (prometheus.Collector, error) { + performanceBaseCollector, err := NewPerformanceBaseCollector( + backendName, monitorType, "lun", metricsIndicators, metricsDataCache) + if err != nil { + return nil, err + } + return &LunCollector{ + BaseCollector: performanceBaseCollector, + }, nil +} + +func NewLunCollector(backendName, monitorType string, metricsIndicators []string, + metricsDataCache *metricsCache.MetricsDataCache) (prometheus.Collector, error) { + buildFunc, ok := lunBuildMap[monitorType] + if !ok { + return nil, fmt.Errorf("can not create filesystem collector, " + + "the monitor type not in object or performance") + } + return buildFunc(backendName, monitorType, metricsIndicators, metricsDataCache) +} diff --git a/server/prometheus-exporter/collector/lun_collector_test.go b/server/prometheus-exporter/collector/lun_collector_test.go new file mode 100644 index 0000000..984577e --- /dev/null +++ b/server/prometheus-exporter/collector/lun_collector_test.go @@ -0,0 +1,70 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package collector + +import ( + "reflect" + "testing" + + "github.com/prometheus/client_golang/prometheus" +) + +func Test_parseLunCapacityUsage(t *testing.T) { + // arrange + mockInData := map[string]string{ + "CAPACITY": "10", + "ALLOCCAPACITY": "5", + } + + // action + got := parseCapacityUsage("", "", mockInData) + + // assert + if !reflect.DeepEqual(got, "50.00") { + t.Errorf("parseStorageStatus() got = %v, want %v", got, "50.00") + } +} + +func Test_buildObjectLunCollector(t *testing.T) { + // arrange + var wantCollector = &LunCollector{ + BaseCollector: &BaseCollector{ + backendName: "fake_backend", + monitorType: "object", + collectorName: "lun", + metricsHelpMap: lunObjectMetricsHelpMap, + metricsLabelMap: lunObjectMetricsLabelMap, + labelParseMap: lunObjectLabelParseMap, + metricsParseMap: lunObjectMetricsParseMap, + metricsDataCache: nil, + metrics: make(map[string]*prometheus.Desc), + }, + } + + // action + got, err := NewLunCollector("fake_backend", "object", []string{""}, + nil) + + // assert + if err != nil { + t.Errorf("NewLunCollector() error = %v", err) + return + } + if !reflect.DeepEqual(got, wantCollector) { + t.Errorf("NewLunCollector() got = %v, want %v", got, nil) + } +} diff --git a/server/prometheus-exporter/collector/perfomance_metrics_parse_map.go b/server/prometheus-exporter/collector/perfomance_metrics_parse_map.go new file mode 100644 index 0000000..4c0bbd9 --- /dev/null +++ b/server/prometheus-exporter/collector/perfomance_metrics_parse_map.go @@ -0,0 +1,94 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package collector includes all huawei storage collectors to gather and export huawei storage metrics. +package collector + +var performanceMetricsIndicatorsMap = map[string]string{ + "22": "total_iops", + "25": "read_iops", + "28": "write_iops", + "21": "total_bandwidth", + "23": "read_bandwidth", + "26": "write_bandwidth", + "370": "avg_io_response_time", + "182": "ops", + "524": "avg_read_ops_response_time", + "525": "avg_write_ops_response_time", +} + +var performanceMetricsLabelMap = map[string][]string{ + "total_iops": {"endpoint", "id", "object", "name"}, + "read_iops": {"endpoint", "id", "object", "name"}, + "write_iops": {"endpoint", "id", "object", "name"}, + "total_bandwidth": {"endpoint", "id", "object", "name"}, + "read_bandwidth": {"endpoint", "id", "object", "name"}, + "write_bandwidth": {"endpoint", "id", "object", "name"}, + "avg_io_response_time": {"endpoint", "id", "object", "name"}, + "ops": {"endpoint", "id", "object", "name"}, + "avg_read_ops_response_time": {"endpoint", "id", "object", "name"}, + "avg_write_ops_response_time": {"endpoint", "id", "object", "name"}, +} +var performanceMetricsHelpMap = map[string]string{ + "total_iops": "Total IOPS(IO/s)", + "read_iops": "Read IOPS(IO/s)", + "write_iops": "Write IOPS(IO/s)", + "total_bandwidth": "Total Bandwidth(MB/s)", + "read_bandwidth": "Read Bandwidth(MB/s)", + "write_bandwidth": "Write Bandwidth(MB/s)", + "avg_io_response_time": "Avg IO Response Time(us)", + "ops": "OPS", + "avg_read_ops_response_time": "Avg Read OPS Response Time(us)", + "avg_write_ops_response_time": "Avg Write OPS Response Time(us)", +} + +var performanceMetricsParseMap = map[string]parseRelation{ + "total_iops": {"22", parseStorageData}, + "read_iops": {"25", parseStorageData}, + "write_iops": {"28", parseStorageData}, + "total_bandwidth": {"21", parseStorageData}, + "read_bandwidth": {"23", parseStorageData}, + "write_bandwidth": {"26", parseStorageData}, + "avg_io_response_time": {"370", parseStorageData}, + "ops": {"182", parseStorageData}, + "avg_read_ops_response_time": {"524", parseStorageData}, + "avg_write_ops_response_time": {"525", parseStorageData}, +} +var performanceLabelParseMap = map[string]parseRelation{ + "endpoint": {"backendName", parseStorageData}, + "id": {"ObjectId", parseStorageData}, + "name": {"ObjectName", parseStorageData}, + "object": {"collectorName", parseStorageData}, +} + +func pickPerformanceParsMap[T any](needPerformanceMetrics []string, allPerformanceParseMap map[string]T) map[string]T { + var performanceParseMap = make(map[string]T) + if len(needPerformanceMetrics) == 0 { + return performanceParseMap + } + for _, indicatorName := range needPerformanceMetrics { + metricsName, ok := performanceMetricsIndicatorsMap[indicatorName] + if !ok { + continue + } + + parseInfo, ok := allPerformanceParseMap[metricsName] + if ok { + performanceParseMap[metricsName] = parseInfo + } + } + return performanceParseMap +} diff --git a/server/prometheus-exporter/collector/perfomance_metrics_parse_map_test.go b/server/prometheus-exporter/collector/perfomance_metrics_parse_map_test.go new file mode 100644 index 0000000..b4ed56d --- /dev/null +++ b/server/prometheus-exporter/collector/perfomance_metrics_parse_map_test.go @@ -0,0 +1,69 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package collector + +import ( + "reflect" + "testing" +) + +func Test_pickPerformanceParsMap_MapString(t *testing.T) { + // arrange + mockMetricsData := []string{"22", "25"} + want := map[string]string{ + "total_iops": "Total IOPS(IO/s)", + "read_iops": "Read IOPS(IO/s)", + } + + // action + got := pickPerformanceParsMap[string](mockMetricsData, performanceMetricsHelpMap) + + // assert + if !reflect.DeepEqual(got, want) { + t.Errorf("pickPerformanceParsMap() got = %v, want %v", got, want) + } +} + +func Test_pickPerformanceParsMap_MapParseRelation(t *testing.T) { + // arrange + mockMetricsData := []string{"22", "25"} + want := map[string]parseRelation{ + "total_iops": {"22", parseStorageData}, + "read_iops": {"25", parseStorageData}, + } + + // action + got := pickPerformanceParsMap[parseRelation](mockMetricsData, performanceMetricsParseMap) + + // assert + for name, wantData := range want { + gotData, ok := got[name] + if !ok { + t.Error("pickPerformanceParsMap() can not got data") + return + } + if gotData.parseKey != wantData.parseKey { + t.Error("pickPerformanceParsMap() got key not same") + return + } + if reflect.ValueOf(gotData.parseFunc).Pointer() != + reflect.ValueOf(wantData.parseFunc).Pointer() { + t.Error("pickPerformanceParsMap() got func not same") + return + } + } +} diff --git a/server/prometheus-exporter/collector/pv_collector.go b/server/prometheus-exporter/collector/pv_collector.go new file mode 100644 index 0000000..e1809c8 --- /dev/null +++ b/server/prometheus-exporter/collector/pv_collector.go @@ -0,0 +1,205 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package collector includes all huawei storage collectors to gather and export huawei storage metrics. +package collector + +import ( + "fmt" + "strings" + + "github.com/prometheus/client_golang/prometheus" + + metricsCache "github.com/huawei/csm/v2/server/prometheus-exporter/metricscache" +) + +var pvBuildMap = map[string]collectorInitFunc{ + "object": buildObjectPVCollector, + "performance": buildPerformancePVCollector, +} + +var pvLabelSlice = []string{"backend", "pv_name", "pvc_name", "object", + "storage_volume_type", "storage_volume_id", "storage_volume_name"} + +var pvObjectMetricsLabelMap = map[string][]string{ + "capacity": pvLabelSlice, + "capacity_usage": pvLabelSlice, +} + +var pvObjectMetricsHelpMap = map[string]string{ + "capacity": "Huawei Storage k8s PV Capacity(GB)", + "capacity_usage": "Huawei Storage k8s PV Capacity Usage(%)", +} + +var pvObjectMetricsParseMap = map[string]parseRelation{ + "capacity": {"CAPACITY", parseStorageSectorsToGB}, + "capacity_usage": {"", parseCapacityUsage}, +} + +var pvTypePrometheusMetrics = map[string][]string{ + "lun": {"lun_total_bandwidth", "lun_pv_lun_total_iops", "lun_avg_io_response_time"}, + "filesystem": {"filesystem_ops", "filesystem_avg_read_ops_response_time", "filesystem_avg_write_ops_response_time"}, +} + +var pvPrometheusMetricsLabelMap = map[string][]string{ + "lun_total_bandwidth": pvLabelSlice, + "lun_pv_lun_total_iops": pvLabelSlice, + "lun_avg_io_response_time": pvLabelSlice, + "filesystem_ops": pvLabelSlice, + "filesystem_avg_read_ops_response_time": pvLabelSlice, + "filesystem_avg_write_ops_response_time": pvLabelSlice, +} + +var pvPrometheusMetricsHelpMap = map[string]string{ + "lun_total_bandwidth": "Total Bandwidth(MB/s)", + "lun_pv_lun_total_iops": "Total IOPS(IO/s)", + "lun_avg_io_response_time": "Avg IO Response Time(us)", + "filesystem_ops": "OPS", + "filesystem_avg_read_ops_response_time": "Avg Read OPS Response Time(us)", + "filesystem_avg_write_ops_response_time": "Avg Write OPS Response Time(us)", +} + +var pvPrometheusMetricsParseMap = map[string]parseRelation{ + "lun_total_bandwidth": {"21", parseStorageData}, + "lun_pv_lun_total_iops": {"22", parseStorageData}, + "lun_avg_io_response_time": {"370", parseStorageData}, + "filesystem_ops": {"182", parseStorageData}, + "filesystem_avg_read_ops_response_time": {"524", parseStorageData}, + "filesystem_avg_write_ops_response_time": {"525", parseStorageData}, +} + +var pvLabelParseMap = map[string]parseRelation{ + "backend": {"sbcName", parseStorageData}, + "pv_name": {"pvName", parseStorageData}, + "pvc_name": {"pvcName", parseStorageData}, + "storage_volume_type": {"sbcStorageType", parseStorageData}, + "storage_volume_id": {"ID", parsePVStorageID}, + "storage_volume_name": {"sameName", parseStorageData}, + "object": {"collectorName", parseStorageData}, +} + +func init() { + RegisterCollector("pv", NewPVCollector) +} + +type PVCollector struct { + *BaseCollector +} + +func parsePVStorageID(inDataKey, metricsName string, inData map[string]string) string { + if len(inData) == 0 { + return "" + } + _, ok := inData["NAME"] + if ok { + return inData["ID"] + } + _, ok = inData["ObjectName"] + if ok { + return inData["ObjectId"] + } + return "" +} + +func parsePVCapacityUsage(inDataKey, metricsName string, inData map[string]string) string { + if len(inData) == 0 { + return "" + } + pvType, ok := inData["sbcStorageType"] + if !ok { + return "" + } + var pvCapacityUsage string + if pvType == "oceanstor-san" { + pvCapacityUsage = parseCapacityUsage(inDataKey, metricsName, inData) + } + if pvType == "oceanstor-nas" { + pvCapacityUsage = parseStorageData("AVAILABLEANDALLOCCAPACITYRATIO", metricsName, inData) + } + return pvCapacityUsage +} + +func buildObjectPVCollector(backendName string, monitorType string, metricsIndicators []string, + metricsDataCache *metricsCache.MetricsDataCache) (prometheus.Collector, error) { + return &PVCollector{ + BaseCollector: (&BaseCollector{}).SetBackendName(backendName). + SetMonitorType(monitorType). + SetCollectorName("pv"). + SetMetricsHelpMap(pvObjectMetricsHelpMap). + SetMetricsLabelMap(pvObjectMetricsLabelMap). + SetLabelParseMap(pvLabelParseMap). + SetMetricsParseMap(pvObjectMetricsParseMap). + SetMetricsDataCache(metricsDataCache). + SetMetrics(make(map[string]*prometheus.Desc)), + }, nil +} + +func buildPerformancePVCollector(backendName string, monitorType string, metricsIndicators []string, + metricsDataCache *metricsCache.MetricsDataCache) (prometheus.Collector, error) { + if len(metricsIndicators) == 0 || metricsIndicators[0] == "" { + return nil, fmt.Errorf("can not create [%s] collector, "+ + "the metricsIndicators is empty or error", "pv") + } + metricsData := strings.Split(metricsIndicators[0], ",") + return &PVCollector{ + BaseCollector: (&BaseCollector{}).SetBackendName(backendName). + SetMonitorType(monitorType). + SetCollectorName("pv"). + SetMetricsHelpMap(pickPVPerformanceParsMap[string](metricsData, pvPrometheusMetricsHelpMap)). + SetMetricsLabelMap(pickPVPerformanceParsMap[[]string](metricsData, pvPrometheusMetricsLabelMap)). + SetLabelParseMap(pvLabelParseMap). + SetMetricsParseMap(pickPVPerformanceParsMap[parseRelation](metricsData, pvPrometheusMetricsParseMap)). + SetMetricsDataCache(metricsDataCache). + SetMetrics(make(map[string]*prometheus.Desc)), + }, nil +} + +func pickPVPerformanceParsMap[T any](needPerformanceMetrics []string, + allPerformanceParseMap map[string]T) map[string]T { + var performanceParseMap = make(map[string]T) + if len(needPerformanceMetrics) == 0 { + return performanceParseMap + } + var allNeedMetricsSlice []string + for _, pvType := range needPerformanceMetrics { + needMetricsSlice, ok := pvTypePrometheusMetrics[pvType] + if !ok { + continue + } + allNeedMetricsSlice = append(allNeedMetricsSlice, needMetricsSlice...) + } + if len(allNeedMetricsSlice) == 0 { + return performanceParseMap + } + + for _, metricsName := range allNeedMetricsSlice { + parseInfo, ok := allPerformanceParseMap[metricsName] + if ok { + performanceParseMap[metricsName] = parseInfo + } + } + return performanceParseMap +} + +func NewPVCollector(backendName, monitorType string, metricsIndicators []string, + metricsDataCache *metricsCache.MetricsDataCache) (prometheus.Collector, error) { + buildFunc, ok := pvBuildMap[monitorType] + if !ok { + return nil, fmt.Errorf("can not create filesystem collector, " + + "the monitor type not in object or performance") + } + return buildFunc(backendName, monitorType, metricsIndicators, metricsDataCache) +} diff --git a/server/prometheus-exporter/collector/pv_collector_test.go b/server/prometheus-exporter/collector/pv_collector_test.go new file mode 100644 index 0000000..6fc2b0e --- /dev/null +++ b/server/prometheus-exporter/collector/pv_collector_test.go @@ -0,0 +1,77 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package collector + +import ( + "reflect" + "testing" +) + +func Test_parsePVStorageIDGetName(t *testing.T) { + // arrange + mockInDataKey := "" + mockMetricsName := "" + mockInData := map[string]string{ + "NAME": "fake_name", + "ID": "fake_data", + } + + // action + got := parsePVStorageID(mockInDataKey, mockMetricsName, mockInData) + + // assert + if !reflect.DeepEqual(got, "fake_data") { + t.Errorf("parseStorageData() got = %v, want %v", got, "fake_data") + } +} + +func Test_parsePVStorageIDGetObjectName(t *testing.T) { + // arrange + mockInDataKey := "" + mockMetricsName := "" + mockInData := map[string]string{ + "ObjectName": "fake_name", + "ObjectId": "fake_data", + } + + // action + got := parsePVStorageID(mockInDataKey, mockMetricsName, mockInData) + + // assert + if !reflect.DeepEqual(got, "fake_data") { + t.Errorf("parseStorageData() got = %v, want %v", got, "fake_data") + } +} + +func Test_parsePVCapacityUsageSan(t *testing.T) { + // arrange + mockInDataKey := "" + mockMetricsName := "" + mockInData := map[string]string{ + "sbcStorageType": "oceanstor-san", + "CAPACITY": "100", + "ALLOCCAPACITY": "10", + } + + // action + got := parsePVCapacityUsage(mockInDataKey, mockMetricsName, mockInData) + + // assert + if !reflect.DeepEqual(got, "10.00") { + t.Errorf("parseStorageData() got = %v, want %v", got, "fake_data") + } +} diff --git a/server/prometheus-exporter/collector/storage_const.go b/server/prometheus-exporter/collector/storage_const.go new file mode 100644 index 0000000..7e20182 --- /dev/null +++ b/server/prometheus-exporter/collector/storage_const.go @@ -0,0 +1,249 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package collector includes all huawei storage collectors to gather and export huawei storage metrics. +package collector + +// StorageProductMode The Product Mode name map for storage +var StorageProductMode = map[string]string{ + "61": "6800 V3", + "62": "6900 V3", + "63": "5600 V3", + "64": "5800 V3", + "68": "5500 V3", + "69": "2600 V3", + "70": "5300 V3", + "71": "2800 V3", + "72": "18500 V3", + "73": "18800 V3", + "74": "2200 V3", + "84": "2600F V3", + "85": "5500F V3", + "86": "5600F V3", + "87": "5800F V3", + "88": "6800F V3", + "89": "18500F V3", + "90": "18800F V3", + "92": "2800 V5", + "93": "5300 V5", + "94": "5300F V5", + "95": "5500 V5", + "96": "5500F V5", + "97": "5600 V5", + "98": "5600F V5", + "99": "5800 V5", + "100": "5800F V5", + "101": "6800 V5", + "102": "6800F V5", + "103": "18500 V5", + "104": "18500F V5", + "105": "18800 V5", + "106": "18800F V5", + "107": "5500 V5 Elite", + "108": "2100 V3", + "805": "Dorado5000 V3", + "806": "Dorado6000 V3", + "807": "Dorado18000 V3", + "808": "Dorado NAS", + "809": "Dorado NAS", + "810": "Dorado3000 V3", + "112": "2200 V3", + "113": "2600 V3", + "114": "2600F V3", + "115": "5300 V5", + "116": "5110 V5", + "117": "5110F V5", + "118": "5210 V5", + "119": "5210F V5", + "120": "5310 V5", + "121": "5310F V5", + "122": "5510 V5", + "123": "5510F V5", + "124": "5610 V5", + "125": "5610F V5", + "126": "5810 V5", + "127": "5810F V5", + "128": "6810 V5", + "129": "6810F V5", + "130": "18510 V5", + "131": "18510F V5", + "132": "18810 V5", + "133": "18810F V5", + "134": "5210 V5 Enhanced", + "135": "5210F V5 Enhanced", + "139": "5110 V5 Enhanced", + "140": "5110F V5 Enhanced", + "141": "2600 V5", + "811": "OceanStor Dorado 5000 V6", + "812": "OceanStor Dorado 5000 V6", + "813": "OceanStor Dorado 6000 V6", + "814": "OceanStor Dorado 6000 V6", + "815": "OceanStor Dorado 8000 V6", + "816": "OceanStor Dorado 8000 V6", + "817": "OceanStor Dorado 18000 V6", + "818": "OceanStor Dorado 18000 V6", + "819": "OceanStor Dorado 3000 V6", + "821": "OceanStor Dorado 5000 V6", + "822": "OceanStor Dorado 6000 V6", + "823": "OceanStor Dorado 8000 V6", + "824": "OceanStor Dorado 18000 V6", + "825": "OceanStor Dorado 5300 V6", + "826": "OceanStor Dorado 5500 V6", + "827": "OceanStor Dorado 5600 V6", + "828": "OceanStor Dorado 5800 V6", + "829": "OceanStor Dorado 6800 V6", + "830": "OceanStor Dorado 185000 V6", + "831": "OceanStor Dorado 188000 V6", + "832": "OceanStor Dorado 18800K V6", +} + +// StorageHealthStatus The Health Status name map for storage +var StorageHealthStatus = map[string]string{ + "0": "--", + "1": "Normal", + "2": "Fault", + "3": "Pre-Fail", + "4": "Partially Broken", + "5": "Degraded", + "6": "Bad Sectors Found", + "7": "Bit Errors Found", + "8": "Consistent", + "9": "Inconsistent", + "10": "Busy", + "11": "No Input", + "12": "Low Battery", + "13": "Single Link Fault", + "14": "Invalid", + "15": "Write Protect", +} + +// StorageRunningStatus The Running Status name map for storage +var StorageRunningStatus = map[string]string{ + "0": "Unknown", + "1": "Normal", + "2": "Running", + "3": "Not running", + "4": "Not existed", + "5": "Sleep in high temperature", + "6": "Starting", + "7": "Power failure protection", + "8": "Spin down", + "9": "Started", + "10": "Link up", + "11": "Link down", + "12": "Powering on", + "13": "Powered off", + "14": "Pre-copy", + "15": "Copyback", + "16": "Reconstruction", + "17": "Expansion", + "18": "Unformatted", + "19": "Formatting", + "20": "Unmapped", + "21": "Initial synchronizing", + "22": "Consistent", + "23": "Synchronizing", + "24": "Synchronized", + "25": "Unsynchronized", + "26": "Split", + "27": "Online", + "28": "Offline", + "29": "Locked", + "30": "Enabled", + "31": "Disabled", + "32": "Balancing", + "33": "To be recovered", + "34": "Interrupted", + "35": "Invalid", + "36": "Not start", + "37": "Queuing", + "38": "Stopped", + "39": "Copying", + "40": "Completed", + "41": "Paused", + "42": "Reverse synchronizing", + "43": "Activated", + "44": "Restore", + "45": "Inactive", + "46": "Idle", + "47": "Powering off", + "48": "Charging", + "49": "Charging completed", + "50": "Discharging", + "51": "Upgrading", + "52": "Power Lost", + "53": "Initializing", + "54": "Apply change", + "55": "Online disable", + "56": "Offline disable", + "57": "Online frozen", + "58": "Offline frozen", + "59": "Closed", + "60": "Removing", + "61": "In service", + "62": "Out of service", + "63": "Running normal", + "64": "Running fail", + "67": "Running failed", + "68": "Waiting", + "69": "Canceling", + "70": "Canceled", + "71": "About to synchronize", + "72": "Synchronizing data", + "73": "Failed to synchronize", + "74": "Fault", + "75": "Migrating", + "76": "Migrated", + "77": "Activating", + "78": "Deactivating", + "79": "Start failed", + "80": "Stop failed", + "81": "Decommissioning", + "82": "Decommissioned", + "83": "Recommissioning", + "84": "Replacing node", + "85": "Scheduling", + "86": "Pausing", + "87": "Suspending", + "88": "Suspended", + "89": "Overload", + "90": "To be switch", + "91": "Switching", + "92": "To be cleanup", + "93": "Forced start", + "94": "Error", + "95": "Job completed", + "96": "Partition Migrating", + "97": "Mount", + "98": "Unmount", + "99": "INSTALLING", + "100": "To Be Synchronized", + "101": "Connecting", + "102": "Service Switching", + "103": "Power-on failed", + "104": "Repairing", + "105": "Abnormal", + "106": "Deleting", + "107": "Modifying", + "108": "Running(clearing data)", + "109": "Running(synchronizing data)", + "110": "Standby", + "111": "STOPPING", + "112": "RESTORE FAULT", + "113": "Cut Over", + "114": "Erasing", + "115": "Verifying", +} diff --git a/server/prometheus-exporter/collector/storage_pool_collector.go b/server/prometheus-exporter/collector/storage_pool_collector.go new file mode 100644 index 0000000..9c7915c --- /dev/null +++ b/server/prometheus-exporter/collector/storage_pool_collector.go @@ -0,0 +1,145 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package collector includes all huawei storage collectors to gather and export huawei storage metrics. +package collector + +import ( + "fmt" + "strconv" + + "github.com/prometheus/client_golang/prometheus" + + metricsCache "github.com/huawei/csm/v2/server/prometheus-exporter/metricscache" +) + +func init() { + RegisterCollector("storagepool", NewStoragePoolCollector) +} + +const ( + storagePoolCapacityKey = "USERTOTALCAPACITY" + storagePoolUsedCapacityKey = "USERCONSUMEDCAPACITY" +) + +var storagePoolPrometheusDescName = map[string]string{ + "object": "storage_pool", + "performance": "storagepool", +} + +var storagePoolObjectMetricsLabelMap = map[string][]string{ + "total_capacity": {"name", "endpoint", "id", "object"}, + "free_capacity": {"name", "endpoint", "id", "object"}, + "capacity_usage": {"name", "endpoint", "id", "object"}, + "used_capacity": {"name", "endpoint", "id", "object"}, +} +var storagePoolObjectMetricsHelpMap = map[string]string{ + "total_capacity": "Total capacity(GB) of storage pool", + "free_capacity": "Free capacity(GB) of storage pool", + "capacity_usage": "Used capacity ratio(%) of storage pool", + "used_capacity": "Used capacity(GB) of storage pool", +} + +var storagePoolObjectMetricsParseMap = map[string]parseRelation{ + "total_capacity": {"USERTOTALCAPACITY", parseStorageSectorsToGB}, + "free_capacity": {"USERFREECAPACITY", parseStorageSectorsToGB}, + "capacity_usage": {"", parseStoragePoolCapacityUsage}, + "used_capacity": {"USERCONSUMEDCAPACITY", parseStorageSectorsToGB}, +} +var storagePoolObjectLabelParseMap = map[string]parseRelation{ + "endpoint": {"backendName", parseStorageData}, + "id": {"ID", parseStorageData}, + "name": {"NAME", parseStorageData}, + "object": {"collectorName", parseStorageData}, +} + +// StoragePoolCollector implements the prometheus.Collector interface and build storage StoragePool info +type StoragePoolCollector struct { + *BaseCollector +} + +func parseStoragePoolCapacityUsage(inDataKey, metricsName string, inData map[string]string) string { + if len(inData) == 0 { + return "" + } + capacity, err := strconv.ParseFloat(inData[storagePoolCapacityKey], bitSize) + if err != nil || capacity == 0 { + return "" + } + usedCapacity, err := strconv.ParseFloat(inData[storagePoolUsedCapacityKey], bitSize) + if err != nil { + return "" + } + return strconv.FormatFloat(usedCapacity/capacity*calculatePercentage, 'f', precisionOfTwo, bitSize) +} + +// Describe implements the prometheus.Collector interface. +// Use BuildDesc to build storage pool Desc then send to prometheus. +func (storagePoolCollector *StoragePoolCollector) Describe(ch chan<- *prometheus.Desc) { + storagePoolCollector.BuildDesc() + for _, i := range storagePoolCollector.metrics { + ch <- i + } +} + +// BuildDesc use StoragePoolCollector.metricsDescMap create storage poo Collector prometheus.Desc +func (storagePoolCollector *StoragePoolCollector) BuildDesc() { + if storagePoolCollector.metrics == nil { + storagePoolCollector.metrics = make(map[string]*prometheus.Desc) + } + storagePoolDes, ok := storagePoolPrometheusDescName[storagePoolCollector.monitorType] + if !ok { + return + } + for metricsName, helpInfo := range storagePoolCollector.metricsHelpMap { + storagePoolCollector.metrics[metricsName] = + prometheus.NewDesc( + prometheus.BuildFQName( + MetricsNamespace, storagePoolDes, metricsName), + helpInfo, + storagePoolCollector.metricsLabelMap[metricsName], + nil) + } +} + +func NewStoragePoolCollector(backendName, monitorType string, metricsIndicators []string, + metricsDataCache *metricsCache.MetricsDataCache) (prometheus.Collector, error) { + if monitorType == "object" { + return &StoragePoolCollector{ + BaseCollector: (&BaseCollector{}).SetBackendName(backendName). + SetMonitorType(monitorType). + SetCollectorName("storagepool"). + SetMetricsHelpMap(storagePoolObjectMetricsHelpMap). + SetMetricsLabelMap(storagePoolObjectMetricsLabelMap). + SetLabelParseMap(storagePoolObjectLabelParseMap). + SetMetricsParseMap(storagePoolObjectMetricsParseMap). + SetMetricsDataCache(metricsDataCache). + SetMetrics(make(map[string]*prometheus.Desc)), + }, nil + } else if monitorType == "performance" { + performanceBaseCollector, err := NewPerformanceBaseCollector( + backendName, monitorType, "storagepool", metricsIndicators, metricsDataCache) + if err != nil { + return nil, err + } + return &StoragePoolCollector{ + BaseCollector: performanceBaseCollector, + }, nil + } + + return nil, fmt.Errorf("can not create storage pool collector, " + + "the monitor type not in object or performance") +} diff --git a/server/prometheus-exporter/collector/storage_pool_collector_test.go b/server/prometheus-exporter/collector/storage_pool_collector_test.go new file mode 100644 index 0000000..6630f09 --- /dev/null +++ b/server/prometheus-exporter/collector/storage_pool_collector_test.go @@ -0,0 +1,112 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package collector + +import ( + "reflect" + "testing" + + "github.com/prometheus/client_golang/prometheus" +) + +func TestStoragePoolCollector_BuildDesc(t *testing.T) { + // arrange + var mockMetricsLabelMap = map[string][]string{ + "fake_key1": {"fake_label1", "fake_label2"}, + "fake_key2": {"fake_label2", "fake_label3"}, + } + var mockMetricsHelpMap = map[string]string{ + "fake_key1": "fake help info1", + "fake_key2": "fake help info2", + } + var mockCollector = StoragePoolCollector{ + BaseCollector: &BaseCollector{ + backendName: "fake_name", + monitorType: "object", + collectorName: "storagepool", + metricsHelpMap: mockMetricsHelpMap, + metricsLabelMap: mockMetricsLabelMap, + metrics: make(map[string]*prometheus.Desc)}, + } + var mockMetrics = map[string]*prometheus.Desc{ + "fake_key1": prometheus.NewDesc( + prometheus.BuildFQName( + MetricsNamespace, "storage_pool", "fake_key1"), + "fake help info1", + mockCollector.metricsLabelMap["fake_key1"], + nil), + "fake_key2": prometheus.NewDesc( + prometheus.BuildFQName( + MetricsNamespace, "storage_pool", "fake_key2"), + "fake help info2", + mockCollector.metricsLabelMap["fake_key2"], + nil), + } + + // action + mockCollector.BuildDesc() + + // assert + if !reflect.DeepEqual(mockMetrics, mockCollector.metrics) { + t.Errorf("BuildDesc() want = %v, but got = %v", mockMetrics, mockCollector) + } +} + +func TestNewStoragePoolCollector_GetObjectCollector(t *testing.T) { + // arrange + var wantCollector = &StoragePoolCollector{ + BaseCollector: &BaseCollector{ + backendName: "fake_backend", + monitorType: "object", + collectorName: "storagepool", + metricsHelpMap: storagePoolObjectMetricsHelpMap, + metricsLabelMap: storagePoolObjectMetricsLabelMap, + labelParseMap: storagePoolObjectLabelParseMap, + metricsParseMap: storagePoolObjectMetricsParseMap, + metricsDataCache: nil, + metrics: make(map[string]*prometheus.Desc), + }, + } + + // action + got, err := NewStoragePoolCollector("fake_backend", "object", []string{""}, + nil) + + // assert + if err != nil { + t.Errorf("NewStoragePoolCollector() error = %v", err) + return + } + if !reflect.DeepEqual(got, wantCollector) { + t.Errorf("NewStoragePoolCollector() got = %v, want %v", got, nil) + } +} + +func TestNewStoragePoolCollector_GetPerformanceCollector(t *testing.T) { + // arrange + var mockMetricsIndicators []string + + // action + _, err := NewStoragePoolCollector("fake_backend", "performance", mockMetricsIndicators, + nil) + + // assert + if err == nil { + t.Errorf("NewStoragePoolCollector() error = %v, wantErr %v", err, true) + return + } +} diff --git a/server/prometheus-exporter/exporterhandler/exporter_handler.go b/server/prometheus-exporter/exporterhandler/exporter_handler.go new file mode 100644 index 0000000..a5134a7 --- /dev/null +++ b/server/prometheus-exporter/exporterhandler/exporter_handler.go @@ -0,0 +1,145 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package exporterhandler provide all handler use by prometheus exporter +package exporterhandler + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + + "github.com/huawei/csm/v2/server/prometheus-exporter/collector" + metricsCache "github.com/huawei/csm/v2/server/prometheus-exporter/metricscache" + "github.com/huawei/csm/v2/utils/log" +) + +// url path is ip/monitorType/monitorBackendName. So when use strings.Split, pathLen is 3. +const pathLen = 3 + +var ( + // Supported monitoring types + monitorTypeLegal = map[string]struct{}{ + "object": {}, + "performance": {}, + } + // Supported monitoring types + metricsObjectLegal = map[string]struct{}{ + "array": {}, + "controller": {}, + "storagepool": {}, + "lun": {}, + "filesystem": {}, + "pv": {}, + } +) + +func checkMetricsObject(ctx context.Context, params map[string][]string, monitorType string) error { + if monitorType == "" { + return fmt.Errorf("the monitorType is empty") + } + + for collectorName, metricsIndicators := range params { + if _, err := metricsObjectLegal[collectorName]; !err { + log.AddContext(ctx).Errorln("the collectorName is error") + return fmt.Errorf("the collectorName is error") + } + + if monitorType == "performance" && (len(metricsIndicators) == 0 || metricsIndicators[0] == "") { + log.AddContext(ctx).Errorf("can not get the [%s] performance indicators", collectorName) + return fmt.Errorf("the metricsIndicators is error") + } + } + + return nil +} + +func parseRequestPath(ctx context.Context, w http.ResponseWriter, r *http.Request) (string, string, error) { + path := strings.Split(r.URL.Path, "/") + if len(path) != pathLen { + http.Error(w, "URL is error.", http.StatusBadRequest) + return "", "", errors.New("URL is error") + } + + monitorType := path[1] + if _, err := monitorTypeLegal[monitorType]; !err { + http.Error(w, "MonitorType is error.", http.StatusBadRequest) + return "", "", errors.New("MonitorType is error") + } + + monitorBackendName := path[2] + params := r.URL.Query() + checkError := checkMetricsObject(ctx, params, monitorType) + if checkError != nil { + http.Error(w, "MetricsObjectType is error.", http.StatusBadRequest) + return "", "", errors.New("MetricsObjectType is error") + } + return monitorBackendName, monitorType, nil +} + +func getBatchData(ctx context.Context, monitorBackendName, monitorType string, + params map[string][]string) *metricsCache.MetricsDataCache { + batchMetricsDataCache := metricsCache.MetricsDataCache{ + BackendName: monitorBackendName, + CacheDataMap: make(map[string]metricsCache.MetricsData), + MergeMetrics: make(map[string]metricsCache.MergeMetricsData)} + log.AddContext(ctx).Infof("start to get batch monitor data, backend: %v, type: %v, params: %v", + monitorBackendName, monitorType, params) + batchParams, err := batchMetricsDataCache.BuildBatchDataClass(ctx, monitorType, params) + if err != nil { + return nil + } + batchMetricsDataCache.SetBatchDataFromSource(ctx, monitorType, batchParams) + batchMetricsDataCache.MergeBatchData(ctx) + log.AddContext(ctx).Infoln("get batch monitor data finished, start to collect data for prometheus") + return &batchMetricsDataCache +} + +// MetricsHandler get the parse request get batch data and build data to prometheus +func MetricsHandler(w http.ResponseWriter, r *http.Request) { + ctx, err := log.SetRequestInfo(context.Background()) + if err != nil { + return + } + monitorBackendName, monitorType, err := parseRequestPath(ctx, w, r) + if err != nil { + return + } + + params := r.URL.Query() + + batchMetricsDataCache := getBatchData(ctx, monitorBackendName, monitorType, params) + if batchMetricsDataCache == nil { + return + } + + allCollectors, err := collector.NewCollectorSet( + ctx, params, monitorBackendName, monitorType, batchMetricsDataCache) + if err != nil { + http.Error(w, "get allCollectors is error.", http.StatusBadRequest) + log.AddContext(ctx).Errorf("get allCollectors failed, the error is [%v]", err) + return + } + registry := prometheus.NewRegistry() + registry.MustRegister(allCollectors) + h := promhttp.HandlerFor(registry, promhttp.HandlerOpts{}) + h.ServeHTTP(w, r) +} diff --git a/server/prometheus-exporter/exporterhandler/exporter_handler_test.go b/server/prometheus-exporter/exporterhandler/exporter_handler_test.go new file mode 100644 index 0000000..8d1416f --- /dev/null +++ b/server/prometheus-exporter/exporterhandler/exporter_handler_test.go @@ -0,0 +1,77 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package exporterhandler + +import ( + "context" + "net/http" + "net/url" + "testing" + + "github.com/agiledragon/gomonkey/v2" +) + +func Test_parseRequestPath(t *testing.T) { + // arrange + ctx := context.TODO() + mockRequest := http.Request{URL: &url.URL{Path: "/object/backend_name"}} + var mockResponse http.ResponseWriter + + // mock + patches := gomonkey. + ApplyFunc(checkMetricsObject, + func(ctx context.Context, params map[string][]string, monitorType string) error { + return nil + }) + defer patches.Reset() + + // action + monitorBackendName, monitorType, _ := parseRequestPath(ctx, mockResponse, &mockRequest) + + // assert + if monitorBackendName != "backend_name" || monitorType != "object" { + t.Errorf("parseRequestPath() error want backend_name and object") + } +} + +func Test_checkMetricsObject_Success(t *testing.T) { + // arrange + ctx := context.TODO() + mockParams := map[string][]string{"array": {""}} + + // action + err := checkMetricsObject(ctx, mockParams, "object") + + // assert + if err != nil { + t.Errorf("checkMetricsObject() error the err is [%v]", err) + } +} + +func Test_checkMetricsObject_PerformanceIndicatorsError(t *testing.T) { + // arrange + ctx := context.TODO() + mockParams := map[string][]string{"array": {""}} + + // action + err := checkMetricsObject(ctx, mockParams, "performance") + + // assert + if err.Error() != "the metricsIndicators is error" { + t.Errorf("checkMetricsObject() error the err is [%v]", err) + } +} diff --git a/server/prometheus-exporter/exporterhandler/health_handler.go b/server/prometheus-exporter/exporterhandler/health_handler.go new file mode 100644 index 0000000..6dc7808 --- /dev/null +++ b/server/prometheus-exporter/exporterhandler/health_handler.go @@ -0,0 +1,31 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package exporterhandler provide all handler use by prometheus exporter +package exporterhandler + +import ( + "net/http" + + "github.com/huawei/csm/v2/utils/log" +) + +// HealthHandler get the health status from plugin to Kubernetes +func HealthHandler(w http.ResponseWriter, r *http.Request) { + log.Debugln("call health") + w.WriteHeader(http.StatusOK) + log.Debugln("health success") +} diff --git a/server/prometheus-exporter/metricscache/kube_pv_data.go b/server/prometheus-exporter/metricscache/kube_pv_data.go new file mode 100644 index 0000000..9af318a --- /dev/null +++ b/server/prometheus-exporter/metricscache/kube_pv_data.go @@ -0,0 +1,151 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package metricscache use to save query the data of the storage metrics once +package metricscache + +import ( + "context" + "errors" + "strings" + + xuanwuV1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + coreV1 "k8s.io/api/core/v1" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + exporterConfig "github.com/huawei/csm/v2/config/exporter" + storageGRPC "github.com/huawei/csm/v2/grpc/lib/go/cmi" + clientSet "github.com/huawei/csm/v2/server/prometheus-exporter/clientset" + "github.com/huawei/csm/v2/utils/log" +) + +// when use kubelet get pv -A -v6 the limit set is 500, so we set same as kubelet. +const getPVLimit = 500 + +// sbc configMap is sbcNameSpace/sbcName so when use strings.Split, it len is 2. +const sbConfigMapLen = 2 + +func getPVDataFromApi(ctx context.Context) []coreV1.PersistentVolume { + usedClientSet := clientSet.GetExporterClientSet() + var allVolumeItems []coreV1.PersistentVolume + var continueKey string + if usedClientSet.KubeClient == nil { + return allVolumeItems + } + + for { + VolumeList, err := usedClientSet.KubeClient.CoreV1().PersistentVolumes().List( + ctx, metaV1.ListOptions{Limit: getPVLimit, Continue: continueKey}) + if err != nil { + log.AddContext(ctx).Errorln("can not get pv list") + break + } + allVolumeItems = append(allVolumeItems, VolumeList.Items...) + continueKey = VolumeList.ListMeta.Continue + if continueKey == "" { + break + } + } + return allVolumeItems +} + +func parseAllBackendInfo(allBackend *xuanwuV1.StorageBackendClaimList) map[string]map[string]string { + var allSBCInfo = make(map[string]map[string]string) + for _, sbcInfo := range allBackend.Items { + if sbcInfo.Status == nil { + continue + } + sbcStorageType := sbcInfo.Status.StorageType + + sbcConfigName := strings.Split(sbcInfo.Spec.ConfigMapMeta, "/") + if len(sbcConfigName) != sbConfigMapLen { + continue + } + sbcNameSpace := sbcConfigName[0] + sbcName := sbcConfigName[1] + + allSBCInfo[sbcName] = map[string]string{"namespace": sbcNameSpace, "sbcStorageType": sbcStorageType} + } + return allSBCInfo +} + +func getAllBackendFromApi(ctx context.Context) map[string]map[string]string { + usedClientSet := clientSet.GetExporterClientSet() + + if usedClientSet.SbcClient == nil { + return nil + } + + allBackend, err := usedClientSet.SbcClient.XuanwuV1().StorageBackendClaims( + exporterConfig.GetStorageBackendNamespace()).List(ctx, metaV1.ListOptions{}) + if err != nil { + return nil + } + allBackendInfo := parseAllBackendInfo(allBackend) + return allBackendInfo + +} + +func buildOutPVData(backendName, collectType string, allSBCInfo map[string]map[string]string, + allPVData []coreV1.PersistentVolume) *storageGRPC.CollectResponse { + outPVData := &storageGRPC.CollectResponse{ + BackendName: backendName, + CollectType: collectType, + Details: []*storageGRPC.CollectDetail{}} + for _, pvData := range allPVData { + pvMapInfo := &parsePVMetrics{ + collectDetail: &storageGRPC.CollectDetail{Data: make(map[string]string)}} + pvMapInfo.setCSIDriverNameMetrics(pvData). + setVolumeHandleMetrics(pvData). + setPVNameMetrics(pvData). + setPVCNameMetrics(pvData) + if pvMapInfo.parseError != nil { + continue + } + pvBackendName, ok := pvMapInfo.collectDetail.Data["sbcName"] + if !ok { + continue + } + + sbcInfo, ok := allSBCInfo[pvBackendName] + if !ok { + continue + } + pvMapInfo.collectDetail.Data["sbcStorageType"], ok = sbcInfo["sbcStorageType"] + if !ok { + continue + } + outPVData.Details = append(outPVData.Details, pvMapInfo.collectDetail) + } + return outPVData +} + +// GetAndParsePVInfo gets and parses pv info with special collect type +func GetAndParsePVInfo(ctx context.Context, backendName, collectType string) (*storageGRPC.CollectResponse, error) { + allPVData := getPVDataFromApi(ctx) + if len(allPVData) == 0 { + log.AddContext(ctx).Warningln("can not get pv data, pv is empty") + return nil, errors.New("can not get pv data, pv is empty") + } + allSBCInfo := getAllBackendFromApi(ctx) + if len(allSBCInfo) == 0 { + log.AddContext(ctx).Warningln("can not get sbc data, sbc is empty") + return nil, errors.New("can not get sbc data, sbc is empty") + } + + outPVData := buildOutPVData(backendName, collectType, allSBCInfo, allPVData) + return outPVData, nil +} diff --git a/server/prometheus-exporter/metricscache/kube_pv_data_test.go b/server/prometheus-exporter/metricscache/kube_pv_data_test.go new file mode 100644 index 0000000..ba82eb3 --- /dev/null +++ b/server/prometheus-exporter/metricscache/kube_pv_data_test.go @@ -0,0 +1,208 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package metricscache + +import ( + "context" + "errors" + "fmt" + "reflect" + "testing" + + xuanwuV1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/agiledragon/gomonkey/v2" + coreV1 "k8s.io/api/core/v1" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + storageGRPC "github.com/huawei/csm/v2/grpc/lib/go/cmi" +) + +func Test_buildOutPVData(t *testing.T) { + // arrange + mockOutPVData := &storageGRPC.CollectResponse{ + BackendName: "fake_backend", + CollectType: "fake_type", + Details: []*storageGRPC.CollectDetail{}} + mockAllPVData := []coreV1.PersistentVolume{{}} + mockAllSBCInfo := map[string]map[string]string{} + mockParse := &parsePVMetrics{} + + // mock + mock := gomonkey.NewPatches() + mock.ApplyPrivateMethod(mockParse, "setCSIDriverNameMetrics", + func(volume coreV1.PersistentVolume) *parsePVMetrics { + return mockParse + }).ApplyPrivateMethod(mockParse, "setVolumeHandleMetrics", + func(volume coreV1.PersistentVolume) *parsePVMetrics { + return mockParse + }).ApplyPrivateMethod(mockParse, "setPVNameMetrics", + func(volume coreV1.PersistentVolume) *parsePVMetrics { + return mockParse + }).ApplyPrivateMethod(mockParse, "setPVCNameMetrics", + func(volume coreV1.PersistentVolume) *parsePVMetrics { + mockParse.parseError = errors.New("fake error") + return mockParse + }) + + // action + got := buildOutPVData("fake_backend", "fake_type", mockAllSBCInfo, mockAllPVData) + + // assert + if !reflect.DeepEqual(got, mockOutPVData) { + t.Errorf("buildOutPVData() got = %v, want %v", got, mockOutPVData) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func Test_parseAllBackendInfo_Success(t *testing.T) { + // arrange + namespace := "mockNamespace" + backendName := "mockBackendName" + storageType := "mockStorageType" + mockBackendList := &xuanwuV1.StorageBackendClaimList{ + TypeMeta: metaV1.TypeMeta{}, + ListMeta: metaV1.ListMeta{}, + Items: []xuanwuV1.StorageBackendClaim{{ + TypeMeta: metaV1.TypeMeta{}, + ObjectMeta: metaV1.ObjectMeta{}, + Spec: xuanwuV1.StorageBackendClaimSpec{ + ConfigMapMeta: namespace + "/" + backendName, + }, + Status: &xuanwuV1.StorageBackendClaimStatus{ + StorageType: storageType, + }, + }}, + } + wantRes := map[string]map[string]string{backendName: {"namespace": namespace, "sbcStorageType": storageType}} + + // act + gotRes := parseAllBackendInfo(mockBackendList) + + // assert + if !reflect.DeepEqual(gotRes, wantRes) { + t.Errorf("Test_parseAllBackendInfo_Success failed, gotRes [%v], wantRes [%v]", gotRes, wantRes) + } +} + +func TestGetAndParsePVInfo_Success(t *testing.T) { + // arrange + ctx := context.TODO() + backendName := "mockBackendName" + collectType := "mockCollectType" + outPVData := &storageGRPC.CollectResponse{ + BackendName: "fake_backend", + CollectType: "fake_type", + Details: []*storageGRPC.CollectDetail{}} + wantRes := outPVData + + // mock + p := gomonkey.NewPatches() + p.ApplyFunc(getPVDataFromApi, func(ctx context.Context) []coreV1.PersistentVolume { + return []coreV1.PersistentVolume{{Spec: coreV1.PersistentVolumeSpec{}}} + }).ApplyFunc(getAllBackendFromApi, func(ctx context.Context) map[string]map[string]string { + return map[string]map[string]string{ + backendName: {"namespace": "mockNamespace", "sbcStorageType": "mockStorageType"}, + } + }).ApplyFunc(buildOutPVData, func(backendName, collectType string, allSBCInfo map[string]map[string]string, + allPVData []coreV1.PersistentVolume) *storageGRPC.CollectResponse { + return outPVData + }) + + // action + gotRes, gotErr := GetAndParsePVInfo(ctx, backendName, collectType) + + // assert + if !reflect.DeepEqual(gotRes, wantRes) { + t.Errorf("TestGetAndParsePVInfo_Success failed, gotRes [%v], wantRes [%v]", gotRes, wantRes) + } + if gotErr != nil { + t.Errorf("TestGetAndParsePVInfo_Success failed, gotErr [%v], wantErr [%v]", gotErr, nil) + } + + // cleanup + t.Cleanup(func() { + p.Reset() + }) +} + +func TestGetAndParsePVInfo_GetPvFail(t *testing.T) { + // arrange + ctx := context.TODO() + backendName := "mockBackendName" + collectType := "mockCollectType" + getPvErr := fmt.Errorf("can not get pv data, pv is empty") + wantErr := getPvErr + + // mock + p := gomonkey.NewPatches() + p.ApplyFunc(getPVDataFromApi, func(ctx context.Context) []coreV1.PersistentVolume { + return nil + }) + + // action + gotRes, gotErr := GetAndParsePVInfo(ctx, backendName, collectType) + + // assert + if gotRes != nil { + t.Errorf("TestGetAndParsePVInfo_GetPvFail failed, gotRes [%v], wantRes [%v]", gotRes, nil) + } + if !reflect.DeepEqual(gotErr, wantErr) { + t.Errorf("TestGetAndParsePVInfo_GetPvFail failed, gotErr [%v], wantErr [%v]", gotErr, wantErr) + } + + // cleanup + t.Cleanup(func() { + p.Reset() + }) +} + +func TestGetAndParsePVInfo_GetBackendFail(t *testing.T) { + // arrange + ctx := context.TODO() + backendName := "mockBackendName" + collectType := "mockCollectType" + getBackendErr := fmt.Errorf("can not get sbc data, sbc is empty") + wantErr := getBackendErr + + // mock + p := gomonkey.NewPatches() + p.ApplyFunc(getPVDataFromApi, func(ctx context.Context) []coreV1.PersistentVolume { + return []coreV1.PersistentVolume{{Spec: coreV1.PersistentVolumeSpec{}}} + }).ApplyFunc(getAllBackendFromApi, func(ctx context.Context) map[string]map[string]string { + return nil + }) + + // action + gotRes, gotErr := GetAndParsePVInfo(ctx, backendName, collectType) + + // assert + if gotRes != nil { + t.Errorf("TestGetAndParsePVInfo_GetBackendFail failed, gotRes [%v], wantRes [%v]", gotRes, nil) + } + if !reflect.DeepEqual(gotErr, wantErr) { + t.Errorf("TestGetAndParsePVInfo_GetBackendFail failed, gotErr [%v], wantErr [%v]", gotErr, wantErr) + } + + // cleanup + t.Cleanup(func() { + p.Reset() + }) +} diff --git a/server/prometheus-exporter/metricscache/merge_pv_metrics.go b/server/prometheus-exporter/metricscache/merge_pv_metrics.go new file mode 100644 index 0000000..e00ad83 --- /dev/null +++ b/server/prometheus-exporter/metricscache/merge_pv_metrics.go @@ -0,0 +1,166 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package metricscache use to save query the data of the storage metrics once +package metricscache + +import ( + "context" + "errors" + "strings" + + storageGRPC "github.com/huawei/csm/v2/grpc/lib/go/cmi" + "github.com/huawei/csm/v2/utils/log" +) + +// MergePVMetricsData implement MergeMetricsData interface +type MergePVMetricsData struct { + *BaseMergeMetricsData +} + +func init() { + RegisterMergeMetricsData("pv", NewMergePVMetricsData) +} + +// NewMergePVMetricsData new a MergePVMetricsData +func NewMergePVMetricsData(backendName, monitorType, metricsType string, + metricsIndicators []string) (MergeMetricsData, error) { + return &MergePVMetricsData{BaseMergeMetricsData: &BaseMergeMetricsData{ + backendName: backendName, monitorType: monitorType, metricsType: metricsType, + mergeIndicators: metricsIndicators}}, nil +} + +func (mergePVMetricsData *MergePVMetricsData) mergeKubePVAndStorageInfo(ctx context.Context, + storageNameKey, pvNameKey, storageType string, pvCacheData []*storageGRPC.CollectDetail, + metricsDataCache *MetricsDataCache) (map[string]map[string]string, error) { + if len(pvCacheData) == 0 { + log.AddContext(ctx).Warningln("can not get the pv data when merge") + return nil, errors.New("can not get the pv data when merge") + } + storageCacheData := metricsDataCache.GetMetricsData(storageType) + if storageCacheData == nil || len(storageCacheData.Details) == 0 { + log.AddContext(ctx).Warningln("can not get the storage data when merge") + return nil, errors.New("can not get the storage data when merge") + } + + var pvCacheDataMap = make(map[string]map[string]string, len(pvCacheData)) + for _, pvData := range pvCacheData { + if pvData.Data[pvNameKey] == "" { + continue + } + storageTypeName, storageTypeExit := pvData.Data["sbcStorageType"] + if storageTypeExit && storageTypeMap[storageTypeName] != storageType { + continue + } + pvCacheDataMap[pvData.Data[pvNameKey]] = pvData.Data + } + + var resultMerge = make(map[string]map[string]string) + for _, mergeData := range storageCacheData.Details { + if len(mergeData.Data) == 0 { + continue + } + sameName := mergeData.Data[storageNameKey] + if sameName == "" { + continue + } + mergeData.Data["sameName"] = sameName + resultMerge[sameName+mergeData.Data["ID"]] = mergeData.Data + + sameData, sameNameExist := pvCacheDataMap[sameName] + if !sameNameExist { + continue + } + + for key, value := range sameData { + resultMerge[sameName+mergeData.Data["ID"]][key] = value + } + } + return resultMerge, nil +} + +func (mergePVMetricsData *MergePVMetricsData) getPVMergeParams(ctx context.Context) (string, []string, error) { + var metricsIndicatorsList []string + var storageNameKey string + if mergePVMetricsData.monitorType == "performance" && len(mergePVMetricsData.mergeIndicators) == 0 { + errorStr := "when get pv merge params, the monitorType is performance but mergeIndicators is empty" + log.AddContext(ctx).Errorln(errorStr) + return storageNameKey, metricsIndicatorsList, errors.New(errorStr) + } + + if mergePVMetricsData.monitorType == "performance" { + storageNameKey = "ObjectName" + metricsIndicatorsList = strings.Split(mergePVMetricsData.mergeIndicators[0], ",") + } else { + storageNameKey = "NAME" + metricsIndicatorsList = []string{"lun", "filesystem"} + } + if len(metricsIndicatorsList) == 0 { + errorStr := "when get pv merge params, the metricsIndicatorsList is empty" + log.AddContext(ctx).Errorln(errorStr) + return storageNameKey, metricsIndicatorsList, errors.New(errorStr) + } + + return storageNameKey, metricsIndicatorsList, nil +} + +// MergeData merge pv data and storage data +func (mergePVMetricsData *MergePVMetricsData) MergeData(ctx context.Context, + metricsDataCache *MetricsDataCache) error { + log.AddContext(ctx).Infoln("start to merge pv and storage data") + storageNameKey, metricsIndicatorsList, err := mergePVMetricsData.getPVMergeParams(ctx) + if err != nil { + log.AddContext(ctx).Errorln("can not get pv merge params, the error is %v", err) + return err + } + + pvCacheData, ok := metricsDataCache.CacheDataMap["pv"] + if !ok { + log.AddContext(ctx).Errorln("can not get pv cache data when MergePVAndStorageData") + return errors.New("can not get pv cache data when MergePVAndStorageData") + } + + pvMetricsDataResponse := pvCacheData.GetMetricsDataResponse() + if pvMetricsDataResponse == nil { + log.AddContext(ctx).Errorln("can not get pv MetricsDataResponse when MergePVAndStorageData") + return errors.New("can not get MetricsDataResponse data when MergePVAndStorageData") + } + + if len(pvMetricsDataResponse.Details) == 0 { + log.AddContext(ctx).Errorln("can not get pv MetricsDataResponse.Details when MergePVAndStorageData") + return errors.New("can not get MetricsDataResponse.Details when MergePVAndStorageData") + } + + pvTempData := pvMetricsDataResponse.Details + pvMetricsDataResponse.Details = nil + for _, storageTypeName := range metricsIndicatorsList { + mergeMapData, err := mergePVMetricsData.mergeKubePVAndStorageInfo( + ctx, storageNameKey, "storageName", storageTypeName, pvTempData, metricsDataCache) + if err != nil { + return err + } + for _, value := range mergeMapData { + _, ok := value["pvName"] + if !ok { + continue + } + pvMetricsDataResponse.Details = append(pvMetricsDataResponse.Details, + &storageGRPC.CollectDetail{Data: value}) + } + } + log.AddContext(ctx).Infoln("merge pv and storage data success") + return nil +} diff --git a/server/prometheus-exporter/metricscache/merge_pv_metrics_test.go b/server/prometheus-exporter/metricscache/merge_pv_metrics_test.go new file mode 100644 index 0000000..082a4b1 --- /dev/null +++ b/server/prometheus-exporter/metricscache/merge_pv_metrics_test.go @@ -0,0 +1,465 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package metricscache + +import ( + "context" + "fmt" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + + storageGRPC "github.com/huawei/csm/v2/grpc/lib/go/cmi" +) + +func TestMergePVMetricsData_mergeKubePVAndStorageInfo_Success(t *testing.T) { + // arrange + ctx := context.TODO() + storageNameKey := "storageNameKey" + pvNameKey := "pvNameKey" + storageType := "storageType" + metricsDataCache := &MetricsDataCache{} + mergePVMetricsData := &MergePVMetricsData{} + + mockName := "name" + mockId := "001" + pvCacheData := []*storageGRPC.CollectDetail{{Data: map[string]string{ + pvNameKey: mockName, + "field1": "field1 context", + "field2": "field2 context", + }}} + + wantRes := map[string]map[string]string{mockName + mockId: { + pvNameKey: mockName, + "field1": "field1 context", + "field2": "field2 context", + storageNameKey: mockName, + "ID": mockId, + "sameName": mockName, + }} + + // mock + p := gomonkey.NewPatches() + p.ApplyMethod(reflect.TypeOf(metricsDataCache), "GetMetricsData", + func(_ *MetricsDataCache, metricsType string) *storageGRPC.CollectResponse { + return &storageGRPC.CollectResponse{ + Details: []*storageGRPC.CollectDetail{{Data: map[string]string{ + storageNameKey: mockName, + "ID": mockId, + }}}, + } + }) + + // action + gotRes, gotErr := mergePVMetricsData.mergeKubePVAndStorageInfo(ctx, storageNameKey, pvNameKey, + storageType, pvCacheData, metricsDataCache) + + // assert + if !reflect.DeepEqual(gotRes, wantRes) { + t.Errorf("TestMergePVMetricsData_mergeKubePVAndStorageInfo_Success failed, "+ + "gotRes [%v], wantRes [%v]", gotRes, wantRes) + } + if gotErr != nil { + t.Errorf("TestMergePVMetricsData_mergeKubePVAndStorageInfo_Success failed, "+ + "gotErr [%v], wantErr [%v]", gotErr, nil) + } + + // cleanup + t.Cleanup(func() { + p.Reset() + }) +} + +func TestMergePVMetricsData_mergeKubePVAndStorageInfo_GetPvDataFailed(t *testing.T) { + // arrange + ctx := context.TODO() + storageNameKey := "storageNameKey" + pvNameKey := "pvNameKey" + storageType := "storageType" + var pvCacheData []*storageGRPC.CollectDetail + metricsDataCache := &MetricsDataCache{} + mergePVMetricsData := &MergePVMetricsData{} + + getPvErr := fmt.Errorf("can not get the pv data when merge") + wantErr := getPvErr + + // action + gotRes, gotErr := mergePVMetricsData.mergeKubePVAndStorageInfo(ctx, storageNameKey, pvNameKey, + storageType, pvCacheData, metricsDataCache) + + // assert + if !reflect.DeepEqual(gotErr, wantErr) { + t.Errorf("TestMergePVMetricsData_mergeKubePVAndStorageInfo_GetPvDataFailed failed, "+ + "gotErr [%v], wantErr [%v]", gotErr, wantErr) + } + if gotRes != nil { + t.Errorf("TestMergePVMetricsData_mergeKubePVAndStorageInfo_GetPvDataFailed failed, "+ + "gotRes [%v], wantErr [%v]", gotRes, nil) + } +} + +func TestMergePVMetricsData_mergeKubePVAndStorageInfo_GetStorageDataFailed(t *testing.T) { + // arrange + ctx := context.TODO() + storageNameKey := "storageNameKey" + pvNameKey := "pvNameKey" + storageType := "storageType" + pvCacheData := []*storageGRPC.CollectDetail{{Data: map[string]string{pvNameKey: "name"}}} + metricsDataCache := &MetricsDataCache{} + mergePVMetricsData := &MergePVMetricsData{} + + getStorageErr := fmt.Errorf("can not get the storage data when merge") + wantErr := getStorageErr + + // mock + p := gomonkey.NewPatches() + p.ApplyMethod(reflect.TypeOf(metricsDataCache), "GetMetricsData", + func(_ *MetricsDataCache, metricsType string) *storageGRPC.CollectResponse { + return nil + }) + + // action + gotRes, gotErr := mergePVMetricsData.mergeKubePVAndStorageInfo(ctx, storageNameKey, pvNameKey, + storageType, pvCacheData, metricsDataCache) + + // assert + if !reflect.DeepEqual(gotErr, wantErr) { + t.Errorf("TestMergePVMetricsData_mergeKubePVAndStorageInfo_GetStorageDataFailed failed, "+ + "gotErr [%v], wantErr [%v]", gotErr, wantErr) + } + if gotRes != nil { + t.Errorf("TestMergePVMetricsData_mergeKubePVAndStorageInfo_GetPvDataFailed failed, "+ + "gotRes [%v], wantErr [%v]", gotRes, nil) + } + + // cleanup + t.Cleanup(func() { + p.Reset() + }) +} + +func TestMergePVMetricsData_getPVMergeParams_PerformanceSuccess(t *testing.T) { + // arrange + ctx := context.TODO() + indicator1 := "indicator1" + indicator2 := "indicator2" + mergePVMetricsData := &MergePVMetricsData{BaseMergeMetricsData: &BaseMergeMetricsData{ + monitorType: "performance", + mergeIndicators: []string{indicator1 + "," + indicator2}, + }} + + wantKey := "ObjectName" + wantList := []string{indicator1, indicator2} + + // action + gotKey, gotList, gotErr := mergePVMetricsData.getPVMergeParams(ctx) + + // assert + if !reflect.DeepEqual(gotKey, wantKey) { + t.Errorf("TestMergePVMetricsData_getPVMergeParams_PerformanceSuccess failed, "+ + "gotKey [%v], wantKey [%v]", gotKey, wantKey) + } + if !reflect.DeepEqual(gotList, wantList) { + t.Errorf("TestMergePVMetricsData_getPVMergeParams_PerformanceSuccess failed, "+ + "gotList [%v], wantList [%v]", gotList, wantList) + } + if gotErr != nil { + t.Errorf("TestMergePVMetricsData_getPVMergeParams_PerformanceSuccess failed, "+ + "gotErr [%v], wantErr [%v]", gotErr, nil) + } +} + +func TestMergePVMetricsData_getPVMergeParams_ObjectSuccess(t *testing.T) { + // arrange + ctx := context.TODO() + mergePVMetricsData := &MergePVMetricsData{BaseMergeMetricsData: &BaseMergeMetricsData{ + monitorType: "object", + mergeIndicators: nil, + }} + + wantKey := "NAME" + wantList := []string{"lun", "filesystem"} + + // action + gotKey, gotList, gotErr := mergePVMetricsData.getPVMergeParams(ctx) + + // assert + if !reflect.DeepEqual(gotKey, wantKey) { + t.Errorf("TestMergePVMetricsData_getPVMergeParams_ObjectSuccess failed, "+ + "gotKey [%v], wantKey [%v]", gotKey, wantKey) + } + if !reflect.DeepEqual(gotList, wantList) { + t.Errorf("TestMergePVMetricsData_getPVMergeParams_ObjectSuccess failed, "+ + "gotList [%v], wantList [%v]", gotList, wantList) + } + if gotErr != nil { + t.Errorf("TestMergePVMetricsData_getPVMergeParams_ObjectSuccess failed, "+ + "gotErr [%v], wantErr [%v]", gotErr, nil) + } +} + +func TestMergePVMetricsData_getPVMergeParams_EmptyIndicatorFailed(t *testing.T) { + // arrange + ctx := context.TODO() + mergePVMetricsData := &MergePVMetricsData{BaseMergeMetricsData: &BaseMergeMetricsData{ + backendName: "", + monitorType: "performance", + metricsType: "", + mergeIndicators: nil, + }} + emptyErr := fmt.Errorf("when get pv merge params, " + + "the monitorType is performance but mergeIndicators is empty") + wantErr := emptyErr + var wantKey string + var wantIndicators []string + + // action + gotKey, gotIndicators, gotErr := mergePVMetricsData.getPVMergeParams(ctx) + + // assert + if !reflect.DeepEqual(gotErr, wantErr) { + t.Errorf("TestMergePVMetricsData_getPVMergeParams_EmptyIndicatorFailed failed, "+ + "gotErr [%v], wantErr [%v]", gotErr, wantErr) + } + if !reflect.DeepEqual(gotKey, wantKey) { + t.Errorf("TestMergePVMetricsData_getPVMergeParams_EmptyIndicatorFailed failed, "+ + "gotKey [%v], wantKey [%v]", gotKey, wantKey) + } + if !reflect.DeepEqual(gotIndicators, wantIndicators) { + t.Errorf("TestMergePVMetricsData_getPVMergeParams_EmptyIndicatorFailed failed, "+ + "gotIndicators [%v], wantIndicators [%v]", gotIndicators, wantIndicators) + } +} + +func TestMergePVMetricsData_MergeData_Success(t *testing.T) { + // arrange + ctx := context.TODO() + pvCacheData := &BaseMetricsData{} + metricsDataCache := &MetricsDataCache{CacheDataMap: map[string]MetricsData{ + "pv": pvCacheData, + }} + mergePVMetricsData := &MergePVMetricsData{} + + // mock + p := gomonkey.NewPatches() + p.ApplyPrivateMethod(mergePVMetricsData, "getPVMergeParams", + func(ctx context.Context) (string, []string, error) { + return "performance", []string{"indicator1"}, nil + }).ApplyMethod(reflect.TypeOf(pvCacheData), "GetMetricsDataResponse", + func(_ *BaseMetricsData) *storageGRPC.CollectResponse { + return &storageGRPC.CollectResponse{ + Details: []*storageGRPC.CollectDetail{{Data: map[string]string{}}}, + } + }).ApplyPrivateMethod(mergePVMetricsData, "mergeKubePVAndStorageInfo", + func(ctx context.Context, storageNameKey, pvNameKey, storageType string, + pvCacheData []*storageGRPC.CollectDetail, metricsDataCache *MetricsDataCache) ( + map[string]map[string]string, error) { + return nil, nil + }) + + // action + gotErr := mergePVMetricsData.MergeData(ctx, metricsDataCache) + + // assert + if gotErr != nil { + t.Errorf("TestMergePVMetricsData_MergeData_Success failed, gotErr [%v], wantErr [%v]", gotErr, nil) + } + + // cleanup + t.Cleanup(func() { + p.Reset() + }) +} + +func TestMergePVMetricsData_MergeData_GetParamsFailed(t *testing.T) { + // arrange + ctx := context.TODO() + pvCacheData := &BaseMetricsData{} + metricsDataCache := &MetricsDataCache{CacheDataMap: map[string]MetricsData{ + "pv": pvCacheData, + }} + mergePVMetricsData := &MergePVMetricsData{} + getParamsErr := fmt.Errorf("get merge params err") + wantErr := getParamsErr + + // mock + p := gomonkey.NewPatches() + p.ApplyPrivateMethod(mergePVMetricsData, "getPVMergeParams", + func(ctx context.Context) (string, []string, error) { + return "", nil, getParamsErr + }) + + // action + gotErr := mergePVMetricsData.MergeData(ctx, metricsDataCache) + + // assert + if !reflect.DeepEqual(gotErr, wantErr) { + t.Errorf("TestMergePVMetricsData_MergeData_GetParamsFailed failed, "+ + "gotErr [%v], wantErr [%v]", gotErr, wantErr) + } + + // cleanup + t.Cleanup(func() { + p.Reset() + }) +} + +func TestMergePVMetricsData_MergeData_GetCacheFailed(t *testing.T) { + // arrange + ctx := context.TODO() + metricsDataCache := &MetricsDataCache{CacheDataMap: map[string]MetricsData{}} + mergePVMetricsData := &MergePVMetricsData{} + getCacheErr := fmt.Errorf("can not get pv cache data when MergePVAndStorageData") + wantErr := getCacheErr + + // mock + p := gomonkey.NewPatches() + p.ApplyPrivateMethod(mergePVMetricsData, "getPVMergeParams", + func(ctx context.Context) (string, []string, error) { + return "performance", []string{"indicator1"}, nil + }) + + // action + gotErr := mergePVMetricsData.MergeData(ctx, metricsDataCache) + + // assert + if !reflect.DeepEqual(gotErr, wantErr) { + t.Errorf("TestMergePVMetricsData_MergeData_GetCacheFailed failed, "+ + "gotErr [%v], wantErr [%v]", gotErr, wantErr) + } + + // cleanup + t.Cleanup(func() { + p.Reset() + }) +} + +func TestMergePVMetricsData_MergeData_GetMetricsFailed(t *testing.T) { + // arrange + ctx := context.TODO() + pvCacheData := &BaseMetricsData{} + metricsDataCache := &MetricsDataCache{CacheDataMap: map[string]MetricsData{ + "pv": pvCacheData, + }} + mergePVMetricsData := &MergePVMetricsData{} + getMetricsErr := fmt.Errorf("can not get MetricsDataResponse data when MergePVAndStorageData") + wantErr := getMetricsErr + + // mock + p := gomonkey.NewPatches() + p.ApplyPrivateMethod(mergePVMetricsData, "getPVMergeParams", + func(ctx context.Context) (string, []string, error) { + return "performance", []string{"indicator1"}, nil + }).ApplyMethod(reflect.TypeOf(pvCacheData), "GetMetricsDataResponse", + func(_ *BaseMetricsData) *storageGRPC.CollectResponse { + return nil + }) + + // action + gotErr := mergePVMetricsData.MergeData(ctx, metricsDataCache) + + // assert + if !reflect.DeepEqual(gotErr, wantErr) { + t.Errorf("TestMergePVMetricsData_MergeData_GetMetricsFailed failed, "+ + "gotErr [%v], wantErr [%v]", gotErr, wantErr) + } + + // cleanup + t.Cleanup(func() { + p.Reset() + }) +} + +func TestMergePVMetricsData_MergeData_GetMetricsDetailsFailed(t *testing.T) { + // arrange + ctx := context.TODO() + pvCacheData := &BaseMetricsData{} + metricsDataCache := &MetricsDataCache{CacheDataMap: map[string]MetricsData{ + "pv": pvCacheData, + }} + mergePVMetricsData := &MergePVMetricsData{} + getMetricsDetailsErr := fmt.Errorf("can not get MetricsDataResponse.Details when MergePVAndStorageData") + wantErr := getMetricsDetailsErr + + // mock + p := gomonkey.NewPatches() + p.ApplyPrivateMethod(mergePVMetricsData, "getPVMergeParams", + func(ctx context.Context) (string, []string, error) { + return "performance", []string{"indicator1"}, nil + }).ApplyMethod(reflect.TypeOf(pvCacheData), "GetMetricsDataResponse", + func(_ *BaseMetricsData) *storageGRPC.CollectResponse { + return &storageGRPC.CollectResponse{Details: nil} + }) + + // action + gotErr := mergePVMetricsData.MergeData(ctx, metricsDataCache) + + // assert + if !reflect.DeepEqual(gotErr, wantErr) { + t.Errorf("TestMergePVMetricsData_MergeData_GetMetricsDetailsFailed failed, "+ + "gotErr [%v], wantErr [%v]", gotErr, wantErr) + } + + // cleanup + t.Cleanup(func() { + p.Reset() + }) +} + +func TestMergePVMetricsData_MergeData_MergeFailed(t *testing.T) { + // arrange + ctx := context.TODO() + pvCacheData := &BaseMetricsData{} + metricsDataCache := &MetricsDataCache{CacheDataMap: map[string]MetricsData{ + "pv": pvCacheData, + }} + mergePVMetricsData := &MergePVMetricsData{} + mergeErr := fmt.Errorf("merge pv and storage info error") + wantErr := mergeErr + + // mock + p := gomonkey.NewPatches() + p.ApplyPrivateMethod(mergePVMetricsData, "getPVMergeParams", + func(ctx context.Context) (string, []string, error) { + return "performance", []string{"indicator1"}, nil + }).ApplyMethod(reflect.TypeOf(pvCacheData), "GetMetricsDataResponse", + func(_ *BaseMetricsData) *storageGRPC.CollectResponse { + return &storageGRPC.CollectResponse{ + Details: []*storageGRPC.CollectDetail{{Data: map[string]string{}}}, + } + }).ApplyPrivateMethod(mergePVMetricsData, "mergeKubePVAndStorageInfo", + func(ctx context.Context, storageNameKey, pvNameKey, storageType string, + pvCacheData []*storageGRPC.CollectDetail, metricsDataCache *MetricsDataCache) ( + map[string]map[string]string, error) { + return nil, mergeErr + }) + + // action + gotErr := mergePVMetricsData.MergeData(ctx, metricsDataCache) + + // assert + if !reflect.DeepEqual(gotErr, wantErr) { + t.Errorf("TestMergePVMetricsData_MergeData_MergeFailed failed, "+ + "gotErr [%v], wantErr [%v]", gotErr, wantErr) + } + + // cleanup + t.Cleanup(func() { + p.Reset() + }) +} diff --git a/server/prometheus-exporter/metricscache/metrics_cache.go b/server/prometheus-exporter/metricscache/metrics_cache.go new file mode 100644 index 0000000..40b6069 --- /dev/null +++ b/server/prometheus-exporter/metricscache/metrics_cache.go @@ -0,0 +1,201 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package metricscache use to save query the data of the storage metrics once +package metricscache + +import ( + "context" + "errors" + "strings" + + storageGRPC "github.com/huawei/csm/v2/grpc/lib/go/cmi" + "github.com/huawei/csm/v2/utils/log" +) + +var storageTypeMap = map[string]string{ + "oceanstor-san": "lun", + "oceanstor-nas": "filesystem", +} +var pvPerformanceMap = map[string][]string{ + "lun": {"21,22,370"}, + "filesystem": {"182,524,525"}, +} + +// MetricsDataCache save one batch data from prometheus request +type MetricsDataCache struct { + BackendName string + CacheDataMap map[string]MetricsData + MergeMetrics map[string]MergeMetricsData +} + +// GetMetricsData get the CollectResponse from storage +func (metricsDataCache *MetricsDataCache) GetMetricsData(metricsType string) *storageGRPC.CollectResponse { + if _, ok := metricsDataCache.CacheDataMap[metricsType]; !ok { + return nil + } + + return metricsDataCache.CacheDataMap[metricsType].GetMetricsDataResponse() +} + +// SetBatchDataFromSource set batch data to CacheDataMap +func (metricsDataCache *MetricsDataCache) SetBatchDataFromSource(ctx context.Context, + monitorType string, params map[string][]string) { + log.AddContext(ctx).Infoln("start to fill batch data from source") + + for collectorName, metricsIndicators := range params { + metricsData, ok := metricsDataCache.CacheDataMap[collectorName] + if !ok { + log.AddContext(ctx).Errorf("set %s cache data error, the monitorType : %s", + collectorName, monitorType) + continue + } + // if the collectorName already set we not set again + metricsDataResponse := metricsData.GetMetricsDataResponse() + if ok && metricsDataResponse != nil { + log.AddContext(ctx).Debugf("the Metrics data of %s response already get.", collectorName) + continue + } + + err := metricsData.SetMetricsData(ctx, collectorName, monitorType, metricsIndicators) + if err != nil { + log.AddContext(ctx).Errorf("set metricsData for %s error, the err is : %v", collectorName, err) + continue + } + } + log.AddContext(ctx).Infoln("fill batch data success") +} + +// MergeBatchData Merge batch data of MergeMetrics is not empty. +// use the MergeData interface to get need Merge Metrics like pv Metrics +func (metricsDataCache *MetricsDataCache) MergeBatchData(ctx context.Context) { + if len(metricsDataCache.MergeMetrics) == 0 { + return + } + log.AddContext(ctx).Infoln("start to merge metrics data") + for mergeMetricsName, mergeMetricsClass := range metricsDataCache.MergeMetrics { + err := mergeMetricsClass.MergeData(ctx, metricsDataCache) + if err != nil { + log.AddContext(ctx).Errorf("can not MergeData the mergeMetricsName is %s", mergeMetricsName) + } + } + log.AddContext(ctx).Infoln("merge metrics data success") +} + +func (metricsDataCache *MetricsDataCache) buildPVBatchParams(ctx context.Context, + monitorType string, params, batchParams map[string][]string) error { + if batchParams == nil { + batchParams = make(map[string][]string) + } + + metricsIndicators, ok := params["pv"] + if !ok { + return errors.New("not need build pv class") + } + if monitorType == "performance" && (len(metricsIndicators) == 0 || metricsIndicators[0] == "") { + log.AddContext(ctx).Errorf("the pv metricsIndicators is error") + return errors.New("not need build pv class") + } + + if monitorType == "performance" { + metricsIndicatorsList := strings.Split(metricsIndicators[0], ",") + for _, metrics := range metricsIndicatorsList { + metricsPerformance, exits := pvPerformanceMap[metrics] + if !exits { + continue + } + batchParams[metrics] = metricsPerformance + } + } else { + batchParams["lun"] = []string{""} + batchParams["filesystem"] = []string{""} + } + return nil +} + +func (metricsDataCache *MetricsDataCache) buildPVClass(ctx context.Context, + monitorType string, params, batchParams map[string][]string) { + log.AddContext(ctx).Infoln("start to build pv class") + if batchParams == nil { + batchParams = make(map[string][]string) + } + metricsIndicators, ok := params["pv"] + if !ok { + return + } + err := metricsDataCache.buildPVBatchParams(ctx, monitorType, params, batchParams) + if err != nil { + log.AddContext(ctx).Debugln("not need build pv class") + return + } + + mergeFunc, exist := mergeMetricsFactories["pv"] + if !exist { + log.AddContext(ctx).Errorf("can not get pv merge func") + return + } + + mergeDataType, err := mergeFunc(metricsDataCache.BackendName, monitorType, "pv", metricsIndicators) + if err != nil { + log.AddContext(ctx).Errorf("can not get pv mergeDataType") + return + } + metricsDataCache.MergeMetrics["pv"] = mergeDataType + log.AddContext(ctx).Infof("build pv class success with batch params: %v", batchParams) + return +} + +func (metricsDataCache *MetricsDataCache) buildStorageClass(ctx context.Context, + monitorType string, params, batchParams map[string][]string) { + log.AddContext(ctx).Infoln("start to build storage class") + if batchParams == nil { + batchParams = make(map[string][]string) + } + for collectorName, metricsIndicators := range params { + _, exist := batchParams[collectorName] + if exist { + continue + } + batchParams[collectorName] = metricsIndicators + } + + for collectorName := range batchParams { + metricsFunc, exist := metricsFactories[collectorName] + if !exist { + log.AddContext(ctx).Errorf("New metrics data error, the factories not have %s", collectorName) + continue + } + metricsData, err := metricsFunc(metricsDataCache.BackendName, collectorName) + if err != nil { + log.AddContext(ctx).Errorf("New metrics data for %s, the monitorType : %s, error: %v", + collectorName, monitorType, err) + continue + } + metricsDataCache.CacheDataMap[collectorName] = metricsData + } + log.AddContext(ctx).Infof("build storage class success with batch params: %v", batchParams) +} + +// BuildBatchDataClass get one batch cache class. use they to get metrics data +func (metricsDataCache *MetricsDataCache) BuildBatchDataClass(ctx context.Context, + monitorType string, params map[string][]string) (map[string][]string, error) { + batchParams := make(map[string][]string) + + metricsDataCache.buildPVClass(ctx, monitorType, params, batchParams) + metricsDataCache.buildStorageClass(ctx, monitorType, params, batchParams) + + return batchParams, nil +} diff --git a/server/prometheus-exporter/metricscache/metrics_cache_test.go b/server/prometheus-exporter/metricscache/metrics_cache_test.go new file mode 100644 index 0000000..64ca7fd --- /dev/null +++ b/server/prometheus-exporter/metricscache/metrics_cache_test.go @@ -0,0 +1,186 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package metricscache + +import ( + "context" + "fmt" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + + "github.com/huawei/csm/v2/grpc/lib/go/cmi" + clientSet "github.com/huawei/csm/v2/server/prometheus-exporter/clientset" +) + +func TestMetricsDataCache_GetMetricsData(t *testing.T) { + // arrange + mockCollectDetail := cmi.CollectDetail{ + Data: map[string]string{"fake_data": "test_data"}, + } + mockCollectResponse := cmi.CollectResponse{ + BackendName: "fake_backend_name", + CollectType: "fake_type", + MetricsType: "fake_collector_name", + Details: []*cmi.CollectDetail{&mockCollectDetail}, + } + mockMetricsData := BaseMetricsData{ + MetricsType: "fake_collector_name", + MetricsDataResponse: &mockCollectResponse, + } + mockpMetricsDataCache := &MetricsDataCache{ + BackendName: "fake_name", + CacheDataMap: map[string]MetricsData{ + "fake_collector_name": &mockMetricsData}, + } + + // action + got := mockpMetricsDataCache.GetMetricsData("fake_collector_name") + + // assert + if !reflect.DeepEqual(got, &mockCollectResponse) { + t.Errorf("parseStorageData() got = %v, want %v", got, "fake_data") + } +} + +func TestMetricsDataCache_SetBatchDataFromSource(t *testing.T) { + // arrange + mockStorageMetricsData := &BaseMetricsData{BackendName: "fake_backend_name"} + mockMetricsDataCache := &MetricsDataCache{ + BackendName: "fake_name", + CacheDataMap: map[string]MetricsData{"fake_metrics": mockStorageMetricsData}, + } + mockClientsSet := &clientSet.ClientsSet{ + StorageGRPCClientSet: &cmi.ClientSet{}} + ctx := context.Background() + called := false + + // mock + mock := gomonkey.NewPatches() + mock.ApplyFunc(clientSet.GetExporterClientSet, func() *clientSet.ClientsSet { + return mockClientsSet + }).ApplyPrivateMethod(mockStorageMetricsData, "GetMetricsDataResponse", + func() *cmi.CollectResponse { + return nil + }).ApplyPrivateMethod(mockStorageMetricsData, "SetMetricsData", + func(ctx context.Context, collectorName, monitorType string, metricsIndicators []string) error { + called = true + return nil + }) + + // action + mockMetricsDataCache.SetBatchDataFromSource(ctx, "fake_type", + map[string][]string{"fake_metrics": {"fake_data"}}) + // assert + if called != true { + t.Errorf("SetBatchDataFromSource() got = %v, want true", called) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestMetricsDataCache_buildPVBatchParams_PerformanceSuccess(t *testing.T) { + // arrange + metricsDataCache := &MetricsDataCache{} + ctx := context.TODO() + monitorType := "performance" + params := map[string][]string{"pv": {"lun,filesystem"}} + batchParams := make(map[string][]string) + wantRes := pvPerformanceMap + + // action + gotErr := metricsDataCache.buildPVBatchParams(ctx, monitorType, params, batchParams) + + // assert + if gotErr != nil { + t.Errorf("TestMetricsDataCache_buildPVBatchParams_PerformanceSuccess failed, "+ + "gotErr [%v], wantErr [%v]", gotErr, nil) + } + if !reflect.DeepEqual(batchParams, wantRes) { + t.Errorf("TestMetricsDataCache_buildPVBatchParams_PerformanceSuccess failed, "+ + "gotRes [%v], wantRes [%v]", batchParams, wantRes) + } + +} + +func TestMetricsDataCache_buildPVBatchParams_ObjectSuccess(t *testing.T) { + // arrange + metricsDataCache := &MetricsDataCache{} + ctx := context.TODO() + monitorType := "object" + params := map[string][]string{"pv": {}} + batchParams := make(map[string][]string) + wantRes := map[string][]string{"lun": {""}, "filesystem": {""}} + + // action + gotErr := metricsDataCache.buildPVBatchParams(ctx, monitorType, params, batchParams) + + // assert + if gotErr != nil { + t.Errorf("TestMetricsDataCache_buildPVBatchParams_ObjectSuccess failed, "+ + "gotErr [%v], wantErr [%v]", gotErr, nil) + } + if !reflect.DeepEqual(batchParams, wantRes) { + t.Errorf("TestMetricsDataCache_buildPVBatchParams_ObjectSuccess failed, "+ + "gotRes [%v], wantRes [%v]", batchParams, wantRes) + } + +} + +func TestMetricsDataCache_buildPVBatchParams_GetIndicatorsFail(t *testing.T) { + // arrange + metricsDataCache := &MetricsDataCache{} + ctx := context.TODO() + monitorType := "performance" + params := map[string][]string{} + batchParams := make(map[string][]string) + wantErr := fmt.Errorf("not need build pv class") + + // action + gotErr := metricsDataCache.buildPVBatchParams(ctx, monitorType, params, batchParams) + + // assert + if !reflect.DeepEqual(gotErr, wantErr) { + t.Errorf("TestMetricsDataCache_buildPVBatchParams_GetIndicatorsFail failed, "+ + "gotRes [%v], wantRes [%v]", gotErr, wantErr) + } + +} + +func TestMetricsDataCache_buildPVBatchParams_EmptyIndicatorsFail(t *testing.T) { + // arrange + metricsDataCache := &MetricsDataCache{} + ctx := context.TODO() + monitorType := "performance" + params := map[string][]string{"pv": {}} + batchParams := make(map[string][]string) + wantErr := fmt.Errorf("not need build pv class") + + // action + gotErr := metricsDataCache.buildPVBatchParams(ctx, monitorType, params, batchParams) + + // assert + if !reflect.DeepEqual(gotErr, wantErr) { + t.Errorf("TestMetricsDataCache_buildPVBatchParams_EmptyIndicatorsFail failed, "+ + "gotRes [%v], wantRes [%v]", gotErr, wantErr) + } + +} diff --git a/server/prometheus-exporter/metricscache/parse_pv.go b/server/prometheus-exporter/metricscache/parse_pv.go new file mode 100644 index 0000000..fd41ead --- /dev/null +++ b/server/prometheus-exporter/metricscache/parse_pv.go @@ -0,0 +1,100 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package metricscache + +import ( + "errors" + "strings" + + coreV1 "k8s.io/api/core/v1" + + exporterConfig "github.com/huawei/csm/v2/config/exporter" + storageGRPC "github.com/huawei/csm/v2/grpc/lib/go/cmi" +) + +// volumeHandle str is sbcName.storageName. So when use strings.Split, volumeHandleStrLen is 2. +const volumeHandleStrLen = 2 + +type parsePVMetrics struct { + collectDetail *storageGRPC.CollectDetail + parseError error +} + +func (pvMetrics *parsePVMetrics) setCSIDriverNameMetrics(volume coreV1.PersistentVolume) *parsePVMetrics { + if pvMetrics.parseError != nil { + return pvMetrics + } + + if volume.Spec.CSI == nil { + pvMetrics.parseError = errors.New("can not get CSIDriverName") + return pvMetrics + } + + if volume.Spec.CSI.Driver != exporterConfig.GetCSIDriverName() { + pvMetrics.parseError = errors.New("unsupported driver") + return pvMetrics + } + + pvMetrics.collectDetail.Data["driverName"] = volume.Spec.CSI.Driver + return pvMetrics +} + +func (pvMetrics *parsePVMetrics) setVolumeHandleMetrics(volume coreV1.PersistentVolume) *parsePVMetrics { + if pvMetrics.parseError != nil { + return pvMetrics + } + + if volume.Spec.CSI == nil { + pvMetrics.parseError = errors.New("can not get volumeHandle") + return pvMetrics + } + volumeHandle := volume.Spec.CSI.VolumeHandle + sbcConfigName := strings.Split(volumeHandle, ".") + if len(sbcConfigName) != volumeHandleStrLen { + pvMetrics.parseError = errors.New("can not parse volumeHandle, split error") + return pvMetrics + } + + pvMetrics.collectDetail.Data["sbcName"] = sbcConfigName[0] + pvMetrics.collectDetail.Data["storageName"] = sbcConfigName[1] + return pvMetrics +} + +func (pvMetrics *parsePVMetrics) setPVNameMetrics(volume coreV1.PersistentVolume) *parsePVMetrics { + if pvMetrics.parseError != nil { + return pvMetrics + } + + pvMetrics.collectDetail.Data["pvName"] = volume.ObjectMeta.Name + return pvMetrics +} + +func (pvMetrics *parsePVMetrics) setPVCNameMetrics(volume coreV1.PersistentVolume) *parsePVMetrics { + if pvMetrics.parseError != nil { + return pvMetrics + } + + if volume.Spec.ClaimRef == nil { + pvMetrics.parseError = errors.New("can not get PVCName") + return pvMetrics + } + + if volume.Spec.ClaimRef.Kind == "PersistentVolumeClaim" { + pvMetrics.collectDetail.Data["pvcName"] = volume.Spec.ClaimRef.Name + } + return pvMetrics +} diff --git a/server/prometheus-exporter/metricscache/pv_metrice_data.go b/server/prometheus-exporter/metricscache/pv_metrice_data.go new file mode 100644 index 0000000..5800344 --- /dev/null +++ b/server/prometheus-exporter/metricscache/pv_metrice_data.go @@ -0,0 +1,55 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package metricscache use to save query the data of the storage metrics once +package metricscache + +import ( + "context" + + "github.com/huawei/csm/v2/utils/log" +) + +// MetricsPVData save one batch data with special MetricsType, +// from prometheus request +type MetricsPVData struct { + *BaseMetricsData +} + +func init() { + RegisterMetricsData("pv", NewMetricsPVData) +} + +// NewMetricsPVData creates a new MetricsPVData with special MetricsType +func NewMetricsPVData(backendName, metricsType string) (MetricsData, error) { + return &MetricsPVData{BaseMetricsData: &BaseMetricsData{ + BackendName: backendName, MetricsType: metricsType}}, nil +} + +// SetMetricsData set pv data MetricsDataResponse +func (metricsData *MetricsPVData) SetMetricsData(ctx context.Context, + collectorName, monitorType string, metricsIndicators []string) error { + log.AddContext(ctx).Infof("start to get pv metrics data with collector name: %v, monitor type: %v, "+ + "indicators: %v", collectorName, monitorType, metricsIndicators) + // get PV data + batchCollectResponse, err := GetAndParsePVInfo(ctx, metricsData.BackendName, metricsData.MetricsType) + if err != nil { + return err + } + metricsData.MetricsDataResponse = batchCollectResponse + log.AddContext(ctx).Infoln("get pv metrics data success") + return nil +} diff --git a/server/prometheus-exporter/metricscache/pv_metrice_data_test.go b/server/prometheus-exporter/metricscache/pv_metrice_data_test.go new file mode 100644 index 0000000..4a2eabf --- /dev/null +++ b/server/prometheus-exporter/metricscache/pv_metrice_data_test.go @@ -0,0 +1,52 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package metricscache + +import ( + "context" + "testing" + + "github.com/agiledragon/gomonkey/v2" + + storageGRPC "github.com/huawei/csm/v2/grpc/lib/go/cmi" +) + +func TestMetricsPVData_SetMetricsData(t *testing.T) { + // arrange + mockPVMetricsData := &MetricsPVData{BaseMetricsData: &BaseMetricsData{BackendName: "fake_backend_name"}} + ctx := context.Background() + + // mock + mock := gomonkey.NewPatches() + mock.ApplyFunc(GetAndParsePVInfo, func(ctx context.Context, + backendName, collectType string) (*storageGRPC.CollectResponse, error) { + return nil, nil + }) + + // action + got := mockPVMetricsData.SetMetricsData(ctx, "fake_name", "object", []string{}) + + // assert + if got != nil { + t.Errorf("SetMetricsData() err got = %v, want nil", got) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} diff --git a/server/prometheus-exporter/metricscache/register_merge_metrics.go b/server/prometheus-exporter/metricscache/register_merge_metrics.go new file mode 100644 index 0000000..99f7e0d --- /dev/null +++ b/server/prometheus-exporter/metricscache/register_merge_metrics.go @@ -0,0 +1,55 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package metricscache use to save query the data of the storage metrics once +package metricscache + +import ( + "context" +) + +// a MergeMetricsData constructor +type mergeMetricsInitFunc = func(backendName, monitorType, metricsType string, + metricsIndicators []string) (MergeMetricsData, error) + +// mergeMetricsFactories are routing table with MergeMetricsData factory routing +// key is metrics name +// value is a MergeMetricsData constructor +// e.g. +// |---------------|-------------------------| +// | metricsName | collectorInitFunc | +// |---------------|-------------------------| +// | pv | NewPVMergeMetricsCache | +// |---------------|-------------------------| +var mergeMetricsFactories = make(map[string]mergeMetricsInitFunc) + +// RegisterMergeMetricsData register a MergeMetricsData constructor to factories +func RegisterMergeMetricsData(collectorName string, factory mergeMetricsInitFunc) { + mergeMetricsFactories[collectorName] = factory +} + +// MergeMetricsData is the interface for need Merge Metrics like pv +type MergeMetricsData interface { + MergeData(ctx context.Context, metricsDataCache *MetricsDataCache) error +} + +// BaseMergeMetricsData is the base for need Merge Metrics +type BaseMergeMetricsData struct { + backendName string + monitorType string + metricsType string + mergeIndicators []string +} diff --git a/server/prometheus-exporter/metricscache/register_metrics_data.go b/server/prometheus-exporter/metricscache/register_metrics_data.go new file mode 100644 index 0000000..afa8b57 --- /dev/null +++ b/server/prometheus-exporter/metricscache/register_metrics_data.go @@ -0,0 +1,67 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package metricscache use to save query the data of the storage metrics once +package metricscache + +import ( + "context" + + storageGRPC "github.com/huawei/csm/v2/grpc/lib/go/cmi" +) + +// a MetricsData constructor +type metricsCacheInitFunc = func(backendName, metricsType string) (MetricsData, error) + +// metricsFactories are routing table with MetricsData factory routing +// key is metrics name +// value is a MetricsData constructor +// e.g. +// |---------------|-------------------------| +// | metricsName | collectorInitFunc | +// |---------------|-------------------------| +// | array | NewStorageMetricsCache | +// |---------------|-------------------------| +var metricsFactories = make(map[string]metricsCacheInitFunc) + +// RegisterMetricsData register a MetricsData constructor to factories +func RegisterMetricsData(collectorName string, factory metricsCacheInitFunc) { + metricsFactories[collectorName] = factory +} + +// MetricsData set metrics data, get data from storage and kubernetes +type MetricsData interface { + GetMetricsDataResponse() *storageGRPC.CollectResponse + SetMetricsData(ctx context.Context, collectorName, monitorType string, metricsIndicators []string) error +} + +// BaseMetricsData save one batch data with special MetricsType +type BaseMetricsData struct { + BackendName string + MetricsType string + MetricsDataResponse *storageGRPC.CollectResponse +} + +// GetMetricsDataResponse implement MetricsData interface, get MetricsDataResponse +func (baseMetricsData *BaseMetricsData) GetMetricsDataResponse() *storageGRPC.CollectResponse { + return baseMetricsData.MetricsDataResponse +} + +// SetMetricsData implement MetricsData interface, set MetricsData from data source +func (baseMetricsData *BaseMetricsData) SetMetricsData( + ctx context.Context, collectorName, monitorType string, metricsIndicators []string) error { + return nil +} diff --git a/server/prometheus-exporter/metricscache/storage_metrice_data.go b/server/prometheus-exporter/metricscache/storage_metrice_data.go new file mode 100644 index 0000000..dd79a78 --- /dev/null +++ b/server/prometheus-exporter/metricscache/storage_metrice_data.go @@ -0,0 +1,107 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package metricscache use to save query the data of the storage metrics once +package metricscache + +import ( + "context" + "errors" + "strings" + + storageGRPC "github.com/huawei/csm/v2/grpc/lib/go/cmi" + clientSet "github.com/huawei/csm/v2/server/prometheus-exporter/clientset" + "github.com/huawei/csm/v2/utils/log" +) + +// StorageMetricsData save one batch data with storage MetricsType, +// from prometheus request +type StorageMetricsData struct { + *BaseMetricsData +} + +func init() { + RegisterMetricsData("array", NewStorageMetricsData) + RegisterMetricsData("controller", NewStorageMetricsData) + RegisterMetricsData("storagepool", NewStorageMetricsData) + RegisterMetricsData("filesystem", NewStorageMetricsData) + RegisterMetricsData("lun", NewStorageMetricsData) +} + +// NewStorageMetricsData new a StorageMetricsData +func NewStorageMetricsData(backendName, metricsType string) (MetricsData, error) { + return &StorageMetricsData{BaseMetricsData: &BaseMetricsData{ + BackendName: backendName, MetricsType: metricsType}}, nil +} + +func (storageMetricsData *StorageMetricsData) buildTheStorageGRPCRequest( + collectorName, monitorType string, metricsIndicators []string) *storageGRPC.CollectRequest { + batchCollectRequest := &storageGRPC.CollectRequest{ + BackendName: storageMetricsData.BackendName, + CollectType: collectorName, + MetricsType: monitorType, + Indicators: []string{}, + } + if monitorType != "performance" { + return batchCollectRequest + } + + if len(metricsIndicators) == 0 || metricsIndicators[0] == "" { + return nil + } + + batchCollectRequest.Indicators = strings.Split(metricsIndicators[0], ",") + return batchCollectRequest +} + +func (storageMetricsData *StorageMetricsData) getStorageData(ctx context.Context, + batchCollectRequest *storageGRPC.CollectRequest, + usedClientSet *clientSet.ClientsSet) (*storageGRPC.CollectResponse, error) { + storageGRPCClient := usedClientSet.StorageGRPCClientSet.CollectorClient + batchCollectResponse, err := storageGRPCClient.Collect(ctx, batchCollectRequest) + if err != nil { + log.AddContext(ctx).Warningf("can not get storage response the err is [%v]", err) + return nil, errors.New("please continue collect next data") + } + if batchCollectResponse.Details == nil { + log.AddContext(ctx).Warningln("can not get storage data") + return nil, errors.New("please continue collect next data") + } + return batchCollectResponse, nil +} + +// SetMetricsData set storage data MetricsDataResponse +func (storageMetricsData *StorageMetricsData) SetMetricsData(ctx context.Context, + collectorName, monitorType string, metricsIndicators []string) error { + log.AddContext(ctx).Infof("start to get storage metrics data with collector name: %v, monitor type: %v, "+ + "indicators: %v", collectorName, monitorType, metricsIndicators) + usedClientSet := clientSet.GetExporterClientSet() + if usedClientSet.InitError != nil { + log.AddContext(ctx).Errorln("can not get Client Set when get data") + return errors.New("can not get Client Set when get data") + } + + // get storage data + batchCollectRequest := storageMetricsData.buildTheStorageGRPCRequest( + collectorName, monitorType, metricsIndicators) + batchCollectResponse, err := storageMetricsData.getStorageData(ctx, batchCollectRequest, usedClientSet) + if err != nil { + return err + } + storageMetricsData.MetricsDataResponse = batchCollectResponse + log.AddContext(ctx).Infoln("get storage metrics data success") + return nil +} diff --git a/server/prometheus-exporter/metricscache/storage_metrice_data_test.go b/server/prometheus-exporter/metricscache/storage_metrice_data_test.go new file mode 100644 index 0000000..1f0b778 --- /dev/null +++ b/server/prometheus-exporter/metricscache/storage_metrice_data_test.go @@ -0,0 +1,44 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package metricscache + +import ( + "reflect" + "testing" + + storageGRPC "github.com/huawei/csm/v2/grpc/lib/go/cmi" +) + +func TestStorageMetricsData_buildTheStorageGRPCRequest(t *testing.T) { + // arrange + wantRequest := &storageGRPC.CollectRequest{ + BackendName: "fake_backend_name", + CollectType: "fake_collector_name", + MetricsType: "object", + Indicators: []string{}, + } + mockStorageData := StorageMetricsData{BaseMetricsData: &BaseMetricsData{BackendName: "fake_backend_name"}} + + // action + got := mockStorageData.buildTheStorageGRPCRequest( + "fake_collector_name", "object", []string{}) + + // assert + if !reflect.DeepEqual(got, wantRequest) { + t.Errorf("buildTheStorageGRPCRequest() got = [%v], want [%v]", got, wantRequest) + } +} diff --git a/storage/api/api.go b/storage/api/api.go new file mode 100644 index 0000000..282cb49 --- /dev/null +++ b/storage/api/api.go @@ -0,0 +1,60 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package api is related with storage api +package api + +import ( + "errors" + "fmt" + + "github.com/huawei/csm/v2/storage/utils" + "github.com/huawei/csm/v2/utils/log" +) + +// StorageApi is used to save url template +type StorageApi struct { + urlTemplate *utils.TextTemplate +} + +// RegisterStorageApi is used to register storage api +func RegisterStorageApi(storageApiMap map[string]string, storageApis map[string]*StorageApi) { + if storageApiMap == nil || storageApis == nil { + log.Errorln("storage api is nil") + return + } + + for name, url := range storageApiMap { + storageApi := initStorageApi(name, url) + storageApis[name] = storageApi + } +} + +// GenerateUrl is used to generate storage request url +func GenerateUrl(storageApis map[string]*StorageApi, name string, args map[string]interface{}) (string, error) { + storageApi, exist := storageApis[name] + if !exist { + msg := fmt.Sprintf("storage api url does not exist, name: %s", name) + log.Errorln(msg) + return "", errors.New(msg) + } + + return storageApi.urlTemplate.Format(args) +} + +func initStorageApi(name, url string) *StorageApi { + return &StorageApi{ + urlTemplate: utils.NewTextTemplate(name, url), + } +} diff --git a/storage/api/centralizedstorage/api_centralized.go b/storage/api/centralizedstorage/api_centralized.go new file mode 100644 index 0000000..3859086 --- /dev/null +++ b/storage/api/centralizedstorage/api_centralized.go @@ -0,0 +1,64 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package centralizedstorage is related with centralized storage api +package centralizedstorage + +import ( + "github.com/huawei/csm/v2/storage/api" +) + +var ( + storageApiMap = map[string]string{ + // system + "GetSystemInfo": "/system/", + + // filesystem + "CreateFileSystem": "/filesystem", + "GetFileSystemByName": "/filesystem?filter=NAME::{{.fsName}}&range=[0-100]", + "GetFileSystemById": "/filesystem/{{.id}}", + "GetFilesystem": "/filesystem?range=[{{.start}}-{{.end}}]", + "GetFilesystemCount": "/filesystem/count", + + // performance + "PerformanceData": "/performance_data?object_type={{.objectType}}&indicators={{.indicators}}", + "PerformanceDataPost": "/performance_data", + + // storage info + "GetStoragePools": "/storagepool", + "GetControllers": "/controller", + + // lun + "GetLuns": "/lun?range=[{{.start}}-{{.end}}]", + "GetLunCount": "/lun/count", + "GetLunByName": "/lun?filter=NAME::{{.lunName}}&range=[0-100]", + + // label + "CreatePvLabel": "/container_pv", + "DeletePvLabel": "/container_pv", + "CreatePodLabel": "/container_pod", + "DeletePodLabel": "/container_pod", + } + + storageApis = make(map[string]*api.StorageApi) +) + +func init() { + api.RegisterStorageApi(storageApiMap, storageApis) +} + +// GenerateUrl is used to generate centralized storage request url +func GenerateUrl(name string, args map[string]interface{}) (string, error) { + return api.GenerateUrl(storageApis, name, args) +} diff --git a/storage/client/centralizedstorage/client.go b/storage/client/centralizedstorage/client.go new file mode 100644 index 0000000..d26507c --- /dev/null +++ b/storage/client/centralizedstorage/client.go @@ -0,0 +1,174 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package centralizedstorage is related with storage client +package centralizedstorage + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "net/http" + "net/http/cookiejar" + "time" + + v1 "k8s.io/api/core/v1" + "k8s.io/client-go/tools/cache" + + "github.com/huawei/csm/v2/storage/client" + "github.com/huawei/csm/v2/storage/constant" + "github.com/huawei/csm/v2/storage/utils" + "github.com/huawei/csm/v2/utils/log" + "github.com/huawei/csm/v2/utils/resource" +) + +const defaultTimeout = 60 * time.Second + +// CentralizedClient is used to use centralized storage related functions +type CentralizedClient struct { + client.Client +} + +// NewCentralizedClient is used to new centralized storage client +func NewCentralizedClient(ctx context.Context, config *constant.StorageBackendConfig) (*CentralizedClient, error) { + centralizedClient := &CentralizedClient{ + Client: client.Client{ + Urls: config.Urls, + User: config.User, + SecretNamespace: config.SecretNamespace, + SecretName: config.SecretName, + StorageBackendNamespace: config.StorageBackendNamespace, + StorageBackendName: config.StorageBackendName, + Client: newHttpClient(), + Semaphore: utils.NewSemaphore(config.ClientMaxThreads), + }, + } + if err := centralizedClient.initHttpClient(ctx); err != nil { + return nil, err + } + + return centralizedClient, nil +} + +func newHttpClient() *http.Client { + jar, err := cookiejar.New(nil) + if err != nil { + log.Warningf("storage client init http client fail, error: %v", err) + } + return &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + Jar: jar, + Timeout: defaultTimeout, + } +} + +func (c *CentralizedClient) initHttpClient(ctx context.Context) error { + jar, err := cookiejar.New(nil) + if err != nil { + log.AddContext(ctx).Errorf("init http client cookiejar fail, error: %v", err) + return err + } + + certPool, skipVerify, err := c.getTlsCertConfig(ctx) + if err != nil { + return err + } + + tlsConfig := tls.Config{ + InsecureSkipVerify: skipVerify, + } + + if certPool != nil { + tlsConfig.RootCAs = certPool + } + + c.Client.Client = &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tlsConfig, + }, + Jar: jar, + Timeout: defaultTimeout, + } + + log.AddContext(ctx).Infof("init http client success, skip verify certificate: %v", skipVerify) + return nil +} + +func (c *CentralizedClient) getTlsCertConfig(ctx context.Context) (*x509.CertPool, bool, error) { + useCert, certSecret, err := c.getCertParametersFromSbcDynamically(ctx) + if err != nil { + log.AddContext(ctx).Errorf("get cert parameters from sbc error: %v", err) + return nil, true, err + } + + // judge to skip certificate verification + if !useCert { + return nil, true, nil + } + + // certSecret format is / + certSecretNameSpace, certSecretName, err := cache.SplitMetaNamespaceKey(certSecret) + if err != nil { + log.AddContext(ctx).Errorf("split cert secret error: %v", err) + return nil, true, err + } + + secret, err := resource.Instance().GetSecret(certSecretName, certSecretNameSpace) + if err != nil { + log.AddContext(ctx).Errorf("get cert secret error: %v", err) + return nil, true, err + } + + certPool, err := c.getCertPool(ctx, secret) + if err != nil { + log.AddContext(ctx).Errorf("get certificate error: %v", err) + return nil, true, err + } + + return certPool, false, nil +} + +func (c *CentralizedClient) getCertPool(ctx context.Context, secret *v1.Secret) (*x509.CertPool, error) { + log.AddContext(ctx).Infof("start get cert from secret %s/%s", secret.Namespace, secret.Name) + defer log.AddContext(ctx).Infof("end get cert from secret %s/%s", secret.Namespace, secret.Name) + + certData, exist := secret.Data[constant.CertificateKeyName] + if !exist { + msg := fmt.Sprintf("certificate not config in secret %s/%s", secret.Namespace, secret.Name) + log.AddContext(ctx).Errorln(msg) + return nil, errors.New(msg) + } + + certBlock, _ := pem.Decode(certData) + if certBlock == nil { + msg := fmt.Sprintf("certificate data decode error in secret %s/%s", secret.Namespace, secret.Name) + log.AddContext(ctx).Errorln(msg) + return nil, errors.New(msg) + } + + cert, err := x509.ParseCertificate(certBlock.Bytes) + if err != nil { + log.AddContext(ctx).Errorf("error parse certificate: %v", err) + return nil, err + } + + certPool := x509.NewCertPool() + certPool.AddCert(cert) + return certPool, nil +} diff --git a/storage/client/centralizedstorage/client_device.go b/storage/client/centralizedstorage/client_device.go new file mode 100644 index 0000000..5bff421 --- /dev/null +++ b/storage/client/centralizedstorage/client_device.go @@ -0,0 +1,40 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package centralizedstorage is related with storage client +package centralizedstorage + +import ( + "context" + + "github.com/huawei/csm/v2/storage/api/centralizedstorage" + "github.com/huawei/csm/v2/utils/log" +) + +// GetSystemInfo is used to get system info +func (c *CentralizedClient) GetSystemInfo(ctx context.Context) (map[string]interface{}, error) { + url, err := centralizedstorage.GenerateUrl("GetSystemInfo", nil) + if err != nil { + return nil, err + } + + resp, err := c.get(ctx, url, nil) + if err != nil { + log.AddContext(ctx).Errorf("storage client get system info error: %v", err) + return nil, err + } + + result, _, err := c.getResultFromResponse(ctx, resp) + return result, err +} diff --git a/storage/client/centralizedstorage/client_filesystem.go b/storage/client/centralizedstorage/client_filesystem.go new file mode 100644 index 0000000..eb74fd6 --- /dev/null +++ b/storage/client/centralizedstorage/client_filesystem.go @@ -0,0 +1,72 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package centralizedstorage is related with storage client +package centralizedstorage + +import ( + "context" + + "github.com/huawei/csm/v2/storage/api/centralizedstorage" + "github.com/huawei/csm/v2/storage/httpcode/filesystem" + "github.com/huawei/csm/v2/utils/log" +) + +// GetFileSystemByName get filesystem by name +func (c *CentralizedClient) GetFileSystemByName(ctx context.Context, name string) (map[string]interface{}, error) { + data := map[string]interface{}{ + "fsName": name, + } + + url, err := centralizedstorage.GenerateUrl("GetFileSystemByName", data) + if err != nil { + log.AddContext(ctx).Errorf("storage client get filesystem by name generate url error: %v", err) + return nil, err + } + + callFunc := func() (map[string]interface{}, *float64, error) { + resp, err := c.get(ctx, url, nil) + if err != nil { + log.AddContext(ctx).Errorf("storage client get filesystem by name error: %v", err) + return nil, nil, err + } + + return c.getResultFromResponseList(ctx, resp) + } + + return c.Client.RetryCall(ctx, filesystem.GetRetryCodes(), callFunc) +} + +// GetFileSystemIdByName get filesystem id by name +func (c *CentralizedClient) GetFileSystemIdByName(ctx context.Context, name string) (string, error) { + data, err := c.GetFileSystemByName(ctx, name) + if err != nil { + return "", err + } + id, ok := data["ID"].(string) + if !ok { + return "", nil + } + return id, nil +} + +// GetFilesystem is used to get filesystems +func (c *CentralizedClient) GetFilesystem(ctx context.Context, start, end int) ([]map[string]interface{}, error) { + return c.pageQuery(ctx, start, end, "GetFilesystem") +} + +// GetFilesystemCount used to get filesystem count +func (c *CentralizedClient) GetFilesystemCount(ctx context.Context) (int, error) { + return c.countQuery(ctx, "GetFilesystemCount") +} diff --git a/storage/client/centralizedstorage/client_filesystem_test.go b/storage/client/centralizedstorage/client_filesystem_test.go new file mode 100644 index 0000000..ff269e0 --- /dev/null +++ b/storage/client/centralizedstorage/client_filesystem_test.go @@ -0,0 +1,153 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package centralizedstorage is related with storage client +package centralizedstorage + +import ( + "context" + "fmt" + "os" + "path" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + + "github.com/huawei/csm/v2/storage/client" + "github.com/huawei/csm/v2/storage/utils" + "github.com/huawei/csm/v2/utils/log" +) + +const ( + logName string = "storage_client_test" + logDir string = "/var/log/xuanwu" +) + +// get ctx +var ctx = context.Background() + +// TestMain used for setup and teardown +func TestMain(m *testing.M) { + // init log + if err := log.InitLogging(logName); err != nil { + _ = fmt.Errorf("init logging: %s failed. error: %v", logName, err) + return + } + + m.Run() + + // remove all log file + logFile := path.Join(logDir, logName) + if err := os.RemoveAll(logFile); err != nil { + log.Errorf("Remove file: %s failed. error: %s", logFile, err) + } +} + +// TestGetFileSystemByNameThenSuccess test GetFileSystemByName() success +func TestGetFileSystemByNameThenSuccess(t *testing.T) { + response := map[string]interface{}{ + "Error": map[string]interface{}{ + "code": 0, + }, + "Data": []map[string]interface{}{{ + "test1": 1, + "test2": 2, + }}, + } + + var cli *client.Client + httpGet := gomonkey.ApplyMethod(reflect.TypeOf(cli), "Call", + func(_ *client.Client, ctx context.Context, method string, + url string, reqData map[string]interface{}) (map[string]interface{}, error) { + return response, nil + }) + defer httpGet.Reset() + + centralizedCli := &CentralizedClient{ + Client: client.Client{ + Semaphore: utils.NewSemaphore(3), + }, + } + _, err := centralizedCli.GetFileSystemByName(ctx, "nameTest") + if err != nil { + t.Errorf("GetFileSystemByName() error: %v", err) + } +} + +// TestGetFileSystemByNameWhenResponseErrorThenFailed test GetFileSystemByName() failed +func TestGetFileSystemByNameWhenResponseErrorThenFailed(t *testing.T) { + response := map[string]interface{}{ + "Error": map[string]interface{}{ + "code": -1, + }, + "Data": []map[string]interface{}{}, + } + + var cli *client.Client + httpGet := gomonkey.ApplyMethod(reflect.TypeOf(cli), "Call", + func(_ *client.Client, ctx context.Context, method string, + url string, reqData map[string]interface{}) (map[string]interface{}, error) { + return response, nil + }) + defer httpGet.Reset() + + centralizedCli := &CentralizedClient{ + Client: client.Client{ + Semaphore: utils.NewSemaphore(3), + }, + } + _, err := centralizedCli.GetFileSystemByName(ctx, "nameTest") + + expectMsg := fmt.Sprintf("storage client response httpcode is not success code, "+ + "code: %v, description: %v", -1, nil) + actualMsg := fmt.Sprintf("%v", err) + + if actualMsg != expectMsg { + t.Errorf("GetFileSystemByName() error: %v", err) + } +} + +// TestGetFileSystemByNameWhenResponseCodeNotExistThenFailed test GetFileSystemByName() failed +func TestGetFileSystemByNameWhenResponseCodeNotExistThenFailed(t *testing.T) { + response := map[string]interface{}{ + "Error": map[string]interface{}{}, + "Data": []map[string]interface{}{}, + } + + var cli *client.Client + httpGet := gomonkey.ApplyMethod(reflect.TypeOf(cli), "Call", + func(_ *client.Client, ctx context.Context, method string, + url string, reqData map[string]interface{}) (map[string]interface{}, error) { + return response, nil + }) + defer httpGet.Reset() + + centralizedCli := &CentralizedClient{ + Client: client.Client{ + Semaphore: utils.NewSemaphore(3), + }, + } + _, err := centralizedCli.GetFileSystemByName(ctx, "nameTest") + + expectMsg := fmt.Sprintf("storage client response httpcode does not exist, response: %v", &Response{ + Error: map[string]interface{}{}, + Data: []interface{}{}, + }) + actualMsg := fmt.Sprintf("%v", err) + + if actualMsg != expectMsg { + t.Errorf("GetFileSystemByName() error: %v", err) + } +} diff --git a/storage/client/centralizedstorage/client_http.go b/storage/client/centralizedstorage/client_http.go new file mode 100644 index 0000000..da207f7 --- /dev/null +++ b/storage/client/centralizedstorage/client_http.go @@ -0,0 +1,269 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package centralizedstorage is related with storage client +package centralizedstorage + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/huawei/csm/v2/storage/httpcode" + "github.com/huawei/csm/v2/utils/log" +) + +// Response is used to receive storage response when client remote call storage interfaces +type Response struct { + Error map[string]interface{} `json:"error"` + Data interface{} `json:"data,omitempty"` +} + +func (c *CentralizedClient) get(ctx context.Context, methodUrl string, + reqData map[string]interface{}) (*Response, error) { + return c.callCentralizedStorage(ctx, "GET", methodUrl, reqData) +} + +func (c *CentralizedClient) post(ctx context.Context, methodUrl string, + reqData map[string]interface{}) (*Response, error) { + return c.callCentralizedStorage(ctx, "POST", methodUrl, reqData) +} + +func (c *CentralizedClient) delete(ctx context.Context, methodUrl string, + reqData map[string]interface{}) (*Response, error) { + return c.callCentralizedStorage(ctx, "DELETE", methodUrl, reqData) +} + +func (c *CentralizedClient) put(ctx context.Context, methodUrl string, + reqData map[string]interface{}) (*Response, error) { + return c.callCentralizedStorage(ctx, "PUT", methodUrl, reqData) +} + +func (c *CentralizedClient) callCentralizedStorage(ctx context.Context, method string, + methodUrl string, reqData map[string]interface{}) (*Response, error) { + if methodUrl == "/sessions" || methodUrl == "/xx/sessions" { + return c.baseCall(ctx, method, methodUrl, reqData) + } + + response, err := c.baseCall(ctx, method, methodUrl, reqData) + if err != nil { + return c.reLoginCall(ctx, method, methodUrl, reqData) + } + + code, _ := c.checkResponseCode(ctx, response) + log.AddContext(ctx).Infof("call check response code: %v", code) + if code != nil && *code == httpcode.NoAuthentication { + log.AddContext(ctx).Infof("%v no authentication, need reLogin", code) + return c.reLoginCall(ctx, method, methodUrl, reqData) + } + + return response, err +} + +func (c *CentralizedClient) baseCall(ctx context.Context, method string, + methodUrl string, reqData map[string]interface{}) (*Response, error) { + c.Semaphore.Acquire() + defer c.Semaphore.Release() + log.AddContext(ctx).Infof("%s call semaphore: %d", c.Curl, c.Semaphore.AvailablePermits()) + + url := c.getRequestUrl(methodUrl) + response, err := c.Call(ctx, method, url, reqData) + if err != nil && strings.Contains(err.Error(), "x509") { + if err = c.initHttpClient(ctx); err != nil { + return nil, err + } + + response, err = c.Call(ctx, method, url, reqData) + } + if err != nil { + return nil, err + } + + return c.convertToCallResponse(ctx, response) +} + +func (c *CentralizedClient) reLoginCall(ctx context.Context, method string, + methodUrl string, reqData map[string]interface{}) (*Response, error) { + log.AddContext(ctx).Infof("storage client reLogin call start. method: %s, url: %s", method, methodUrl) + defer log.AddContext(ctx).Infof("storage client reLogin call success. method: %s, url: %s", method, methodUrl) + + if err := c.ReLogin(ctx); err != nil { + return nil, err + } + + return c.baseCall(ctx, method, methodUrl, reqData) +} + +func (c *CentralizedClient) getRequestUrl(methodUrl string) string { + // If the API is api/v2, need to reconstruct the request URL. The differences are as follows: + // default c.Curl is: 'https://${ip}:${port}/deviceManager/rest/${deviceId}/' + // api/v2 real url is: 'https://${ip}:${port}/api/v2/remote_execute' + if strings.HasPrefix(methodUrl, "/api/v2") { + urlSplit := strings.Split(c.Curl, "/deviceManager/rest") + if len(urlSplit) < 1 { + return c.Curl + methodUrl + } + return urlSplit[0] + methodUrl + } + + if c.DeviceId != "" && methodUrl != "/xx/sessions" { + return c.Curl + "/" + c.DeviceId + methodUrl + } + + return c.Curl + methodUrl +} + +func (c *CentralizedClient) convertToCallResponse(ctx context.Context, + response map[string]interface{}) (*Response, error) { + jsData, err := json.Marshal(response) + if err != nil { + msg := fmt.Sprintf("storage client call response to json error: %v", err) + log.AddContext(ctx).Errorln(msg) + return nil, errors.New(msg) + } + + var resp Response + err = json.Unmarshal(jsData, &resp) + if err != nil { + msg := fmt.Sprintf("storage client call json to response error: %v", err) + log.AddContext(ctx).Errorln(msg) + return nil, errors.New(msg) + } + + return &resp, nil +} + +func (c *CentralizedClient) getResultFromResponseList(ctx context.Context, + response *Response) (map[string]interface{}, *float64, error) { + respCode, err := c.checkResponseCode(ctx, response) + if err != nil { + return nil, respCode, err + } + + if response.Data == nil { + log.AddContext(ctx).Infoln("find response data is nil") + return nil, respCode, nil + } + + respData, exist := response.Data.([]interface{}) + if !exist { + msg := fmt.Sprintf( + "storage client response data can not convert to []interface{}, response data: %v", response.Data) + log.AddContext(ctx).Errorln(msg) + return nil, respCode, errors.New(msg) + } + + if len(respData) == 0 { + log.AddContext(ctx).Infoln("storage client find response data list is empty") + return nil, respCode, nil + } + + if len(respData) > 1 { + msg := fmt.Sprintf("storage client find more than one data in response data list: %v", respData) + log.AddContext(ctx).Errorf(msg) + return nil, respCode, errors.New(msg) + } + + data, exist := respData[0].(map[string]interface{}) + if !exist { + msg := fmt.Sprintf( + "storage client response data can not convert to map[string]interface{}, response data: %v", respData[0]) + log.AddContext(ctx).Errorln(msg) + return nil, respCode, errors.New(msg) + } + + return data, respCode, nil +} + +func (c *CentralizedClient) getResultListFromResponseList(ctx context.Context, + response *Response) ([]map[string]interface{}, *float64, error) { + respCode, err := c.checkResponseCode(ctx, response) + if err != nil { + return nil, respCode, err + } + + if response.Data == nil { + log.AddContext(ctx).Infoln("find response data is nil") + return nil, respCode, nil + } + + respData, exist := response.Data.([]interface{}) + if !exist { + msg := fmt.Sprintf("response data list can not convert to []interface{}, data: %v", response.Data) + log.AddContext(ctx).Errorln(msg) + return nil, respCode, errors.New(msg) + } + + if len(respData) == 0 { + log.AddContext(ctx).Infoln("find response data list is empty") + return nil, respCode, nil + } + + var resultList []map[string]interface{} + for _, data := range respData { + result, exist := data.(map[string]interface{}) + if !exist { + msg := fmt.Sprintf("response data can not convert to map[string]interface{}, data: %v", data) + log.AddContext(ctx).Errorln(msg) + return nil, respCode, errors.New(msg) + } + + resultList = append(resultList, result) + } + + return resultList, respCode, nil +} + +func (c *CentralizedClient) getResultFromResponse(ctx context.Context, + response *Response) (map[string]interface{}, *float64, error) { + respCode, err := c.checkResponseCode(ctx, response) + if err != nil { + return nil, respCode, err + } + + if response.Data == nil { + log.AddContext(ctx).Infoln("find response data is nil") + return nil, respCode, nil + } + + respData, exist := response.Data.(map[string]interface{}) + if !exist { + msg := fmt.Sprintf( + "storage client response data can not convert to map[string]interface{}, response data: %v", response.Data) + log.AddContext(ctx).Errorln(msg) + return nil, respCode, errors.New(msg) + } + + return respData, respCode, nil +} + +func (c *CentralizedClient) checkResponseCode(ctx context.Context, response *Response) (*float64, error) { + respCode, exist := response.Error["code"].(float64) + if !exist { + msg := fmt.Sprintf("storage client response httpcode does not exist, response: %v", response) + log.AddContext(ctx).Errorf(msg) + return nil, errors.New(msg) + } + + if respCode != httpcode.SuccessCode { + msg := fmt.Sprintf("storage client response httpcode is not success code, "+ + "code: %v, description: %v", respCode, response.Error["description"]) + log.AddContext(ctx).Errorf(msg) + return &respCode, errors.New(msg) + } + + return &respCode, nil +} diff --git a/storage/client/centralizedstorage/client_http_test.go b/storage/client/centralizedstorage/client_http_test.go new file mode 100644 index 0000000..1f6f687 --- /dev/null +++ b/storage/client/centralizedstorage/client_http_test.go @@ -0,0 +1,54 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package centralizedstorage is related with storage client +package centralizedstorage + +import ( + "context" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + + "github.com/huawei/csm/v2/storage/client" + "github.com/huawei/csm/v2/storage/utils" +) + +func TestGetThenSuccess(t *testing.T) { + response := map[string]interface{}{ + "Error": map[string]interface{}{ + "code": 0, + }, + "Data": []map[string]interface{}{}, + } + + var cli *client.Client + httpGet := gomonkey.ApplyMethod(reflect.TypeOf(cli), "Call", + func(_ *client.Client, ctx context.Context, method string, + url string, reqData map[string]interface{}) (map[string]interface{}, error) { + return response, nil + }) + defer httpGet.Reset() + + centralizedCli := &CentralizedClient{ + Client: client.Client{ + Semaphore: utils.NewSemaphore(3), + }, + } + _, err := centralizedCli.get(ctx, "urlTest", nil) + if err != nil { + t.Errorf("get() error: %v", err) + } +} diff --git a/storage/client/centralizedstorage/client_label.go b/storage/client/centralizedstorage/client_label.go new file mode 100644 index 0000000..1d71c03 --- /dev/null +++ b/storage/client/centralizedstorage/client_label.go @@ -0,0 +1,193 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package centralizedstorage + +import ( + "context" + "errors" + "fmt" + + "github.com/huawei/csm/v2/storage/api/centralizedstorage" + "github.com/huawei/csm/v2/storage/httpcode" + "github.com/huawei/csm/v2/storage/httpcode/label" + "github.com/huawei/csm/v2/utils/log" +) + +// PvLabelRequest create pv label request +type PvLabelRequest struct { + ResourceId string + ResourceType string + PvName string + ClusterName string +} + +// PodLabelRequest create and delete label request +type PodLabelRequest struct { + ResourceId string + ResourceType string + PodName string + NameSpace string +} + +// CreatePvLabel create pv label +func (c *CentralizedClient) CreatePvLabel(ctx context.Context, request PvLabelRequest) (map[string]interface{}, + error) { + data := map[string]interface{}{ + "resourceId": request.ResourceId, + "resourceType": request.ResourceType, + "pvName": request.PvName, + "clusterName": request.ClusterName, + } + + return c.CreateLabel(ctx, "CreatePvLabel", data, label.PvLabelExist) +} + +// DeletePvLabel delete pv label +func (c *CentralizedClient) DeletePvLabel(ctx context.Context, requestId, resourceType string) (map[string]interface{}, + error) { + data := map[string]interface{}{ + "resourceId": requestId, + "resourceType": resourceType, + } + + return c.DeleteLabel(ctx, "DeletePvLabel", data, label.PvLabelNotExist) +} + +// CreatePodLabel create pod label +func (c *CentralizedClient) CreatePodLabel(ctx context.Context, request PodLabelRequest) (map[string]interface{}, + error) { + data := map[string]interface{}{ + "resourceId": request.ResourceId, + "resourceType": request.ResourceType, + "podName": request.PodName, + "nameSpace": request.NameSpace, + } + + return c.CreateLabel(ctx, "CreatePodLabel", data, label.PodLabelExist) +} + +// DeletePodLabel delete pod label +func (c *CentralizedClient) DeletePodLabel(ctx context.Context, request PodLabelRequest) (map[string]interface{}, + error) { + data := map[string]interface{}{ + "resourceId": request.ResourceId, + "resourceType": request.ResourceType, + "podName": request.PodName, + "nameSpace": request.NameSpace, + } + + return c.DeleteLabel(ctx, "DeletePodLabel", data, label.PodLabelNotExist) +} + +// CreateLabel create label +func (c *CentralizedClient) CreateLabel(ctx context.Context, urlKey string, data map[string]interface{}, + permittedCode float64) (map[string]interface{}, error) { + + url, err := centralizedstorage.GenerateUrl(urlKey, data) + if err != nil { + log.AddContext(ctx).Errorf("create label get url failed, url: %s, error: %v", urlKey, err) + return nil, err + } + + callFunc := func() (map[string]interface{}, *float64, error) { + resp, err := c.post(ctx, url, data) + if err != nil { + log.AddContext(ctx).Errorf("create label failed, url: %s ,error: %v", urlKey, err) + return nil, nil, err + } + + return getResponse(ctx, resp, url, permittedCode) + } + + return c.Client.RetryCall(ctx, httpcode.RetryCodes, callFunc) +} + +// DeleteLabel create label +func (c *CentralizedClient) DeleteLabel(ctx context.Context, urlKey string, data map[string]interface{}, + permittedCode float64) (map[string]interface{}, error) { + + url, err := centralizedstorage.GenerateUrl(urlKey, data) + if err != nil { + log.AddContext(ctx).Errorf("delete label get url failed, url: %s, error: %v", urlKey, err) + return nil, err + } + + callFunc := func() (map[string]interface{}, *float64, error) { + resp, err := c.delete(ctx, url, data) + if err != nil { + log.AddContext(ctx).Errorf("delete label failed, url: %s error: %v", urlKey, err) + return nil, nil, err + } + + return getResponse(ctx, resp, url, permittedCode) + } + + return c.Client.RetryCall(ctx, httpcode.RetryCodes, callFunc) +} + +func getResponse(ctx context.Context, resp *Response, url string, permittedCode float64) (map[string]interface{}, + *float64, error) { + + respCode, err := getResponseCode(resp) + if err != nil { + log.AddContext(ctx).Errorf("get response code failed, url: %s, error: %v", url, err) + return nil, respCode, err + } + + if respCode != nil && *respCode == permittedCode { + log.AddContext(ctx).Infoln("The specified resource object has been associated with the current label") + *respCode = httpcode.SuccessCode + } + + if respCode != nil && *respCode != httpcode.SuccessCode { + msg := fmt.Sprintf("storage client response httpcode is not success code, "+ + "code: %v, description: %v", respCode, resp.Error["description"]) + log.AddContext(ctx).Errorf(msg) + return nil, respCode, errors.New(msg) + } + + responseData, err := getResponseData(resp) + if err != nil { + log.AddContext(ctx).Errorf("get response data failed, url: %s, error: %v", url, err) + return nil, respCode, err + } + + return responseData, respCode, nil +} + +func getResponseCode(response *Response) (*float64, error) { + respCode, exist := response.Error["code"].(float64) + if !exist { + msg := fmt.Sprintf("storage client response httpcode does not exist, response: %v", response) + return nil, errors.New(msg) + } + return &respCode, nil +} + +func getResponseData(response *Response) (map[string]interface{}, error) { + if response.Data == nil { + return map[string]interface{}{}, errors.New("response data is nil") + } + + respData, exist := response.Data.(map[string]interface{}) + if !exist { + msg := fmt.Sprintf("storage client response data can not convert to map[string]interface{},"+ + " response data: %v", response.Data) + return map[string]interface{}{}, errors.New(msg) + } + return respData, nil +} diff --git a/storage/client/centralizedstorage/client_label_test.go b/storage/client/centralizedstorage/client_label_test.go new file mode 100644 index 0000000..1c6d3f1 --- /dev/null +++ b/storage/client/centralizedstorage/client_label_test.go @@ -0,0 +1,87 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package centralizedstorage + +import ( + "context" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + + "github.com/huawei/csm/v2/storage/client" + "github.com/huawei/csm/v2/storage/httpcode/label" + "github.com/huawei/csm/v2/storage/utils" +) + +func Test_CentralizedClient_CreateLabel(t *testing.T) { + // arrange + var cli *client.Client + var centralizedCli = &CentralizedClient{ + Client: client.Client{Semaphore: utils.NewSemaphore(3)}, + } + var data = map[string]interface{}{} + + // mock + gomonkey.ApplyMethod(reflect.TypeOf(cli), "Call", + func(_ *client.Client, ctx context.Context, method string, + url string, reqData map[string]interface{}) (map[string]interface{}, error) { + return map[string]interface{}{ + "Error": map[string]interface{}{ + "code": 0, + }, + "Data": map[string]interface{}{}, + }, nil + }) + + // act + _, err := centralizedCli.CreateLabel(context.Background(), "CreatePodLabel", data, label.PodLabelExist) + + // assert + if err != nil { + t.Errorf("Test_CentralizedClient_CreateLabel() error: %v", err) + } +} + +func Test_CentralizedClient_DeleteLabel(t *testing.T) { + // arrange + var cli *client.Client + var centralizedCli = &CentralizedClient{ + Client: client.Client{Semaphore: utils.NewSemaphore(3)}, + } + var data = map[string]interface{}{} + + // mock + gomonkey.ApplyMethod(reflect.TypeOf(cli), "Call", + func(_ *client.Client, ctx context.Context, method string, + url string, reqData map[string]interface{}) (map[string]interface{}, error) { + return map[string]interface{}{ + "Error": map[string]interface{}{ + "code": 0, + }, + "Data": map[string]interface{}{}, + }, nil + }) + + // act + _, err := centralizedCli.DeleteLabel(context.Background(), "DeletePodLabel", data, label.PodLabelNotExist) + + // assert + if err != nil { + t.Errorf("Test_CentralizedClient_DeletePodLabel() error: %v", err) + } +} diff --git a/storage/client/centralizedstorage/client_lun.go b/storage/client/centralizedstorage/client_lun.go new file mode 100644 index 0000000..9265b71 --- /dev/null +++ b/storage/client/centralizedstorage/client_lun.go @@ -0,0 +1,144 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package centralizedstorage + +import ( + "context" + "errors" + "fmt" + "strconv" + + "github.com/huawei/csm/v2/storage/api/centralizedstorage" + "github.com/huawei/csm/v2/storage/httpcode" + "github.com/huawei/csm/v2/storage/httpcode/filesystem" + "github.com/huawei/csm/v2/utils/log" +) + +const ( + decimalBase = 10 + int64BitSize = 64 +) + +// GetLuns is used to get luns information +func (c *CentralizedClient) GetLuns(ctx context.Context, start, end int) ([]map[string]interface{}, error) { + return c.pageQuery(ctx, start, end, "GetLuns") +} + +// GetLunCount used to get lun count +func (c *CentralizedClient) GetLunCount(ctx context.Context) (int, error) { + return c.countQuery(ctx, "GetLunCount") +} + +// GetLunByName used to get lun by name +func (c *CentralizedClient) GetLunByName(ctx context.Context, name string) (map[string]interface{}, error) { + data := map[string]interface{}{ + "lunName": name, + } + + url, err := centralizedstorage.GenerateUrl("GetLunByName", data) + if err != nil { + log.AddContext(ctx).Errorf("storage client get lun by name generate url error: %v", err) + return nil, err + } + + callFunc := func() (map[string]interface{}, *float64, error) { + resp, err := c.get(ctx, url, nil) + if err != nil { + log.AddContext(ctx).Errorf("storage client get lun by name error: %v", err) + return nil, nil, err + } + + return c.getResultFromResponseList(ctx, resp) + } + + return c.Client.RetryCall(ctx, filesystem.GetRetryCodes(), callFunc) +} + +// GetLunIdByName used to get lun id by name +func (c *CentralizedClient) GetLunIdByName(ctx context.Context, name string) (string, error) { + data, err := c.GetLunByName(ctx, name) + if err != nil { + return "", err + } + id, ok := data["ID"].(string) + if !ok { + return "", nil + } + return id, nil +} + +// countQuery used to query count information +func (c *CentralizedClient) countQuery(ctx context.Context, urlKey string) (int, error) { + data := map[string]interface{}{} + url, err := centralizedstorage.GenerateUrl(urlKey, data) + if err != nil { + return 0, err + } + + callFunc := func() (map[string]interface{}, *float64, error) { + resp, err := c.get(ctx, url, nil) + if err != nil { + log.AddContext(ctx).Errorf("count query failed, url: %s error: %v", urlKey, err) + return nil, nil, err + } + + return c.getResultFromResponse(ctx, resp) + } + + result, err := c.Client.RetryCall(ctx, httpcode.RetryCodes, callFunc) + if err != nil { + return 0, err + } + count, ok := result["COUNT"].(string) + if !ok { + return 0, errors.New(fmt.Sprintf("%s not found count, return result is %v", urlKey, result)) + } + + parseInt, err := strconv.ParseInt(count, decimalBase, int64BitSize) + if err != nil { + return 0, err + } + + return int(parseInt), nil +} + +// pageQuery is used to page query +func (c *CentralizedClient) pageQuery(ctx context.Context, start, end int, + urlKey string) ([]map[string]interface{}, error) { + + data := map[string]interface{}{ + "start": start, + "end": end, + } + + url, err := centralizedstorage.GenerateUrl(urlKey, data) + if err != nil { + return nil, err + } + + callFunc := func() ([]map[string]interface{}, *float64, error) { + resp, err := c.get(ctx, url, nil) + if err != nil { + log.AddContext(ctx).Errorf("page query failed, url: %s error: %v", urlKey, err) + return nil, nil, err + } + + return c.getResultListFromResponseList(ctx, resp) + } + + return c.Client.RetryListCall(ctx, httpcode.RetryCodes, callFunc) +} diff --git a/storage/client/centralizedstorage/client_lun_test.go b/storage/client/centralizedstorage/client_lun_test.go new file mode 100644 index 0000000..862843c --- /dev/null +++ b/storage/client/centralizedstorage/client_lun_test.go @@ -0,0 +1,87 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package centralizedstorage + +import ( + "context" + "testing" +) + +var mockCountResponse = map[string]interface{}{ + "Error": map[string]interface{}{ + "code": 0, + }, + "Data": map[string]string{ + "COUNT": "10", + }, +} + +func TestCentralizedClient_countQuery(t *testing.T) { + httpGet := MockHttpGet(mockCountResponse) + defer httpGet.Reset() + + tests := []struct { + name string + urlKey string + }{ + { + name: "TestGetFilesystemCount", + urlKey: "GetFilesystemCount", + }, + { + name: "TestGetLunCount", + urlKey: "GetLunCount", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := centralizedCli.countQuery(context.Background(), tt.urlKey) + if err != nil { + t.Errorf("countQuery() error = %v,", err) + } + }) + } +} + +func TestCentralizedClient_pageQuery(t *testing.T) { + httpGet := MockHttpGet(mockGetresponse) + defer httpGet.Reset() + + tests := []struct { + name string + urlKey string + }{ + { + name: "TestGetFilesystem", + urlKey: "GetFilesystem", + }, + { + name: "TestGetLuns", + urlKey: "GetLuns", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := centralizedCli.pageQuery(context.Background(), 0, 100, tt.urlKey) + if err != nil { + t.Errorf("pageQuery() error = %v,", err) + } + }) + } +} diff --git a/storage/client/centralizedstorage/client_performance.go b/storage/client/centralizedstorage/client_performance.go new file mode 100644 index 0000000..8b8822c --- /dev/null +++ b/storage/client/centralizedstorage/client_performance.go @@ -0,0 +1,85 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package centralizedstorage + +import ( + "context" + "fmt" + "strings" + + "github.com/huawei/csm/v2/storage/api/centralizedstorage" + "github.com/huawei/csm/v2/storage/httpcode" + "github.com/huawei/csm/v2/utils/log" +) + +// GetPerformance query storage performance +func (c *CentralizedClient) GetPerformance(ctx context.Context, objectType int, + indicators []int) ([]map[string]interface{}, error) { + var temp = make([]string, len(indicators)) + for k, v := range indicators { + temp[k] = fmt.Sprintf("%d", v) + } + var indicatorsParam = "[" + strings.Join(temp, ",") + "]" + + data := map[string]interface{}{ + "objectType": objectType, + "indicators": indicatorsParam, + } + + url, err := centralizedstorage.GenerateUrl("PerformanceData", data) + if err != nil { + log.AddContext(ctx).Errorf("get system performance url error: %v", err) + return nil, err + } + + callFunc := func() ([]map[string]interface{}, *float64, error) { + resp, err := c.get(ctx, url, nil) + if err != nil { + log.AddContext(ctx).Errorf("get performance error: %v", err) + return nil, nil, err + } + + return c.getResultListFromResponseList(ctx, resp) + } + return c.Client.RetryListCall(ctx, httpcode.RetryCodes, callFunc) +} + +// GetPerformanceByPost query storage performance by post +func (c *CentralizedClient) GetPerformanceByPost(ctx context.Context, objectType int, + indicators []int) ([]map[string]interface{}, error) { + data := map[string]interface{}{ + "object_type": objectType, + "indicators": indicators, + } + + url, err := centralizedstorage.GenerateUrl("PerformanceDataPost", nil) + if err != nil { + log.AddContext(ctx).Errorf("get system performance url error: %v", err) + return nil, err + } + + callFunc := func() ([]map[string]interface{}, *float64, error) { + resp, err := c.post(ctx, url, data) + if err != nil { + log.AddContext(ctx).Errorf("get performance by post error: %v", err) + return nil, nil, err + } + + return c.getResultListFromResponseList(ctx, resp) + } + return c.Client.RetryListCall(ctx, httpcode.RetryCodes, callFunc) +} diff --git a/storage/client/centralizedstorage/client_performance_test.go b/storage/client/centralizedstorage/client_performance_test.go new file mode 100644 index 0000000..f7775ad --- /dev/null +++ b/storage/client/centralizedstorage/client_performance_test.go @@ -0,0 +1,82 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package centralizedstorage + +import ( + "context" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + + "github.com/huawei/csm/v2/storage/client" + "github.com/huawei/csm/v2/storage/httpcode" +) + +func TestCentralizedClient_GetPerformance(t *testing.T) { + httpGet := MockHttpGet(mockGetresponse) + defer httpGet.Reset() + + _, err := centralizedCli.GetPerformance(context.Background(), 40, []int{}) + if err != nil { + t.Errorf("GetPerformance() error = %v,", err) + } +} + +func TestCentralizedClient_GetPerformanceByPost_RetrySuccess(t *testing.T) { + // arrange + mockRetryResponse := map[string]interface{}{ + "Error": map[string]interface{}{ + "code": httpcode.RetryCodes[0], + }, + "Data": []map[string]interface{}{}, + } + + mockSuccessResponse := map[string]interface{}{ + "Error": map[string]interface{}{ + "code": 0, + }, + "Data": []map[string]interface{}{}, + } + + // mock + retryTimes := 0 + var cli *client.Client + p := gomonkey.ApplyMethod(reflect.TypeOf(cli), "Call", + func(_ *client.Client, ctx context.Context, method string, + url string, reqData map[string]interface{}) (map[string]interface{}, error) { + if retryTimes < 2 { + retryTimes++ + return mockRetryResponse, nil + } + return mockSuccessResponse, nil + }) + + // action + _, err := centralizedCli.GetPerformanceByPost(context.Background(), 40, []int{}) + + // assert + if retryTimes != 2 || err != nil { + t.Errorf("TestCentralizedClient_GetPerformanceByPost_RetrySuccess failed, "+ + "want retries = 2, actul retries = %d, want err = nil, got err = %v,", retryTimes, err) + } + + // cleanup + t.Cleanup(func() { + p.Reset() + }) +} diff --git a/storage/client/centralizedstorage/client_session.go b/storage/client/centralizedstorage/client_session.go new file mode 100644 index 0000000..2da1310 --- /dev/null +++ b/storage/client/centralizedstorage/client_session.go @@ -0,0 +1,315 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package centralizedstorage is related with storage client +package centralizedstorage + +import ( + "context" + "errors" + "fmt" + "strings" + + coreV1 "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + + "github.com/huawei/csm/v2/storage/constant" + "github.com/huawei/csm/v2/utils/log" + "github.com/huawei/csm/v2/utils/resource" +) + +// Login is used to login storage client +func (c *CentralizedClient) Login(ctx context.Context) error { + log.AddContext(ctx).Infof("storage client login start, urls: %v", c.Urls) + password, err := c.getPasswordFromSecret(ctx) + if err != nil { + return err + } + + reqData := map[string]interface{}{ + "username": c.User, + "password": string(password), + "scope": "0", + } + + for i := range password { + password[i] = 0 + } + + resp, err := c.loginCall(ctx, reqData) + reqData["password"] = "" + if err != nil { + log.AddContext(ctx).Errorf("storage client login error: %v", err) + return err + } + + respData, _, err := c.getResultFromResponse(ctx, resp) + if err != nil { + return err + } + + if err = c.checkLoginAccountState(ctx, respData); err != nil { + return err + } + + if err = c.setClientWithLoginResponseData(ctx, respData); err != nil { + log.AddContext(ctx).Errorf("storage client login set client error: %v", err) + return err + } + + log.AddContext(ctx).Infof("storage client login success, url: %s", c.Curl) + return nil +} + +// ReLogin is used to reLogin storage client +func (c *CentralizedClient) ReLogin(ctx context.Context) error { + log.AddContext(ctx).Infof("storage client reLogin start...") + defer log.AddContext(ctx).Infof("storage client reLogin success...") + + oldToken := c.Token + + c.ReLoginMutex.Lock() + defer c.ReLoginMutex.Unlock() + if c.Token != "" && oldToken != c.Token { + // other thread had already done relogin, so no need to relogin again + return nil + } + + c.Logout(ctx) + err := c.Login(ctx) + if err != nil { + log.AddContext(ctx).Errorf("storage client try to relogin error: %v", err) + return err + } + + return nil +} + +// Logout is used to logout storage client +func (c *CentralizedClient) Logout(ctx context.Context) { + log.AddContext(ctx).Infof("storage client logout start...") + defer log.AddContext(ctx).Infof("storage client logout success...") + + resp, err := c.delete(ctx, "/sessions", nil) + if err != nil { + log.AddContext(ctx).Errorf("storage client logout %s error: %v", c.Curl, err) + return + } + + _, err = c.checkResponseCode(ctx, resp) + if err != nil { + log.AddContext(ctx).Errorf("storage client logout %s error: %v", c.Curl, err) + return + } + + log.AddContext(ctx).Infof("storage client logout %s success", c.Curl) +} + +func (c *CentralizedClient) setClientWithLoginResponseData(ctx context.Context, respData map[string]interface{}) error { + var exist bool + c.DeviceId, exist = respData["deviceid"].(string) + if !exist { + msg := fmt.Sprintf( + "storage client login response deviceid: %v can not convert to string", respData["deviceid"]) + log.AddContext(ctx).Errorln(msg) + return errors.New(msg) + } + + c.Token, exist = respData["iBaseToken"].(string) + if !exist { + msg := fmt.Sprintf( + "storage client login response iBaseToken can not convert to string") + log.AddContext(ctx).Errorln(msg) + return errors.New(msg) + } + + c.VStore, exist = respData["vstoreName"].(string) + if !exist { + log.AddContext(ctx).Infof( + "storage client login response vstoreName: %v can not convert to string", respData["vstoreName"]) + } + + return nil +} + +func (c *CentralizedClient) loginCall(ctx context.Context, reqData map[string]interface{}) (*Response, error) { + for _, url := range c.Urls { + c.Curl = url + "/deviceManager/rest" + log.AddContext(ctx).Infof("storage client try to login: %s", c.Curl) + + resp, err := c.post(ctx, "/xx/sessions", reqData) + if err == nil { + return resp, err + } + + log.AddContext(ctx).Infof("storage client %s login error, going to try another url", c.Curl) + } + + return nil, errors.New("storage client all url connect error") +} + +func (c *CentralizedClient) getPasswordFromSecret(ctx context.Context) ([]byte, error) { + secret, err := resource.Instance().GetSecret(c.SecretName, c.SecretNamespace) + if err != nil && !apiErrors.IsNotFound(err) { + msg := fmt.Sprintf("storage client get secret with name %s and namespace %s failed, error: %v", + c.SecretName, c.SecretNamespace, err) + log.AddContext(ctx).Errorln(msg) + return nil, errors.New(msg) + } + + // when the sbc change the password by using oceanctl, the secret of sbc will be changed. + // in this case, need to get the latest secret from sbc. + if apiErrors.IsNotFound(err) { + log.AddContext(ctx).Infof("secret [%s/%s] not found, try to get new one from sbc dynamically", + c.SecretNamespace, c.SecretName) + + secret, err = c.getSecretFromSbcDynamically(ctx) + if err != nil { + err = fmt.Errorf("get secret from sbc dynamiclly failed, error is [%v]", err) + log.AddContext(ctx).Errorln(err) + return nil, err + } + + log.AddContext(ctx).Infof("get secret [%s/%s] from sbc dynamically", secret.Namespace, secret.Name) + } + + if secret == nil || secret.Data == nil { + msg := fmt.Sprintf("storage client get secret with name %s and namespace %s, "+ + "secret is nil or the data not exist in secret", c.SecretName, c.SecretNamespace) + log.AddContext(ctx).Errorln(msg) + return nil, errors.New(msg) + } + + password, exist := secret.Data["password"] + if !exist { + msg := fmt.Sprintf("storage client get secret with name %s and namespace %s, "+ + "password field not exist in secret data", c.SecretName, c.SecretNamespace) + log.AddContext(ctx).Errorln(msg) + return nil, errors.New(msg) + } + + return password, nil +} + +func (c *CentralizedClient) getSecretFromSbcDynamically(ctx context.Context) (*coreV1.Secret, error) { + config, err := rest.InClusterConfig() + if err != nil { + return nil, fmt.Errorf("getting cluster config error, error is [%v]", err) + } + dynamicClient, err := dynamic.NewForConfig(config) + + gvr := schema.GroupVersionResource{ + Group: "xuanwu.huawei.io", + Version: "v1", + Resource: "storagebackendclaims", + } + unstructuredResource, err := dynamicClient.Resource(gvr).Namespace(c.StorageBackendNamespace). + Get(context.TODO(), c.StorageBackendName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("get unstructuredResource of sbc [%s/%s] failed, "+ + "error is [%v]", c.StorageBackendNamespace, c.StorageBackendName, err) + } + + secretMeta, found, err := unstructured.NestedString( + unstructuredResource.UnstructuredContent(), "spec", "secretMeta") + if !found || err != nil { + return nil, fmt.Errorf("get secret meta from sbc [%s/%s] failed, "+ + "error is [%v]", c.StorageBackendNamespace, c.StorageBackendName, err) + } + + // secretMeta format is / + secretNameSpace := strings.Split(secretMeta, "/")[0] + secretName := strings.Split(secretMeta, "/")[1] + return resource.Instance().GetSecret(secretName, secretNameSpace) +} + +func (c *CentralizedClient) getCertParametersFromSbcDynamically(ctx context.Context) (bool, string, error) { + config, err := rest.InClusterConfig() + if err != nil { + return false, "", fmt.Errorf("getting cluster config error, error is [%v]", err) + } + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + return false, "", fmt.Errorf("getting dynamicClient error, error is [%v]", err) + } + + gvr := schema.GroupVersionResource{ + Group: "xuanwu.huawei.io", + Version: "v1", + Resource: "storagebackendclaims", + } + unstructuredResource, err := dynamicClient.Resource(gvr).Namespace(c.StorageBackendNamespace). + Get(context.TODO(), c.StorageBackendName, metav1.GetOptions{}) + if err != nil { + return false, "", fmt.Errorf("get unstructuredResource of sbc [%s/%s] failed, "+ + "error is [%v]", c.StorageBackendNamespace, c.StorageBackendName, err) + } + + useCert, found, err := unstructured.NestedBool( + unstructuredResource.UnstructuredContent(), "spec", "useCert") + if err != nil { + return false, "", fmt.Errorf("get isUseCert parameter from sbc [%s/%s] failed, "+ + "error is [%v]", c.StorageBackendNamespace, c.StorageBackendName, err) + } + if !found { + log.AddContext(ctx).Infof("useCert is not found, skip the cert") + return false, "", nil + } + + if !useCert { + log.AddContext(ctx).Infof("useCert is false, skip the cert") + return false, "", nil + } + + certSecret, found, err := unstructured.NestedString( + unstructuredResource.UnstructuredContent(), "spec", "certSecret") + if err != nil { + return false, "", fmt.Errorf("get certSecret parameter from sbc [%s/%s] failed, "+ + "error is [%v]", c.StorageBackendNamespace, c.StorageBackendName, err) + } + if !found { + return false, "", fmt.Errorf("get certSecret parameter from sbc [%s/%s] failed, "+ + "certSecret parameter is not found", c.StorageBackendNamespace, c.StorageBackendName) + } + + return true, certSecret, nil +} + +func (c *CentralizedClient) checkLoginAccountState(ctx context.Context, respData map[string]interface{}) error { + accountState, exist := respData["accountstate"].(float64) + if !exist { + msg := fmt.Sprintf("login response accountstate: %v can not convert to float64", + respData["accountstate"]) + log.AddContext(ctx).Errorln(msg) + return errors.New(msg) + } + + // check accountstate + if float64(constant.LoginNormal) == accountState || + float64(constant.LoginPasswordIsAboutToExpire) == accountState || + float64(constant.NextLoginPasswordMustBeChanged) == accountState || + float64(constant.LoginPasswordNeverExpires) == accountState { + log.AddContext(ctx).Infof("login valid accountstate: %s", constant.LoginAccountStateMap[accountState]) + return nil + } + + msg := fmt.Sprintf("login invalid accountstate: %s", constant.LoginAccountStateMap[accountState]) + log.AddContext(ctx).Errorln(msg) + return errors.New(msg) +} diff --git a/storage/client/centralizedstorage/client_session_test.go b/storage/client/centralizedstorage/client_session_test.go new file mode 100644 index 0000000..9a396fd --- /dev/null +++ b/storage/client/centralizedstorage/client_session_test.go @@ -0,0 +1,184 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package centralizedstorage is related with storage client +package centralizedstorage + +import ( + "context" + "errors" + "fmt" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + coreV1 "k8s.io/api/core/v1" + + "github.com/huawei/csm/v2/storage/client" + "github.com/huawei/csm/v2/storage/utils" + "github.com/huawei/csm/v2/utils/resource" +) + +// TestLoginThenSuccess test Login() then success +func TestLoginThenSuccess(t *testing.T) { + response := map[string]interface{}{ + "Error": map[string]interface{}{ + "code": 0, + }, + "Data": map[string]interface{}{ + "deviceid": "1", + "iBaseToken": "2", + "accountstate": 1, + }, + } + var cli *client.Client + call := gomonkey.ApplyMethod(reflect.TypeOf(cli), "Call", + func(_ *client.Client, ctx context.Context, method string, + url string, reqData map[string]interface{}) (map[string]interface{}, error) { + return response, nil + }) + defer call.Reset() + + secret := &coreV1.Secret{ + Data: map[string][]byte{ + "password": []byte{'1'}, + }, + } + var coreCli *resource.Client + getSecret := gomonkey.ApplyMethod(reflect.TypeOf(coreCli), "GetSecret", + func(_ *resource.Client, name string, namespace string) (*coreV1.Secret, error) { + return secret, nil + }) + defer getSecret.Reset() + + centralizedCli := &CentralizedClient{ + Client: client.Client{ + Semaphore: utils.NewSemaphore(3), + }, + } + centralizedCli.Urls = []string{"url"} + + err := centralizedCli.Login(ctx) + if err != nil { + t.Errorf("Login() error: %v", err) + } +} + +// TestLoginWhenUnConnectedThenFailed test Login() then failed +func TestLoginWhenUnConnectedThenFailed(t *testing.T) { + var cli *client.Client + httpGet := gomonkey.ApplyMethod(reflect.TypeOf(cli), "Call", + func(_ *client.Client, ctx context.Context, method string, + url string, reqData map[string]interface{}) (map[string]interface{}, error) { + return nil, errors.New("unconnected") + }) + defer httpGet.Reset() + + secret := &coreV1.Secret{ + Data: map[string][]byte{ + "password": []byte{'1'}, + }, + } + var coreCli *resource.Client + getSecret := gomonkey.ApplyMethod(reflect.TypeOf(coreCli), "GetSecret", + func(_ *resource.Client, name string, namespace string) (*coreV1.Secret, error) { + return secret, nil + }) + defer getSecret.Reset() + + centralizedCli := &CentralizedClient{ + Client: client.Client{ + Semaphore: utils.NewSemaphore(3), + }, + } + centralizedCli.Urls = []string{"url"} + + expectError := errors.New("storage client all url connect error") + actualError := centralizedCli.Login(ctx) + + if actualError == nil || actualError.Error() != expectError.Error() { + t.Errorf("Login() error: expect error: %v, actual error: %v", expectError, actualError) + } +} + +// TestLoginWhenIBaseTokenNotExistThenFailed test Login() when iBaseToken not exist then failed +func TestLoginWhenIBaseTokenNotExistThenFailed(t *testing.T) { + response := map[string]interface{}{ + "Error": map[string]interface{}{ + "code": 0, + }, + "Data": map[string]interface{}{ + "deviceid": "1", + "accountstate": 1, + }, + } + + var cli *client.Client + httpGet := gomonkey.ApplyMethod(reflect.TypeOf(cli), "Call", + func(_ *client.Client, ctx context.Context, method string, + url string, reqData map[string]interface{}) (map[string]interface{}, error) { + return response, nil + }) + defer httpGet.Reset() + + secret := &coreV1.Secret{ + Data: map[string][]byte{ + "password": []byte{'1'}, + }, + } + var coreCli *resource.Client + getSecret := gomonkey.ApplyMethod(reflect.TypeOf(coreCli), "GetSecret", + func(_ *resource.Client, name string, namespace string) (*coreV1.Secret, error) { + return secret, nil + }) + defer getSecret.Reset() + + centralizedCli := &CentralizedClient{ + Client: client.Client{ + Semaphore: utils.NewSemaphore(3), + }, + } + centralizedCli.Urls = []string{"url"} + + expectErr := fmt.Errorf( + "storage client login response iBaseToken can not convert to string") + actualErr := centralizedCli.Login(ctx) + if actualErr == nil || expectErr.Error() != actualErr.Error() { + t.Errorf("Login() error, expect error: %v, actual error: %v", expectErr, actualErr) + } +} + +// TestLogoutThenSuccess test Logout() then success +func TestLogoutThenSuccess(t *testing.T) { + response := map[string]interface{}{ + "Error": map[string]interface{}{ + "code": 0, + }, + } + + var cli *client.Client + httpGet := gomonkey.ApplyMethod(reflect.TypeOf(cli), "Call", + func(_ *client.Client, ctx context.Context, method string, + url string, reqData map[string]interface{}) (map[string]interface{}, error) { + return response, nil + }) + defer httpGet.Reset() + + centralizedCli := &CentralizedClient{ + Client: client.Client{ + Semaphore: utils.NewSemaphore(3), + }, + } + centralizedCli.Logout(ctx) +} diff --git a/storage/client/centralizedstorage/client_system.go b/storage/client/centralizedstorage/client_system.go new file mode 100644 index 0000000..a73c161 --- /dev/null +++ b/storage/client/centralizedstorage/client_system.go @@ -0,0 +1,58 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package centralizedstorage + +import ( + "context" + + "github.com/huawei/csm/v2/storage/api/centralizedstorage" + "github.com/huawei/csm/v2/storage/httpcode" + "github.com/huawei/csm/v2/utils/log" +) + +// GetStoragePools is used to get storage pools +func (c *CentralizedClient) GetStoragePools(ctx context.Context) ([]map[string]interface{}, error) { + return c.GetByUrl(ctx, "GetStoragePools") +} + +// GetControllers is used to get storage controllers +func (c *CentralizedClient) GetControllers(ctx context.Context) ([]map[string]interface{}, error) { + return c.GetByUrl(ctx, "GetControllers") +} + +// GetByUrl is used to query storage information based on a specified URL, requiring no parameters when querying +func (c *CentralizedClient) GetByUrl(ctx context.Context, urlKey string) ([]map[string]interface{}, error) { + data := map[string]interface{}{} + + url, err := centralizedstorage.GenerateUrl(urlKey, data) + if err != nil { + log.AddContext(ctx).Errorf("get url failed, url: %s, error: %v", urlKey, err) + return nil, err + } + + callFunc := func() ([]map[string]interface{}, *float64, error) { + resp, err := c.get(ctx, url, nil) + if err != nil { + log.AddContext(ctx).Errorf("get by url failed, url: %s error: %v", urlKey, err) + return nil, nil, err + } + + return c.getResultListFromResponseList(ctx, resp) + } + + return c.Client.RetryListCall(ctx, httpcode.RetryCodes, callFunc) +} diff --git a/storage/client/centralizedstorage/client_system_test.go b/storage/client/centralizedstorage/client_system_test.go new file mode 100644 index 0000000..2633ea0 --- /dev/null +++ b/storage/client/centralizedstorage/client_system_test.go @@ -0,0 +1,78 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package centralizedstorage + +import ( + "context" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + + "github.com/huawei/csm/v2/storage/client" + "github.com/huawei/csm/v2/storage/utils" +) + +var centralizedCli = &CentralizedClient{ + Client: client.Client{ + Semaphore: utils.NewSemaphore(3), + }, +} + +var mockGetresponse = map[string]interface{}{ + "Error": map[string]interface{}{ + "code": 0, + }, + "Data": []map[string]interface{}{}, +} + +func MockHttpGet(response map[string]interface{}) *gomonkey.Patches { + var cli *client.Client + return gomonkey.ApplyMethod(reflect.TypeOf(cli), "Call", + func(_ *client.Client, ctx context.Context, method string, + url string, reqData map[string]interface{}) (map[string]interface{}, error) { + return response, nil + }) +} + +func TestCentralizedClient_GetByUrl(t *testing.T) { + httpGet := MockHttpGet(mockGetresponse) + defer httpGet.Reset() + + tests := []struct { + name string + urlKey string + }{ + { + name: "TestGetStoragePools", + urlKey: "GetStoragePools", + }, + { + name: "TestGetControllers", + urlKey: "GetControllers", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := centralizedCli.GetByUrl(context.Background(), tt.urlKey) + if err != nil { + t.Errorf("GetByUrl() error = %v,", err) + } + }) + } +} diff --git a/storage/client/client.go b/storage/client/client.go new file mode 100644 index 0000000..ad740a1 --- /dev/null +++ b/storage/client/client.go @@ -0,0 +1,256 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2022-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package client is related with storage common client and operation +package client + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "strings" + "sync" + + "github.com/huawei/csm/v2/storage/utils" + "github.com/huawei/csm/v2/utils/log" +) + +const ( + // 20000 is the approximate characters number of one page + // If a log exceeds one page, it will be compressed + charLimit = 20000 + + sessionsSubStr = "/sessions" +) + +// Client is used to extract storage common attribute +type Client struct { + Curl string + Urls []string + User string + DeviceId string + Token string + VStore string + Client HttpClient + + SecretNamespace string + SecretName string + + StorageBackendNamespace string + StorageBackendName string + + ReLoginMutex sync.Mutex + Semaphore *utils.Semaphore +} + +// HttpClient is used to define http interface +type HttpClient interface { + Do(req *http.Request) (*http.Response, error) +} + +// Call is used to remote call storage interfaces +func (c *Client) Call(ctx context.Context, method string, url string, + reqData map[string]interface{}) (map[string]interface{}, error) { + if !strings.Contains(url, sessionsSubStr) { + log.AddContext(ctx).Infof("call request %s %s, request: %v", method, url, reqData) + } + log.AddContext(ctx).Infof("call reloginLock: %v", c.ReLoginMutex) + + req, err := c.getRequest(ctx, method, url, reqData) + if err != nil { + log.AddContext(ctx).Errorf( + "client http request error, method: %s, url: %s, error: %v", method, url, err) + return nil, err + } + + resp, err := c.getResponse(ctx, req) + if err != nil { + log.AddContext(ctx).Errorf("client http response error, method: %s, url: %s, error: %v", + method, url, err) + return nil, err + } + + if !strings.Contains(url, sessionsSubStr) { + responseStr := fmt.Sprintf("%v", resp) + if len(responseStr) > charLimit { + compressedStr, err := utils.CompressStr(responseStr) + if err != nil { + log.AddContext(ctx).Warningf("compress storage response fail, "+ + "the log will be printed without compression, err: %v", err) + } + log.AddContext(ctx).Infof("call response %s %s, response compressed by deflate algorithm: %s", + method, url, compressedStr) + log.AddContext(ctx).Debugf("call response %s %s, response: %s", method, url, responseStr) + } else { + log.AddContext(ctx).Infof("call response %s %s, response: %s", method, url, responseStr) + } + } + return resp, nil +} + +// RetryCall is used to retry remote call storage interfaces +func (c *Client) RetryCall(ctx context.Context, retryCodes []float64, + call func() (map[string]interface{}, *float64, error)) (map[string]interface{}, error) { + var err error + var code *float64 + var respData map[string]interface{} + + retryFunc := func() bool { + respData, code, err = call() + // if code not exist, then do not retry + if code == nil { + return false + } + + if err != nil { + if !utils.IsFloat64InList(retryCodes, *code) { + return false + } else { + log.AddContext(ctx).Infoln("storage client retry call...") + return true + } + } + + return false + } + + utils.RetryCallFunc(retryFunc) + + return respData, err +} + +// RetryListCall is used to retry remote call storage interfaces +func (c *Client) RetryListCall(ctx context.Context, retryCodes []float64, + call func() ([]map[string]interface{}, *float64, error)) ([]map[string]interface{}, error) { + var err error + var code *float64 + var respData []map[string]interface{} + + retryFunc := func() bool { + respData, code, err = call() + // if code not exist, then do not retry + if code == nil { + return false + } + + if err != nil { + if !utils.IsFloat64InList(retryCodes, *code) { + return false + } else { + log.AddContext(ctx).Infoln("storage client retry list call...") + return true + } + } + + return false + } + + utils.RetryCallFunc(retryFunc) + + return respData, err +} + +func (c *Client) getRequest(ctx context.Context, method string, url string, + reqData map[string]interface{}) (*http.Request, error) { + log.AddContext(ctx).Debugln("get request start...") + defer log.AddContext(ctx).Debugln("get request end...") + + reqBody, err := c.getRequestBody(ctx, url, reqData) + + if err != nil { + log.AddContext(ctx).Errorf("client http request body error, url: %s, error: %s", url, err.Error()) + return nil, err + } + + req, err := c.newRequest(ctx, method, url, reqBody) + if err != nil { + log.AddContext(ctx).Errorf("client http new request error, url: %s, error: %s", url, err.Error()) + return nil, err + } + + return req, nil +} + +func (c *Client) getResponse(ctx context.Context, req *http.Request) (map[string]interface{}, error) { + log.AddContext(ctx).Debugln("get response start...") + defer log.AddContext(ctx).Debugln("get response end...") + + clientResp, err := c.Client.Do(req) + if err != nil { + log.AddContext(ctx).Errorf("client http response error, error: %v", err) + return nil, err + } + defer clientResp.Body.Close() + + log.AddContext(ctx).Debugln("start read response body...") + body, err := ioutil.ReadAll(clientResp.Body) + if err != nil { + log.AddContext(ctx).Errorf("client read response body error: %v", err) + return nil, err + } + log.AddContext(ctx).Debugln("read response body success...") + + var resp map[string]interface{} + err = json.Unmarshal(body, &resp) + if err != nil { + log.AddContext(ctx).Errorf("client response body convert response error, body: %s, error: %v", body, err) + return nil, err + } + + return resp, nil +} + +func (c *Client) getRequestBody(ctx context.Context, url string, reqData map[string]interface{}) (io.Reader, error) { + log.AddContext(ctx).Debugln("get request body start...") + defer log.AddContext(ctx).Debugln("get request body end...") + + if reqData == nil { + return nil, nil + } + + reqBytes, err := json.Marshal(reqData) + if err != nil { + if strings.Contains(url, sessionsSubStr) { + log.AddContext(ctx).Errorf("client http request body error: %v", err) + } else { + log.AddContext(ctx).Errorf("client http request body error, data: %v, error: %v", reqData, err) + } + + return nil, err + } + + return bytes.NewReader(reqBytes), nil +} + +func (c *Client) newRequest(ctx context.Context, method string, reqUrl string, + reqBody io.Reader) (*http.Request, error) { + req, err := http.NewRequest(method, reqUrl, reqBody) + if err != nil { + log.AddContext(ctx).Errorf("client http new request error: %s", err.Error()) + return req, err + } + + req.Header.Set("Connection", "keep-alive") + req.Header.Set("Content-Type", "application/json") + + if c.Token != "" { + req.Header.Set("iBaseToken", c.Token) + } + + return req, nil +} diff --git a/storage/client/client_test.go b/storage/client/client_test.go new file mode 100644 index 0000000..a2f062a --- /dev/null +++ b/storage/client/client_test.go @@ -0,0 +1,138 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package client is related with storage common client and operation +package client + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "net/http/cookiejar" + "os" + "path" + "reflect" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + + "github.com/huawei/csm/v2/utils/log" +) + +const ( + logName string = "storage_client_test" + logDir string = "/var/log/xuanwu" +) + +// get ctx +var ctx = context.Background() + +// TestMain used for setup and teardown +func TestMain(m *testing.M) { + // init log + if err := log.InitLogging(logName); err != nil { + _ = fmt.Errorf("init logging: %s failed. error: %v", logName, err) + return + } + + m.Run() + + // remove all log file + logFile := path.Join(logDir, logName) + if err := os.RemoveAll(logFile); err != nil { + log.Errorf("Remove file: %s failed. error: %s", logFile, err) + } +} + +// TestCloneFileSystemThenSuccess test Call() success +func TestCallThenSuccess(t *testing.T) { + resp := map[string]interface{}{ + "code": "0", + } + js, err := json.Marshal(resp) + if err != nil { + t.Errorf("Call() error: %v", err) + } + httpCli := &http.Client{} + httpCLiDo := gomonkey.ApplyMethod(reflect.TypeOf(httpCli), "Do", + func(_ *http.Client, req *http.Request) (*http.Response, error) { + return &http.Response{ + Body: ioutil.NopCloser(bytes.NewReader(js)), + }, nil + }) + defer httpCLiDo.Reset() + + jar, err := cookiejar.New(nil) + if err != nil { + t.Errorf("Call() error: %v", err) + } + cli := &Client{ + Client: &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + Jar: jar, + Timeout: 60 * time.Second, + }, + } + _, err = cli.Call(ctx, "method", "url", map[string]interface{}{}) + if err != nil { + t.Errorf("Call() error: %v", err) + } +} + +func TestClient_getRequest_JsonMarshalFailed(t *testing.T) { + // arrange + jar, err := cookiejar.New(nil) + if err != nil { + t.Errorf("Call() error: %v", err) + } + + cli := &Client{ + Client: &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + Jar: jar, + Timeout: 60 * time.Second, + }, + } + reqData := map[string]interface{}{"username": "test_user", "password": "123456"} + wantErr := errors.New("mock err") + + // mock + p := gomonkey.NewPatches() + p.ApplyFunc(json.Marshal, func(v any) ([]byte, error) { + return nil, wantErr + }) + + // action + _, gotErr := cli.getRequest(ctx, "method", sessionsSubStr, reqData) + + // assert + if !reflect.DeepEqual(gotErr, wantErr) { + t.Errorf("TestClient_getRequest_JsonMarshalFailed failed, want err = %v, get err = %v", wantErr, gotErr) + } + + // cleanup + t.Cleanup(func() { + p.Reset() + }) +} diff --git a/storage/constant/constants.go b/storage/constant/constants.go new file mode 100644 index 0000000..794ef17 --- /dev/null +++ b/storage/constant/constants.go @@ -0,0 +1,37 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package constant is related with storage client constant +package constant + +// StorageBackendConfig contains storage standard info +type StorageBackendConfig struct { + StorageType string + Urls []string + Pools []string + User string + + SecretNamespace string + SecretName string + + StorageBackendNamespace string + StorageBackendName string + + ClientMaxThreads int +} + +const ( + // CertificateKeyName refer to certificate config key name + CertificateKeyName = "tls.crt" +) diff --git a/storage/constant/enum_login.go b/storage/constant/enum_login.go new file mode 100644 index 0000000..19648bb --- /dev/null +++ b/storage/constant/enum_login.go @@ -0,0 +1,49 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package constant is related with storage client constant +package constant + +// AccountState is login account state +type AccountState float64 + +// login account state +const ( + LoginNormal AccountState = 1 + LoginPasswordExpired AccountState = 3 + LoginInitialPassword AccountState = 4 + LoginPasswordIsAboutToExpire AccountState = 5 + NextLoginPasswordMustBeChanged AccountState = 6 + LoginPasswordNeverExpires AccountState = 7 + LoginAuthenticateEmailAddress AccountState = 8 + LoginPasswordNeedInitialized AccountState = 9 + LoginAuthenticateRadius AccountState = 10 + LoginChallengeRadiusResponse AccountState = 11 +) + +var ( + // LoginAccountStateMap is login account state map + LoginAccountStateMap = map[float64]string{ + 1: "normal", + 3: "password expired", + 4: "initial password, which must be reset", + 5: "The password is about to expire", + 6: "The password must be changed upon the next login", + 7: "The password never expires", + 8: "one-time password for authenticating the email address", + 9: "The device is in the first login state and the password needs to be initialized", + 10: "RADIUS one-time password authentication is required", + 11: "RADIUS challenge response is required", + } +) diff --git a/storage/httpcode/filesystem/filesystem.go b/storage/httpcode/filesystem/filesystem.go new file mode 100644 index 0000000..331ceb5 --- /dev/null +++ b/storage/httpcode/filesystem/filesystem.go @@ -0,0 +1,38 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package filesystem is used to list filesystem related api response code +package filesystem + +import "github.com/huawei/csm/v2/storage/httpcode" + +const ( + operatorFail float64 = -1 + + // FileSystemNotExist means file system not exist + FileSystemNotExist float64 = 1073752065 + + // FileSystemExist means file system exist + FileSystemExist float64 = 1077948993 + + // CloneFileSystemNotEmpty means clone file system not empty + CloneFileSystemNotEmpty float64 = 1073844244 +) + +var retryCodes = []float64{operatorFail, httpcode.SystemBusy1, httpcode.SystemBusy2} + +// GetRetryCodes is used to get api response code which can retry call api +func GetRetryCodes() []float64 { + return retryCodes +} diff --git a/storage/httpcode/httpcode.go b/storage/httpcode/httpcode.go new file mode 100644 index 0000000..eeb310d --- /dev/null +++ b/storage/httpcode/httpcode.go @@ -0,0 +1,30 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package httpcode is related with http call response code +package httpcode + +const ( + // SuccessCode means call api success + SuccessCode float64 = 0 + // SystemBusy1 means call system busy + SystemBusy1 float64 = 1077949006 + // SystemBusy2 means call system busy + SystemBusy2 float64 = 1077948995 + // NoAuthentication means no authentication + NoAuthentication float64 = -401 +) + +// RetryCodes means these code need to retry +var RetryCodes = []float64{SystemBusy1, SystemBusy2} diff --git a/storage/httpcode/label/label.go b/storage/httpcode/label/label.go new file mode 100644 index 0000000..6e8b30b --- /dev/null +++ b/storage/httpcode/label/label.go @@ -0,0 +1,32 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package label contains labels constant +package label + +const ( + // PvLabelExist means pv label exist + PvLabelExist float64 = 1073754416 + + // PvLabelNotExist means pv label not exist + PvLabelNotExist float64 = 1073754399 + + // PodLabelExist means pv pod exist + PodLabelExist float64 = 1073754414 + + // PodLabelNotExist means pv pod not exist + PodLabelNotExist float64 = 1073754412 +) diff --git a/storage/utils/client.go b/storage/utils/client.go new file mode 100644 index 0000000..cac4e93 --- /dev/null +++ b/storage/utils/client.go @@ -0,0 +1,46 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package utils is related with storage client utils +package utils + +import ( + "flag" + "time" +) + +const ( + maxRetryNumber = 5 + sleepTime = 2 * time.Second +) + +var storageClientMaxRetryTimes = flag.Int("storage-client-max-retry-times", maxRetryNumber, "maximum number of retries") +var storageClientRetryInterval = flag.Duration("storage-client-retry-interval", sleepTime, "retry interval") + +// RetryCallFunc is used to retry call func +// the func return true will retry call func, return false will end call +func RetryCallFunc(retryFunc func() bool) { + retryNumber := 0 + for { + if shouldRetry := retryFunc(); !shouldRetry { + break + } + + if retryNumber++; retryNumber > *storageClientMaxRetryTimes { + break + } + + time.Sleep(*storageClientRetryInterval) + } +} diff --git a/storage/utils/list.go b/storage/utils/list.go new file mode 100644 index 0000000..bcae745 --- /dev/null +++ b/storage/utils/list.go @@ -0,0 +1,61 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package utils is related with storage client utils +package utils + +import ( + "bytes" + "compress/flate" + "fmt" +) + +// IsFloat64InList is used to check float64 element in list +func IsFloat64InList(list []float64, element float64) bool { + for _, v := range list { + if element == v { + return true + } + } + + return false +} + +// CleanBytes is used to clean bytes memory +func CleanBytes(bytes []byte) { + for i := 0; i < len(bytes); i++ { + bytes[i] = 0 + } +} + +// CompressStr compress long string by deflate algorithm +// The result will be hexadecimal encoded +// The encoded result can be reverted to original str by DeCompressStr +func CompressStr(str string) (string, error) { + var buf bytes.Buffer + w, err := flate.NewWriter(&buf, flate.BestCompression) + defer w.Close() + if err != nil { + return str, err + } + _, err = w.Write([]byte(str)) + if err != nil { + return str, err + } + err = w.Flush() + if err != nil { + return str, err + } + return fmt.Sprintf("%x", buf.Bytes()), nil +} diff --git a/storage/utils/semaphore.go b/storage/utils/semaphore.go new file mode 100644 index 0000000..86d7ef6 --- /dev/null +++ b/storage/utils/semaphore.go @@ -0,0 +1,45 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package utils is related with storage client utils +package utils + +// Semaphore stores semaphore info +type Semaphore struct { + permits int + channel chan int +} + +// NewSemaphore is used to new semaphore +func NewSemaphore(permits int) *Semaphore { + return &Semaphore{ + channel: make(chan int, permits), + permits: permits, + } +} + +// Acquire is used to get semaphore +func (s *Semaphore) Acquire() { + s.channel <- 0 +} + +// Release is used to remove semaphore +func (s *Semaphore) Release() { + <-s.channel +} + +// AvailablePermits get available permits +func (s *Semaphore) AvailablePermits() int { + return s.permits - len(s.channel) +} diff --git a/storage/utils/template.go b/storage/utils/template.go new file mode 100644 index 0000000..c5aa8ff --- /dev/null +++ b/storage/utils/template.go @@ -0,0 +1,50 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package utils is related with storage client utils +package utils + +import ( + "errors" + "strings" + "text/template" + + "github.com/huawei/csm/v2/utils/log" +) + +// TextTemplate is for text template +type TextTemplate struct { + text *template.Template +} + +// Format generate a text with args +func (t *TextTemplate) Format(args map[string]interface{}) (string, error) { + str := new(strings.Builder) + err := t.text.Execute(str, args) + if err != nil { + log.Errorln("format text template error") + return "", errors.New("format text template error") + } + return str.String(), nil +} + +// NewTextTemplate is used to create text template +func NewTextTemplate(templateName string, str string) *TextTemplate { + temp, err := template.New(templateName).Parse(str) + if err != nil { + log.Errorln("init text template error, name: %s, str: %s", templateName, str) + return nil + } + return &TextTemplate{text: temp} +} diff --git a/utils/log/console.go b/utils/log/console.go new file mode 100644 index 0000000..67f5307 --- /dev/null +++ b/utils/log/console.go @@ -0,0 +1,69 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package log output logged entries to respective logging hooks +package log + +import ( + "fmt" + "io" + "os" + + "github.com/sirupsen/logrus" +) + +// ConsoleHook sends log entries to stdout/stderr. +type ConsoleHook struct { + formatter logrus.Formatter +} + +// newConsoleHook creates a new log hook for writing to stdout/stderr. +func newConsoleHook(logFormat logrus.Formatter) (*ConsoleHook, error) { + return &ConsoleHook{formatter: logFormat}, nil +} + +// Levels returns all supported levels +func (hook *ConsoleHook) Levels() []logrus.Level { + return logrus.AllLevels +} + +// Fire ensure logging of respective log entries +func (hook *ConsoleHook) Fire(entry *logrus.Entry) error { + + // Determine output stream + var logWriter io.Writer + switch entry.Level { + case logrus.DebugLevel, logrus.InfoLevel, logrus.WarnLevel: + logWriter = os.Stdout + case logrus.ErrorLevel, logrus.FatalLevel: + logWriter = os.Stderr + default: + return fmt.Errorf("unknown log level: %v", entry.Level) + } + + lineBytes, err := hook.formatter.Format(entry) + if err != nil { + _, printErr := fmt.Fprintf(os.Stderr, "Unable to read entry, %v", err) + if printErr != nil { + return fmt.Errorf("print os.Stderr failed: %v", printErr) + } + return err + } + + if _, err := logWriter.Write(lineBytes); err != nil { + return err + } + + return nil +} diff --git a/utils/log/file.go b/utils/log/file.go new file mode 100644 index 0000000..db958e1 --- /dev/null +++ b/utils/log/file.go @@ -0,0 +1,308 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package log output logged entries to respective logging hooks +package log + +import ( + "flag" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/sirupsen/logrus" +) + +const ( + logFilePermission = 0640 + backupFilePermission = 0440 + logDirectoryPermission = 0750 + defaultFileSize = 1024 * 1024 * 20 // 20M + + backupTimeFormat = "20060102-150405" + defaultMaxBackups = 9 + + decimalBase = 10 + int64BitSize = 64 +) + +var ( + logFileSizeThreshold = flag.String("log-file-size", + strconv.Itoa(defaultFileSize), + "Maximum file size before log rotation") + maxBackups = flag.Uint("max-backups", + defaultMaxBackups, + "maximum number of backup log file") +) + +// FileHook sends log entries to a file. +type FileHook struct { + logFileHandle *fileHandler + logRotationThreshold int64 + formatter logrus.Formatter + logRotateMutex *sync.Mutex +} + +// ensure interface implementation +var _ flushable = &FileHook{} +var _ closable = &FileHook{} + +// newFileHook creates a new log hook for writing to a file. +func newFileHook(logFilePath string, logFormat logrus.Formatter) (*FileHook, error) { + logFileRootDir := filepath.Dir(logFilePath) + dir, err := os.Lstat(logFileRootDir) + if os.IsNotExist(err) { + if err := os.MkdirAll(logFileRootDir, logDirectoryPermission); err != nil { + return nil, fmt.Errorf("could not create log directory %v. %v", logFileRootDir, err) + } + } + if dir != nil && !dir.IsDir() { + return nil, fmt.Errorf("log path %v exists and is not a directory, please remove it", logFileRootDir) + } + + filesizeThreshold, err := getNumInByte() + if err != nil { + return nil, fmt.Errorf("error in evaluating max log file size: %v. Check 'logFileSize' flag", err) + } + + return &FileHook{ + logRotationThreshold: filesizeThreshold, + formatter: logFormat, + logFileHandle: newFileHandler(logFilePath), + logRotateMutex: &sync.Mutex{}, + }, nil +} + +// Close file handler +func (hook *FileHook) close() { + // All writes are synced and no file descriptor are left to close with current implementation +} + +// Flush commits the current contents of the file +func (hook *FileHook) flush() { + // All writes are synced and no file descriptor are left to close with current implementation +} + +// Levels returns all supported levels +func (hook *FileHook) Levels() []logrus.Level { + return logrus.AllLevels +} + +// Fire ensure logging of respective log entries +func (hook *FileHook) Fire(entry *logrus.Entry) error { + // Get formatted entry + lineBytes, err := hook.formatter.Format(entry) + if err != nil { + return fmt.Errorf("could not read log entry. %v", err) + } + + // Write log entry to file + _, err = hook.logFileHandle.writeString(string(lineBytes)) + if err != nil { + // let logrus print error message + return fmt.Errorf("write log message [%s] error. %v", lineBytes, err) + } + + // Rotate the file as needed + if err = hook.maybeDoLogfileRotation(); err != nil { + return err + } + + return nil +} + +// checkNeedsRotation checks to see if a file has grown too large +func (hook *FileHook) checkNeedsRotation() bool { + fileInfo, err := hook.logFileHandle.stat() + if err != nil { + return false + } + + return fileInfo.Size() >= hook.logRotationThreshold +} + +// maybeDoLogfileRotation check and perform log rotation +func (hook *FileHook) maybeDoLogfileRotation() error { + if hook.checkNeedsRotation() { + hook.logRotateMutex.Lock() + defer hook.logRotateMutex.Unlock() + + if hook.checkNeedsRotation() { + // Do the rotation. + err := hook.logFileHandle.rotate() + if err != nil { + return err + } + } + } + + return nil +} + +type fileHandler struct { + rwLock *sync.RWMutex + filePath string +} + +func newFileHandler(logFilePath string) *fileHandler { + return &fileHandler{ + filePath: logFilePath, + } +} + +func (f *fileHandler) stat() (os.FileInfo, error) { + return os.Stat(f.filePath) +} + +func (f *fileHandler) writeString(s string) (int, error) { + file, err := os.OpenFile(f.filePath, os.O_CREATE|os.O_APPEND|os.O_RDWR, logFilePermission) + if err != nil { + return 0, fmt.Errorf("failed to open log file with error [%v]", err) + } + defer file.Close() + return file.WriteString(s) +} + +func (f *fileHandler) rotate() error { + // Do the rotation. + rotatedLogFileLocation := f.filePath + time.Now().Format(backupTimeFormat) + if err := os.Rename(f.filePath, rotatedLogFileLocation); err != nil { + return fmt.Errorf("failed to create backup file. %v", err) + } + if err := os.Chmod(rotatedLogFileLocation, backupFilePermission); err != nil { + return fmt.Errorf("failed to chmod backup file. %s", err) + } + // try to remove old backup files + backupFiles, err := f.sortedBackupLogFiles() + if err != nil { + return err + } + + if *maxBackups < uint(len(backupFiles)) { + oldBackupFiles := backupFiles[*maxBackups:] + + for _, file := range oldBackupFiles { + err := os.Remove(filepath.Join(filepath.Dir(f.filePath), file.Name())) + if err != nil { + return fmt.Errorf("failed to remove old backup file [%s]. %v", file.Name(), err) + } + } + } + return nil +} + +type logFileInfo struct { + timestamp time.Time + os.FileInfo +} + +func (f *fileHandler) sortedBackupLogFiles() ([]logFileInfo, error) { + files, err := ioutil.ReadDir(filepath.Dir(f.filePath)) + if err != nil { + return nil, fmt.Errorf("can't read log file directory: %v", err) + } + + logFiles := make([]logFileInfo, 0) + baseLogFileName := filepath.Base(f.filePath) + + // take out log files from directory + for _, f := range files { + if f.IsDir() { + continue + } + // ignore files other than log file and current log file itself + fileName := f.Name() + if !strings.HasPrefix(fileName, baseLogFileName) || fileName == baseLogFileName { + continue + } + + timestamp, err := time.Parse(backupTimeFormat, fileName[len(baseLogFileName):]) + if err != nil { + logrus.Warningf("Failed parsing log file suffix timestamp. %v", err) + continue + } + + logFiles = append(logFiles, logFileInfo{timestamp: timestamp, FileInfo: f}) + } + + sort.Sort(byTimeFormat(logFiles)) + + return logFiles, nil +} + +type byTimeFormat []logFileInfo + +func (by byTimeFormat) isOutBounds(i, j int) bool { + return i >= len(by) || j >= len(by) +} + +func (by byTimeFormat) Less(i, j int) bool { + if by.isOutBounds(i, j) { + return false + } + + return by[i].timestamp.After(by[j].timestamp) +} + +func (by byTimeFormat) Swap(i, j int) { + if by.isOutBounds(i, j) { + return + } + + by[i], by[j] = by[j], by[i] +} + +func (by byTimeFormat) Len() int { + return len(by) +} + +func getNumInByte() (int64, error) { + var sum int64 = 0 + var err error + + maxDataNum := strings.ToUpper(*logFileSizeThreshold) + lastLetter := maxDataNum[len(maxDataNum)-1:] + + // 1.最后一位是M + // 1.1 获取M前面的数字 * 1024 * 1024 + // 2.最后一位是K + // 2.1 获取K前面的数字 * 1024 + // 3.最后一位是数字或者B + // 3.1 若最后一位是数字，则直接返回 若最后一位是B，则获取前面的数字返回 + if lastLetter >= "0" && lastLetter <= "9" { + sum, err = strconv.ParseInt(maxDataNum, decimalBase, int64BitSize) + if err != nil { + return 0, err + } + } else { + sum, err = strconv.ParseInt(maxDataNum[:len(maxDataNum)-1], decimalBase, int64BitSize) + if err != nil { + return 0, err + } + + if lastLetter == "M" { + sum *= 1024 * 1024 + } else if lastLetter == "K" { + sum *= 1024 + } + } + + return sum, nil +} diff --git a/utils/log/logger.go b/utils/log/logger.go new file mode 100644 index 0000000..0148e90 --- /dev/null +++ b/utils/log/logger.go @@ -0,0 +1,427 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package log output logged entries to respective logging hooks +package log + +import ( + "bytes" + "context" + "crypto/rand" + "flag" + "fmt" + "io/ioutil" + "os" + + "github.com/sirupsen/logrus" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" +) + +var ( + logger LoggingInterface + + loggingModule = flag.String("logging-module", + "file", + "Flag enable one of available logging module (file, console)") + logLevel = flag.String("log-level", + "info", + "Set logging level (debug, info, error, warning, fatal)") + logFileDir = flag.String("log-file-dir", + defaultLogDir, + "The flag to specify logging directory. The flag is only supported if logging module is file") +) + +type key string + +const ( + defaultLogDir = "/var/log/xuanwu" + timestampFormat = "2006-01-02 15:04:05.000000" + + xuanWuRequestID key = "xuanwu.requestid" + xuanwuChainRequestID = "xuanwu-chain-requestid" + requestID = "requestID" +) + +// LoggingInterface is an interface exposes logging functionality +type LoggingInterface interface { + Logger + + flushable + + closable + + AddContext(ctx context.Context) Logger +} + +// GetXuanWuRequestID get xuanWuRequestID +func GetXuanWuRequestID() key { + return xuanWuRequestID +} + +// Closable is an interface for closing logging streams. +// The interface should be implemented by hooks. +type closable interface { + close() +} + +// Flushable is an interface to commit current content of logging stream +type flushable interface { + flush() +} + +// Logger exposes logging functionality +type Logger interface { + Debugf(format string, args ...interface{}) + + Debugln(args ...interface{}) + + Infof(format string, args ...interface{}) + + Infoln(args ...interface{}) + + Warningf(format string, args ...interface{}) + + Warningln(args ...interface{}) + + Errorf(format string, args ...interface{}) + + Errorln(args ...interface{}) + + Fatalf(format string, args ...interface{}) + + Fatalln(args ...interface{}) + + AddField(field string, value interface{}) Logger +} + +type loggerImpl struct { + *logrus.Entry + hooks []logrus.Hook + formatter logrus.Formatter +} + +var _ LoggingInterface = &loggerImpl{} + +func parseLogLevel() (logrus.Level, error) { + switch *logLevel { + case "debug": + return logrus.DebugLevel, nil + case "info": + return logrus.InfoLevel, nil + case "warning": + return logrus.WarnLevel, nil + case "error": + return logrus.ErrorLevel, nil + case "fatal": + return logrus.FatalLevel, nil + default: + return logrus.FatalLevel, fmt.Errorf("invalid logging level [%v]", logLevel) + } +} + +// Default init function for log module +func init() { + *loggingModule = "console" + err := InitLogging("dummy-name") + if err != nil { + logrus.Fatalf("Failed to initialize logging module") + } +} + +// InitLogging configures logging. Logs are written to a log file or stdout/stderr. +// Since logrus doesn't support multiple writers, each log stream is implemented as a hook. +func InitLogging(logName string) error { + var tmpLogger loggerImpl + tmpLogger.Entry = new(logrus.Entry) + + // initialize logrus in wrapper + tmpLogger.Logger = logrus.New() + + // No output except for the hooks + tmpLogger.Logger.SetOutput(ioutil.Discard) + + // set logging level + level, err := parseLogLevel() + if err != nil { + return err + } + tmpLogger.Logger.SetLevel(level) + + // initialize log formatter + formatter := &PlainTextFormatter{TimestampFormat: timestampFormat, pid: os.Getpid()} + + hooks := make([]logrus.Hook, 0) + switch *loggingModule { + case "file": + logFilePath := fmt.Sprintf("%s/%s", *logFileDir, logName) + // Write to the log file + logFileHook, err := newFileHook(logFilePath, formatter) + if err != nil { + return fmt.Errorf("could not initialize logging to file: %v", err) + } + hooks = append(hooks, logFileHook) + case "console": + // Write to stdout/stderr + logConsoleHook, err := newConsoleHook(formatter) + if err != nil { + return fmt.Errorf("could not initialize logging to console: %v", err) + } + hooks = append(hooks, logConsoleHook) + default: + return fmt.Errorf("invalid logging module [%v]. Support only 'file' or 'console'", loggingModule) + } + + tmpLogger.hooks = hooks + for _, hook := range tmpLogger.hooks { + // initialize logrus with hooks + tmpLogger.Logger.AddHook(hook) + } + + logger = &tmpLogger + return nil +} + +// PlainTextFormatter is a formatter to ensure formatted logging output +type PlainTextFormatter struct { + // TimestampFormat to use for display when a full timestamp is printed + TimestampFormat string + + // process identity number + pid int +} + +var _ logrus.Formatter = &PlainTextFormatter{} + +// Format ensure unified and formatted logging output +func (f *PlainTextFormatter) Format(entry *logrus.Entry) ([]byte, error) { + b := entry.Buffer + if entry.Buffer == nil { + b = &bytes.Buffer{} + } + + _, _ = fmt.Fprintf(b, "%s %d", entry.Time.Format(f.TimestampFormat), f.pid) + if len(entry.Data) != 0 { + for key, value := range entry.Data { + _, _ = fmt.Fprintf(b, "[%s:%v] ", key, value) + } + } + + _, _ = fmt.Fprintf(b, "%s %s\n", getLogLevel(entry.Level), entry.Message) + + return b.Bytes(), nil +} + +func getLogLevel(level logrus.Level) string { + switch level { + case logrus.DebugLevel: + return "[DEBUG]: " + case logrus.InfoLevel: + return "[INFO]: " + case logrus.WarnLevel: + return "[WARNING]: " + case logrus.ErrorLevel: + return "[ERROR]: " + case logrus.FatalLevel: + return "[FATAL]: " + default: + return "[UNKNOWN]: " + } +} + +// Debugf ensures output of formatted debug logs +func Debugf(format string, args ...interface{}) { + logger.Debugf(format, args...) +} + +// Debugln ensures output of Debug logs +func Debugln(args ...interface{}) { + logger.Debugln(args...) +} + +// Infof ensures output of formatted info logs +func Infof(format string, args ...interface{}) { + logger.Infof(format, args...) +} + +// Infoln ensures output of info logs +func Infoln(args ...interface{}) { + logger.Infoln(args...) +} + +// Warningf ensures output of formatted warning logs +func Warningf(format string, args ...interface{}) { + logger.Warningf(format, args...) +} + +// Warningln ensures output of warning logs +func Warningln(args ...interface{}) { + logger.Warningln(args...) +} + +// Errorf ensures output of formatted error logs +func Errorf(format string, args ...interface{}) { + logger.Errorf(format, args...) +} + +// Errorln ensures output of error logs +func Errorln(args ...interface{}) { + logger.Errorln(args...) +} + +// Fatalf ensures output of formatted fatal logs +func Fatalf(format string, args ...interface{}) { + logger.Fatalf(format, args...) +} + +// Fatalln ensures output of fatal logs +func Fatalln(args ...interface{}) { + logger.Fatalln(args...) +} + +// AddContext ensures appending context info in log +func AddContext(ctx context.Context) Logger { + return logger.AddContext(ctx) +} + +// AddField add value into field +func AddField(field string, value interface{}) Logger { + return logger.AddField(field, value) +} + +func (logger *loggerImpl) flush() { + for _, hook := range logger.hooks { + flushable, ok := hook.(flushable) + if ok { + flushable.flush() + } + } +} + +func (logger *loggerImpl) close() { + for _, hook := range logger.hooks { + flushable, ok := hook.(closable) + if ok { + flushable.close() + } + } +} + +// AddContext ensures appending context info in log +func (logger *loggerImpl) AddContext(ctx context.Context) Logger { + if ctx.Value(xuanWuRequestID) == nil { + return logger + } + return logger.AddField(requestID, ctx.Value(xuanWuRequestID)) +} + +// AddField ensures appending field info in log +func (logger *loggerImpl) AddField(field string, value interface{}) Logger { + entry := logger.WithFields(logrus.Fields{ + field: value, + }) + return &loggerImpl{ + Entry: entry, + hooks: logger.hooks, + formatter: logger.formatter, + } +} + +// EnsureGRPCContext ensures adding request id in incoming unary grpc context +func EnsureGRPCContext(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, + handler grpc.UnaryHandler) (interface{}, error) { + newCtx, err := HandleRequestId(ctx) + if err != nil { + return handler(ctx, req) + } + + return handler(newCtx, req) +} + +type serverStreamWithContext struct { + grpc.ServerStream + ctx context.Context +} + +// Context implement context func of serverStreamWithContext +func (ss serverStreamWithContext) Context() context.Context { + return ss.ctx +} + +// NewServerStreamWithContext returns a new serverStreamWithContext +func NewServerStreamWithContext(stream grpc.ServerStream, ctx context.Context) grpc.ServerStream { + return serverStreamWithContext{ + ServerStream: stream, + ctx: ctx, + } +} + +// EnsureStreamGRPCContext ensures adding request id in incoming stream grpc context +func EnsureStreamGRPCContext(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, + handler grpc.StreamHandler) (err error) { + ctx := stream.Context() + newCtx, err := HandleRequestId(ctx) + if err != nil { + return handler(srv, NewServerStreamWithContext(stream, ctx)) + } + + return handler(srv, NewServerStreamWithContext(stream, newCtx)) +} + +// SetRequestInfo is used to set the context with requestID value +func SetRequestInfo(ctx context.Context) (context.Context, error) { + randomID, err := rand.Prime(rand.Reader, 32) + if err != nil { + Errorf("Failed in random ID generation for GRPC request ID logging: [%v]", err) + return ctx, err + } + + // the requestID value in metadata can be transferred between services via grpc + ctx = metadata.AppendToOutgoingContext(ctx, xuanwuChainRequestID, randomID.String()) + return context.WithValue(ctx, xuanWuRequestID, randomID.String()), nil +} + +// HandleRequestId is used to handle the requestId when the context is transferred between services via grpc +func HandleRequestId(ctx context.Context) (context.Context, error) { + md, ok := metadata.FromIncomingContext(ctx) + // if ctx metadata not exist, generate one with metadata and value + if !ok { + Debugln("ctx not include metadata info, generate a new ctx with metadata and value") + return SetRequestInfo(ctx) + } + + // If ctx metadata exist and metadata includes xuanwuRequestId info, + // then return ctx with value and append requestId to metadata again. + // When the service acts as a new client and connects to other server, + // the requestID information needs to be added to metadata again. + // So, the requestId can be transferred between multiple services. + if reqIDs, ok := md[xuanwuChainRequestID]; ok && len(reqIDs) == 1 { + ctx = metadata.AppendToOutgoingContext(ctx, xuanwuChainRequestID, reqIDs[0]) + return context.WithValue(ctx, xuanWuRequestID, reqIDs[0]), nil + } + + // if ctx metadata exist, but metadata not include requestId info, generate one with metadata and value + Debugln("ctx metadata not include requestId info, generate a new ctx with metadata and value") + return SetRequestInfo(ctx) +} + +// Flush ensures to commit current content of logging stream +func Flush() { + logger.flush() +} + +// Close ensures closing output stream +func Close() { + logger.close() +} diff --git a/utils/log/logger_test.go b/utils/log/logger_test.go new file mode 100644 index 0000000..b63f334 --- /dev/null +++ b/utils/log/logger_test.go @@ -0,0 +1,32 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package log output logged entries to respective logging hooks +package log + +import ( + "context" + "testing" +) + +// TestSetRequestInfo test SetRequestInfo function +func TestSetRequestInfo(t *testing.T) { + t.Run("Test set request info", func(t *testing.T) { + _, err := SetRequestInfo(context.TODO()) + if err != nil { + t.Errorf("SetRequestInfo() error = %v", err) + return + } + }) +} diff --git a/utils/resource/configmap.go b/utils/resource/configmap.go new file mode 100644 index 0000000..99aee18 --- /dev/null +++ b/utils/resource/configmap.go @@ -0,0 +1,62 @@ +/* +Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package resource used to access the k8s core resource by API +package resource + +import ( + "context" + + coreV1 "k8s.io/api/core/v1" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ConfigmapOps can get information of configmap +type ConfigmapOps interface { + // CreateConfigmap creates the given configmap + CreateConfigmap(*coreV1.ConfigMap) (*coreV1.ConfigMap, error) + // GetConfigmap gets the configmap object given its name and namespace + GetConfigmap(name, namespace string) (*coreV1.ConfigMap, error) + // UpdateConfigmap update the configmap object given its name and namespace + UpdateConfigmap(*coreV1.ConfigMap) (*coreV1.ConfigMap, error) +} + +// CreateConfigmap creates the given configmap +func (c *Client) CreateConfigmap(configmap *coreV1.ConfigMap) (*coreV1.ConfigMap, error) { + if err := c.initClient(); err != nil { + return nil, err + } + + return c.kubernetes.CoreV1().ConfigMaps(configmap.Namespace).Create( + context.TODO(), configmap, metaV1.CreateOptions{}) +} + +// GetConfigmap gets the configmap object given its name and namespace +func (c *Client) GetConfigmap(name, namespace string) (*coreV1.ConfigMap, error) { + if err := c.initClient(); err != nil { + return nil, err + } + + return c.kubernetes.CoreV1().ConfigMaps(namespace).Get(context.TODO(), name, metaV1.GetOptions{}) +} + +// UpdateConfigmap update the given configmap +func (c *Client) UpdateConfigmap(configmap *coreV1.ConfigMap) (*coreV1.ConfigMap, error) { + if err := c.initClient(); err != nil { + return nil, err + } + + return c.kubernetes.CoreV1().ConfigMaps(configmap.Namespace).Update( + context.TODO(), configmap, metaV1.UpdateOptions{}) +} diff --git a/utils/resource/core.go b/utils/resource/core.go new file mode 100644 index 0000000..808bb21 --- /dev/null +++ b/utils/resource/core.go @@ -0,0 +1,127 @@ +/* +Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package resource is used to obtain core resources in Kubernetes. +package resource + +import ( + "fmt" + "os" + "sync" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/record" +) + +var ( + instance Ops + once sync.Once +) + +// Ops includes k8s options +type Ops interface { + SecretOps + PodOps + ConfigmapOps + PVOps +} + +// Instance returns a singleton instance of the client. +func Instance() Ops { + once.Do(func() { + if instance == nil { + instance = &Client{} + } + }) + return instance +} + +// WatchFunc is a callback provided to the Watch functions +// which is invoked when the given object is changed. +type WatchFunc func(object runtime.Object) error + +// Client is a wrapper for kubernetes core client. +type Client struct { + config *rest.Config + kubernetes kubernetes.Interface + // eventRecorders is a map of component to event recorders + eventRecorders map[string]record.EventRecorder + eventRecordersLock sync.Mutex + eventBroadcaster record.EventBroadcaster +} + +// initClient the k8s client if uninitialized +func (c *Client) initClient() error { + if c.kubernetes != nil { + return nil + } + + return c.setClient() +} + +// setClient instantiates a client. +func (c *Client) setClient() error { + var err error + + if c.config != nil { + err = c.loadClient() + } else { + kubeconfig := os.Getenv("KUBECONFIG") + if len(kubeconfig) > 0 { + err = c.loadClientFromKubeconfig(kubeconfig) + } else { + err = c.loadClientFromServiceAccount() + } + + } + return err +} + +func (c *Client) loadClient() error { + if c.config == nil { + return fmt.Errorf("rest config is not provided") + } + + var err error + + c.kubernetes, err = kubernetes.NewForConfig(c.config) + if err != nil { + return err + } + return nil +} + +// loadClientFromServiceAccount loads a k8s client from a ServiceAccount specified in the pod running px +func (c *Client) loadClientFromServiceAccount() error { + config, err := rest.InClusterConfig() + if err != nil { + return err + } + + c.config = config + return c.loadClient() +} + +func (c *Client) loadClientFromKubeconfig(kubeconfig string) error { + config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + return err + } + + c.config = config + return c.loadClient() +} diff --git a/utils/resource/pods.go b/utils/resource/pods.go new file mode 100644 index 0000000..b8397b2 --- /dev/null +++ b/utils/resource/pods.go @@ -0,0 +1,49 @@ +/* +Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package resource is used to obtain core resources in Kubernetes. +package resource + +import ( + "context" + + coreV1 "k8s.io/api/core/v1" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// PodOps can get information of pod +type PodOps interface { + GetPodListFilterByNamespace(namespace string, listOptions metaV1.ListOptions) ( + *coreV1.PodList, error) + GetPodByNameSpaceAndName(namespace, name string, getOptions metaV1.GetOptions) ( + *coreV1.Pod, error) +} + +// GetPodListFilterByNamespace gets pod list from k8s cluster by namespace and listOption +func (c *Client) GetPodListFilterByNamespace(namespace string, listOptions metaV1.ListOptions) ( + *coreV1.PodList, error) { + if err := c.initClient(); err != nil { + return nil, err + } + return c.kubernetes.CoreV1().Pods(namespace).List(context.TODO(), listOptions) +} + +// GetPodByNameSpaceAndName gets pod from k8s cluster by namespace, name and listOption +func (c *Client) GetPodByNameSpaceAndName(namespace, name string, getOptions metaV1.GetOptions) ( + *coreV1.Pod, error) { + if err := c.initClient(); err != nil { + return nil, err + } + return c.kubernetes.CoreV1().Pods(namespace).Get(context.TODO(), name, getOptions) +} diff --git a/utils/resource/pv.go b/utils/resource/pv.go new file mode 100644 index 0000000..4e0073c --- /dev/null +++ b/utils/resource/pv.go @@ -0,0 +1,49 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package resource is used to obtain core resources in Kubernetes. +package resource + +import ( + "context" + + coreV1 "k8s.io/api/core/v1" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// PVOps is related with pv +type PVOps interface { + // GetPV get pv by given name + GetPV(name string) (*coreV1.PersistentVolume, error) + // ListPV get pv by options + ListPV(options metaV1.ListOptions) (*coreV1.PersistentVolumeList, error) +} + +// GetPV get pv by given name +func (c *Client) GetPV(name string) (*coreV1.PersistentVolume, error) { + if err := c.initClient(); err != nil { + return nil, err + } + + return c.kubernetes.CoreV1().PersistentVolumes().Get(context.TODO(), name, metaV1.GetOptions{}) +} + +// ListPV get pv by options +func (c *Client) ListPV(options metaV1.ListOptions) (*coreV1.PersistentVolumeList, error) { + if err := c.initClient(); err != nil { + return nil, err + } + + return c.kubernetes.CoreV1().PersistentVolumes().List(context.TODO(), options) +} diff --git a/utils/resource/secrets.go b/utils/resource/secrets.go new file mode 100644 index 0000000..5fadae5 --- /dev/null +++ b/utils/resource/secrets.go @@ -0,0 +1,38 @@ +/* +Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package resource is used to obtain core resources in Kubernetes. +package resource + +import ( + "context" + + coreV1 "k8s.io/api/core/v1" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// SecretOps can get information of secret +type SecretOps interface { + // GetSecret gets the secrets object given its name and namespace + GetSecret(name string, namespace string) (*coreV1.Secret, error) +} + +// GetSecret gets the secrets object given its name and namespace +func (c *Client) GetSecret(name string, namespace string) (*coreV1.Secret, error) { + if err := c.initClient(); err != nil { + return nil, err + } + + return c.kubernetes.CoreV1().Secrets(namespace).Get(context.TODO(), name, metaV1.GetOptions{}) +} diff --git a/utils/version/constants.go b/utils/version/constants.go new file mode 100644 index 0000000..0a28dff --- /dev/null +++ b/utils/version/constants.go @@ -0,0 +1,35 @@ +/* +Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package version used to set and clean the service version +package version + +var ( + buildVersion string + buildArch string +) + +var ( + common = buildVersion + // OSArch the architecture of service + OSArch = buildArch + // ContainerMonitorInterfaceVersion the version of service containerMonitorInterface + ContainerMonitorInterfaceVersion = common + // CsmLivenessProbeVersion the version of service csm-liveness-probe + CsmLivenessProbeVersion = common + // CsmTopoServiceVersion the version of service csm-topo-service + CsmTopoServiceVersion = common + // CsmPrometheusCollectorVersion the version of service csm-prometheus-collector + CsmPrometheusCollectorVersion = common +) diff --git a/utils/version/version.go b/utils/version/version.go new file mode 100644 index 0000000..160cb94 --- /dev/null +++ b/utils/version/version.go @@ -0,0 +1,94 @@ +/* +Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package version used to set and clean the service version +package version + +import ( + "errors" + "fmt" + "os" + "sync" + "time" + + coreV1 "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + + "github.com/huawei/csm/v2/utils/log" + "github.com/huawei/csm/v2/utils/resource" +) + +var mutex sync.Mutex + +// InitVersionConfigMapWithName used for init the version configmap of the service with a configmap name +func InitVersionConfigMapWithName(containerName string, version string, namespaceEnv string, + defaultNamespace string, cmName string) error { + log.Infof("Init version is %s, osArch is %s", version, OSArch) + + namespace := os.Getenv(namespaceEnv) + if namespace == "" { + namespace = defaultNamespace + } + + mutex.Lock() + defer mutex.Unlock() + + cm, err := resource.Instance().GetConfigmap(cmName, namespace) + if apiErrors.IsNotFound(err) { + err = createConfigMap(containerName, version, namespace, cmName) + if err != nil { + return err + } + } else if err != nil { + errMsg := fmt.Sprintf("get configMap err: %s", err) + return errors.New(errMsg) + } + + for true { + cm, err = resource.Instance().GetConfigmap(cmName, namespace) + if err != nil { + errMsg := fmt.Sprintf("get configMap err: %s", err) + return errors.New(errMsg) + } + + if cm.Data == nil { + cm.Data = make(map[string]string) + } + cm.Data[containerName] = version + cm, err = resource.Instance().UpdateConfigmap(cm) + if err != nil && apiErrors.IsConflict(err) { + time.Sleep(time.Second) + continue + } else if err != nil { + errMsg := fmt.Sprintf("update configMap err: %s", err) + return errors.New(errMsg) + } + break + } + return nil +} + +func createConfigMap(containerName, version, namespace, cmName string) error { + cm := &coreV1.ConfigMap{} + cm.Name = cmName + cm.Namespace = namespace + cm.Data = make(map[string]string) + cm.Data[containerName] = version + _, err := resource.Instance().CreateConfigmap(cm) + if err != nil && !apiErrors.IsAlreadyExists(err) { + errMsg := fmt.Sprintf("create configMap err: %s", err) + return errors.New(errMsg) + } + return nil +}