From d97bc3245f2f370deb658ccf5e54fcdc38e5929d Mon Sep 17 00:00:00 2001 From: Serge Logvinov Date: Sun, 31 Dec 2023 18:11:25 +0200 Subject: [PATCH] feat: pv/pvc cli helper Implement very common tools to work with Proxmox CSI disks. Like: * migrate - helps to move block devices from one Proxmox node to another. * rename - helps to change name of pvc. Signed-off-by: Serge Logvinov --- .goreleaser.yml | 51 ++++++++ Dockerfile | 30 ++++- Makefile | 29 +++-- cmd/pvecsi-mutate/main.go | 103 ++++++++++++++++ cmd/pvecsi-mutate/migrate.go | 224 +++++++++++++++++++++++++++++++++++ cmd/pvecsi-mutate/rename.go | 171 ++++++++++++++++++++++++++ cmd/pvecsi-mutate/utils.go | 144 ++++++++++++++++++++++ go.mod | 3 + go.sum | 10 ++ pkg/csi/driver.go | 2 +- pkg/log/log.go | 82 +++++++++++++ pkg/proxmox/volume.go | 100 ++++++++++++++++ pkg/tools/config.go | 48 ++++++++ pkg/tools/nodes.go | 86 ++++++++++++++ pkg/tools/pv.go | 101 ++++++++++++++++ 15 files changed, 1168 insertions(+), 16 deletions(-) create mode 100644 .goreleaser.yml create mode 100644 cmd/pvecsi-mutate/main.go create mode 100644 cmd/pvecsi-mutate/migrate.go create mode 100644 cmd/pvecsi-mutate/rename.go create mode 100644 cmd/pvecsi-mutate/utils.go create mode 100644 pkg/log/log.go create mode 100644 pkg/proxmox/volume.go create mode 100644 pkg/tools/config.go create mode 100644 pkg/tools/nodes.go create mode 100644 pkg/tools/pv.go diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 0000000..2534b64 --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,51 @@ +project_name: pvecsi-mutate + +before: + hooks: + - go mod download + - make release-update + +dist: bin +builds: + - dir: cmd/pvecsi-mutate + binary: pvecsi-mutate-{{ .Os }}-{{ .Arch }} + no_unique_dist_dir: true + env: + - CGO_ENABLED=0 + goos: + - linux + - darwin + goarch: + - amd64 + - arm64 + +checksum: + name_template: "checksums.txt" + +snapshot: + name_template: edge + +dockers: + - use: buildx + image_templates: + - ghcr.io/sergelogvinov/pvecsi-mutate:{{ .Version }}-amd64 + goos: linux + goarch: amd64 + build_flag_templates: + - "--label=org.opencontainers.image.version={{.Version}}" + - "--target=pvecsi-mutate-goreleaser" + - "--platform=linux/amd64" + - use: buildx + image_templates: + - ghcr.io/sergelogvinov/pvecsi-mutate:{{ .Version }}-arm64 + goos: linux + goarch: arm64 + build_flag_templates: + - "--label=org.opencontainers.image.version={{.Version}}" + - "--target=pvecsi-mutate-goreleaser" + - "--platform=linux/arm64" +docker_manifests: + - name_template: ghcr.io/sergelogvinov/{{ .ProjectName }}:{{ .Version }} + image_templates: + - ghcr.io/sergelogvinov/{{ .ProjectName }}:{{ .Version }}-amd64 + - ghcr.io/sergelogvinov/{{ .ProjectName }}:{{ .Version }}-arm64 diff --git a/Dockerfile b/Dockerfile index d176279..7ae51ac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN go mod download ######################################## -FROM --platform=${BUILDPLATFORM} golang:1.21.4-alpine3.18 AS builder +FROM --platform=${BUILDPLATFORM} golang:1.21.6-alpine3.18 AS builder RUN apk update && apk add --no-cache make ENV GO111MODULE on WORKDIR /src @@ -24,7 +24,7 @@ RUN make build-all-archs ######################################## -FROM --platform=${TARGETARCH} scratch AS controller +FROM --platform=${TARGETARCH} scratch AS proxmox-csi-controller LABEL org.opencontainers.image.source="https://github.com/sergelogvinov/proxmox-csi-plugin" \ org.opencontainers.image.licenses="Apache-2.0" \ org.opencontainers.image.description="Proxmox VE CSI plugin" @@ -65,7 +65,7 @@ RUN /tools/deps-check.sh ######################################## -FROM --platform=${TARGETARCH} scratch AS node +FROM --platform=${TARGETARCH} scratch AS proxmox-csi-node LABEL org.opencontainers.image.source="https://github.com/sergelogvinov/proxmox-csi-plugin" \ org.opencontainers.image.licenses="Apache-2.0" \ org.opencontainers.image.description="Proxmox VE CSI plugin" @@ -77,3 +77,27 @@ ARG TARGETARCH COPY --from=builder /src/bin/proxmox-csi-node-${TARGETARCH} /bin/proxmox-csi-node ENTRYPOINT ["/bin/proxmox-csi-node"] + +######################################## + +FROM alpine:3.18 AS pvecsi-mutate +LABEL org.opencontainers.image.source="https://github.com/sergelogvinov/proxmox-csi-plugin" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.description="Proxmox VE CSI tools" + +ARG TARGETARCH +COPY --from=builder /src/bin/pvecsi-mutate-${TARGETARCH} /bin/pvecsi-mutate + +ENTRYPOINT ["/bin/pvecsi-mutate"] + +######################################## + +FROM alpine:3.18 AS pvecsi-mutate-goreleaser +LABEL org.opencontainers.image.source="https://github.com/sergelogvinov/proxmox-csi-plugin" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.description="Proxmox VE CSI tools" + +ARG TARGETARCH +COPY pvecsi-mutate-linux-${TARGETARCH} /bin/pvecsi-mutate + +ENTRYPOINT ["/bin/pvecsi-mutate"] diff --git a/Makefile b/Makefile index 1142a73..63e12a4 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,13 @@ REGISTRY ?= ghcr.io USERNAME ?= sergelogvinov -PROJECT ?= proxmox-csi -IMAGE ?= $(REGISTRY)/$(USERNAME)/$(PROJECT) +OCIREPO ?= $(REGISTRY)/$(USERNAME) HELMREPO ?= $(REGISTRY)/$(USERNAME)/charts PLATFORM ?= linux/arm64,linux/amd64 PUSH ?= false -SHA ?= $(shell git describe --match=none --always --abbrev=8 --dirty) +SHA ?= $(shell git describe --match=none --always --abbrev=7 --dirty) TAG ?= $(shell git describe --tag --always --match v[0-9]\*) -GO_LDFLAGS := -ldflags "-w -s -X main.version=$(SHA)" +GO_LDFLAGS := -ldflags "-w -s -X main.version=$(TAG) -X main.commit=$(SHA)" OS ?= $(shell go env GOOS) ARCH ?= $(shell go env GOARCH) @@ -16,7 +15,7 @@ ARCHS = amd64 arm64 BUILD_ARGS := --platform=$(PLATFORM) ifeq ($(PUSH),true) -BUILD_ARGS += --push=$(PUSH) +BUILD_ARGS += --push=$(PUSH) --output type=image,annotation-index.org.opencontainers.image.source="https://github.com/$(USERNAME)/proxmox-csi-plugin" else BUILD_ARGS += --output type=docker endif @@ -57,12 +56,16 @@ build-all-archs: clean: ## Clean rm -rf bin .cache +build-pvecsi-mutate: + CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build $(GO_LDFLAGS) \ + -o bin/pvecsi-mutate-$(ARCH) ./cmd/pvecsi-mutate + build-%: CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build $(GO_LDFLAGS) \ -o bin/proxmox-csi-$*-$(ARCH) ./cmd/$* .PHONY: build -build: build-controller build-node ## Build +build: build-controller build-node build-pvecsi-mutate ## Build .PHONY: run run: build-controller ## Run @@ -133,19 +136,21 @@ image-%: docker buildx build $(BUILD_ARGS) \ --build-arg TAG=$(TAG) \ --build-arg SHA=$(SHA) \ - -t $(IMAGE)-$*:$(TAG) \ + -t $(OCIREPO)/$*:$(TAG) \ --target $* \ -f Dockerfile . .PHONY: images-checks images-checks: images image-tools-check - trivy image --exit-code 1 --ignore-unfixed --severity HIGH,CRITICAL --no-progress $(IMAGE)-controller:$(TAG) - trivy image --exit-code 1 --ignore-unfixed --severity HIGH,CRITICAL --no-progress $(IMAGE)-node:$(TAG) + trivy image --exit-code 1 --ignore-unfixed --severity HIGH,CRITICAL --no-progress $(OCIREPO)/proxmox-csi-controller:$(TAG) + trivy image --exit-code 1 --ignore-unfixed --severity HIGH,CRITICAL --no-progress $(OCIREPO)/proxmox-csi-node:$(TAG) + trivy image --exit-code 1 --ignore-unfixed --severity HIGH,CRITICAL --no-progress $(OCIREPO)/pvecsi-mutate:$(TAG) .PHONY: images-cosign images-cosign: - @cosign sign --yes $(COSING_ARGS) --recursive $(IMAGE)-controller:$(TAG) - @cosign sign --yes $(COSING_ARGS) --recursive $(IMAGE)-node:$(TAG) + @cosign sign --yes $(COSING_ARGS) --recursive $(OCIREPO)/proxmox-csi-controller:$(TAG) + @cosign sign --yes $(COSING_ARGS) --recursive $(OCIREPO)/proxmox-csi-node:$(TAG) + @cosign sign --yes $(COSING_ARGS) --recursive $(OCIREPO)/pvecsi-mutate:$(TAG) .PHONY: images -images: image-controller image-node ## Build images +images: image-proxmox-csi-controller image-proxmox-csi-node image-pvecsi-mutate ## Build images diff --git a/cmd/pvecsi-mutate/main.go b/cmd/pvecsi-mutate/main.go new file mode 100644 index 0000000..b9adc65 --- /dev/null +++ b/cmd/pvecsi-mutate/main.go @@ -0,0 +1,103 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Proxmox PV Migrate utility +package main + +import ( + "context" + "fmt" + "os" + "strings" + + log "github.com/sirupsen/logrus" + cobra "github.com/spf13/cobra" + + clilog "github.com/sergelogvinov/proxmox-csi-plugin/pkg/log" +) + +var ( + command = "pvecsi-mutate" + version = "v0.0.0" + commit = "none" + + cloudconfig string + kubeconfig string + + flagLogLevel = "log-level" + + flagProxmoxConfig = "config" + flagKubeConfig = "kubeconfig" + + logger *log.Entry +) + +func main() { + if exitCode := run(); exitCode != 0 { + os.Exit(exitCode) + } +} + +func run() int { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + l := log.New() + l.SetOutput(os.Stdout) + l.SetLevel(log.InfoLevel) + + logger = l.WithContext(ctx) + + cmd := cobra.Command{ + Use: command, + Version: fmt.Sprintf("%s (commit: %s)", version, commit), + Short: "A command-line utility to manipulate PersistentVolume/PersistentVolumeClaim on Proxmox VE", + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + f := cmd.Flags() + loglvl, _ := f.GetString(flagLogLevel) //nolint: errcheck + + clilog.Configure(logger, loglvl) + + return nil + }, + SilenceUsage: true, + SilenceErrors: true, + } + + cmd.PersistentFlags().String(flagLogLevel, clilog.LevelInfo, + fmt.Sprintf("log level, must be one of: %s", strings.Join(clilog.Levels, ", "))) + + cmd.PersistentFlags().StringVar(&cloudconfig, flagProxmoxConfig, "", "proxmox cluster config file") + cmd.PersistentFlags().StringVar(&kubeconfig, flagKubeConfig, "", "kubernetes config file") + + cmd.AddCommand(buildMigrateCmd()) + cmd.AddCommand(buildRenameCmd()) + + err := cmd.ExecuteContext(ctx) + if err != nil { + errorString := err.Error() + if strings.Contains(errorString, "arg(s)") || strings.Contains(errorString, "flag") || strings.Contains(errorString, "command") { + fmt.Fprintf(os.Stderr, "Error: %s\n\n", errorString) + fmt.Fprintln(os.Stderr, cmd.UsageString()) + } else { + logger.Errorf("Error: %s\n", errorString) + } + + return 1 + } + + return 0 +} diff --git a/cmd/pvecsi-mutate/migrate.go b/cmd/pvecsi-mutate/migrate.go new file mode 100644 index 0000000..152a2c1 --- /dev/null +++ b/cmd/pvecsi-mutate/migrate.go @@ -0,0 +1,224 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "fmt" + "strings" + "time" + + cobra "github.com/spf13/cobra" + + proxmox "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/cluster" + "github.com/sergelogvinov/proxmox-csi-plugin/pkg/csi" + vm "github.com/sergelogvinov/proxmox-csi-plugin/pkg/proxmox" + tools "github.com/sergelogvinov/proxmox-csi-plugin/pkg/tools" + volume "github.com/sergelogvinov/proxmox-csi-plugin/pkg/volume" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientkubernetes "k8s.io/client-go/kubernetes" +) + +type migrateCmd struct { + pclient *proxmox.Cluster + kclient *clientkubernetes.Clientset + namespace string +} + +func buildMigrateCmd() *cobra.Command { + c := &migrateCmd{} + + cmd := cobra.Command{ + Use: "migrate pvc proxmox-node", + Aliases: []string{"m"}, + Short: "Migrate data from one Proxmox node to another", + Args: cobra.ExactArgs(2), + PreRunE: c.migrationValidate, + RunE: c.runMigration, + SilenceUsage: true, + SilenceErrors: true, + } + + setMigrateCmdFlags(&cmd) + + return &cmd +} + +func setMigrateCmdFlags(cmd *cobra.Command) { + flags := cmd.Flags() + + flags.StringP("namespace", "n", "", "namespace of the persistentvolumeclaims") + + flags.BoolP("force", "f", false, "force migration even if the persistentvolumeclaims is in use") + flags.Int("timeout", 3600, "task timeout in seconds") +} + +// nolint: cyclop, gocyclo +func (c *migrateCmd) runMigration(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + force, _ := flags.GetBool("force") //nolint: errcheck + + var err error + + ctx := context.Background() + pvc := args[0] + node := args[1] + + kubePVC, kubePV, err := tools.PVCResources(ctx, c.kclient, c.namespace, pvc) + if err != nil { + return fmt.Errorf("failed to get resources: %v", err) + } + + vol, err := volume.NewVolumeFromVolumeID(kubePV.Spec.CSI.VolumeHandle) + if err != nil { + return fmt.Errorf("failed to parse volume ID: %v", err) + } + + if vol.Node() == node { + return fmt.Errorf("persistentvolumeclaims %s is already on proxmox node %s", pvc, node) + } + + cluster, err := c.pclient.GetProxmoxCluster(vol.Cluster()) + if err != nil { + return fmt.Errorf("failed to get Proxmox cluster: %v", err) + } + + pods, vmName, err := tools.PVCPodUsage(ctx, c.kclient, c.namespace, pvc) + if err != nil { + return fmt.Errorf("failed to find pods using pvc: %v", err) + } + + cordonedNodes := []string{} + + if len(pods) > 0 { + if force { + logger.Infof("persistentvolumeclaims is using by pods: %s on node %s, trying to force migration\n", strings.Join(pods, ","), vmName) + + var csiNodes []string + + csiNodes, err = tools.CSINodes(ctx, c.kclient, csi.DriverName) + if err != nil { + return err + } + + cordonedNodes = append(cordonedNodes, csiNodes...) + + logger.Infof("cordoning nodes: %s", strings.Join(cordonedNodes, ",")) + + if _, err = tools.CondonNodes(ctx, c.kclient, cordonedNodes); err != nil { + return fmt.Errorf("failed to cordon nodes: %v", err) + } + + logger.Infof("terminated pods: %s", strings.Join(pods, ",")) + + for _, pod := range pods { + if err = c.kclient.CoreV1().Pods(c.namespace).Delete(ctx, pod, metav1.DeleteOptions{}); err != nil { + return fmt.Errorf("failed to delete pod: %v", err) + } + } + + for { + p, _, e := tools.PVCPodUsage(ctx, c.kclient, c.namespace, pvc) + if e != nil { + return fmt.Errorf("failed to find pods using pvc: %v", e) + } + + if len(p) == 0 { + break + } + + logger.Infof("waiting pods: %s", strings.Join(p, " ")) + + time.Sleep(2 * time.Second) + } + + time.Sleep(5 * time.Second) + } else { + return fmt.Errorf("persistentvolumeclaims is using by pods: %s on node %s, cannot move volume", strings.Join(pods, ","), vmName) + } + } + + if err = vm.WaitForVolumeDetach(cluster, vmName, vol.Disk()); err != nil { + return fmt.Errorf("failed to wait for volume detach: %v", err) + } + + logger.Infof("moving disk %s to proxmox node %s", vol.Disk(), node) + + taskTimeout, _ := flags.GetInt("timeout") //nolint: errcheck + if err = vm.MoveQemuDisk(cluster, vol, node, taskTimeout); err != nil { + return fmt.Errorf("failed to move disk: %v", err) + } + + logger.Infof("replacing persistentvolume topology") + + if err = replacePVTopology(ctx, c.kclient, c.namespace, kubePVC, kubePV, vol, node); err != nil { + return fmt.Errorf("failed to replace PV topology: %v", err) + } + + if force { + logger.Infof("uncordoning nodes: %s", strings.Join(cordonedNodes, ",")) + + if err = tools.UncondonNodes(ctx, c.kclient, cordonedNodes); err != nil { + return fmt.Errorf("failed to uncordon nodes: %v", err) + } + } + + logger.Infof("persistentvolumeclaims %s has been migrated to proxmox node %s", pvc, node) + + return nil +} + +func (c *migrateCmd) migrationValidate(cmd *cobra.Command, _ []string) error { + flags := cmd.Flags() + + cfg, err := proxmox.ReadCloudConfigFromFile(cloudconfig) + if err != nil { + return fmt.Errorf("failed to read config: %v", err) + } + + for _, c := range cfg.Clusters { + if c.Username == "" || c.Password == "" { + return fmt.Errorf("this command requires Proxmox root account, please provide username and password in config file (cluster=%s)", c.Region) + } + } + + c.pclient, err = proxmox.NewCluster(&cfg, nil) + if err != nil { + return fmt.Errorf("failed to create Proxmox cluster client: %v", err) + } + + if c.pclient.CheckClusters() != nil { + return fmt.Errorf("failed to initialize Proxmox clusters: %v", err) + } + + namespace, _ := flags.GetString("namespace") //nolint: errcheck + + kclientConfig, namespace, err := tools.BuildConfig(kubeconfig, namespace) + if err != nil { + return fmt.Errorf("failed to create kubernetes config: %v", err) + } + + c.kclient, err = clientkubernetes.NewForConfig(kclientConfig) + if err != nil { + return fmt.Errorf("failed to create kubernetes client: %v", err) + } + + c.namespace = namespace + + return nil +} diff --git a/cmd/pvecsi-mutate/rename.go b/cmd/pvecsi-mutate/rename.go new file mode 100644 index 0000000..0aeb7a8 --- /dev/null +++ b/cmd/pvecsi-mutate/rename.go @@ -0,0 +1,171 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "fmt" + "strings" + "time" + + cobra "github.com/spf13/cobra" + + tools "github.com/sergelogvinov/proxmox-csi-plugin/pkg/tools" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientkubernetes "k8s.io/client-go/kubernetes" +) + +type renameCmd struct { + kclient *clientkubernetes.Clientset + namespace string +} + +func buildRenameCmd() *cobra.Command { + c := &renameCmd{} + + cmd := cobra.Command{ + Use: "rename pvc-old pvc-new", + Aliases: []string{"re"}, + Short: "Rename PersistentVolumeClaim", + Args: cobra.ExactArgs(2), + PreRunE: c.renameValidate, + RunE: c.runRename, + SilenceUsage: true, + SilenceErrors: true, + } + + setrenameCmdFlags(&cmd) + + return &cmd +} + +func setrenameCmdFlags(cmd *cobra.Command) { + flags := cmd.Flags() + + flags.StringP("namespace", "n", "", "namespace of the persistentvolumeclaims") + + flags.BoolP("force", "f", false, "force migration even if the persistentvolumeclaims is in use") + flags.Int("timeout", 120, "task timeout in seconds") +} + +// nolint: cyclop, gocyclo +func (c *renameCmd) runRename(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + force, _ := flags.GetBool("force") //nolint: errcheck + + var err error + + ctx := context.Background() + + srcPVC, srcPV, err := tools.PVCResources(ctx, c.kclient, c.namespace, args[0]) + if err != nil { + return fmt.Errorf("failed to get resources: %v", err) + } + + pods, vmName, err := tools.PVCPodUsage(ctx, c.kclient, c.namespace, args[0]) + if err != nil { + return fmt.Errorf("failed to find pods using pvc: %v", err) + } + + cordonedNodes := []string{} + + if len(pods) > 0 { + if force { + logger.Infof("persistentvolumeclaims is using by pods: %s on node %s, trying to force migration\n", strings.Join(pods, ","), vmName) + + var csiNodes []string + + csiNodes, err = tools.CSINodes(ctx, c.kclient, srcPV.Spec.CSI.Driver) + if err != nil { + return err + } + + cordonedNodes = append(cordonedNodes, csiNodes...) + + logger.Infof("cordoning nodes: %s", strings.Join(cordonedNodes, ",")) + + if _, err = tools.CondonNodes(ctx, c.kclient, cordonedNodes); err != nil { + return fmt.Errorf("failed to cordon nodes: %v", err) + } + + logger.Infof("terminated pods: %s", strings.Join(pods, ",")) + + for _, pod := range pods { + if err = c.kclient.CoreV1().Pods(c.namespace).Delete(ctx, pod, metav1.DeleteOptions{}); err != nil { + return fmt.Errorf("failed to delete pod: %v", err) + } + } + + for { + p, _, e := tools.PVCPodUsage(ctx, c.kclient, c.namespace, args[0]) + if e != nil { + return fmt.Errorf("failed to find pods using pvc: %v", e) + } + + if len(p) == 0 { + break + } + + logger.Infof("waiting pods: %s", strings.Join(p, " ")) + + time.Sleep(2 * time.Second) + } + + time.Sleep(5 * time.Second) + } else { + return fmt.Errorf("persistentvolumeclaims is using by pods: %s on node %s, cannot move volume", strings.Join(pods, ","), vmName) + } + } + + err = renamePVC(ctx, c.kclient, c.namespace, srcPVC, srcPV, args[1]) + if err != nil { + return fmt.Errorf("failed to rename persistentvolumeclaims: %v", err) + } + + if force { + logger.Infof("uncordoning nodes: %s", strings.Join(cordonedNodes, ",")) + + if err = tools.UncondonNodes(ctx, c.kclient, cordonedNodes); err != nil { + return fmt.Errorf("failed to uncordon nodes: %v", err) + } + } + + logger.Infof("persistentvolumeclaims %s has been renamed", args[0]) + + return nil +} + +func (c *renameCmd) renameValidate(cmd *cobra.Command, _ []string) error { + flags := cmd.Flags() + + namespace, _ := flags.GetString("namespace") //nolint: errcheck + + kclientConfig, namespace, err := tools.BuildConfig(kubeconfig, namespace) + if err != nil { + return fmt.Errorf("failed to create kubernetes config: %v", err) + } + + c.kclient, err = clientkubernetes.NewForConfig(kclientConfig) + if err != nil { + return fmt.Errorf("failed to create kubernetes client: %v", err) + } + + c.namespace = namespace + + return nil +} diff --git a/cmd/pvecsi-mutate/utils.go b/cmd/pvecsi-mutate/utils.go new file mode 100644 index 0000000..36d4e4a --- /dev/null +++ b/cmd/pvecsi-mutate/utils.go @@ -0,0 +1,144 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "fmt" + + "github.com/sergelogvinov/proxmox-csi-plugin/pkg/csi" + tools "github.com/sergelogvinov/proxmox-csi-plugin/pkg/tools" + volume "github.com/sergelogvinov/proxmox-csi-plugin/pkg/volume" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + clientkubernetes "k8s.io/client-go/kubernetes" +) + +func replacePVTopology( + ctx context.Context, + clientset *clientkubernetes.Clientset, + namespace string, + pvc *corev1.PersistentVolumeClaim, + pv *corev1.PersistentVolume, + vol *volume.Volume, + node string, +) error { + newPVC := pvc.DeepCopy() + newPVC.ObjectMeta.UID = "" + newPVC.ObjectMeta.ResourceVersion = "" + delete(newPVC.ObjectMeta.Annotations, csi.DriverName+"/migrate") + delete(newPVC.ObjectMeta.Annotations, csi.DriverName+"/migrate-node") + newPVC.Status = corev1.PersistentVolumeClaimStatus{} + + newPV := pv.DeepCopy() + newPV.ObjectMeta.UID = "" + newPV.ObjectMeta.ResourceVersion = "" + delete(newPV.ObjectMeta.Annotations, csi.DriverName+"/migrate") + delete(newPV.ObjectMeta.Annotations, csi.DriverName+"/migrate-node") + newPV.Spec.ClaimRef = nil + newPV.Status = corev1.PersistentVolumeStatus{} + newPV.Spec.CSI.VolumeHandle = volume.NewVolume(vol.Region(), node, vol.Storage(), vol.Disk()).VolumeID() + newPV.Spec.NodeAffinity.Required = &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: corev1.LabelTopologyRegion, + Operator: "In", + Values: []string{vol.Region()}, + }, + { + Key: corev1.LabelTopologyZone, + Operator: "In", + Values: []string{node}, + }, + }, + }, + }, + } + + policy := metav1.DeletePropagationForeground + if err := clientset.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, pvc.Name, metav1.DeleteOptions{PropagationPolicy: &policy}); err != nil { + return fmt.Errorf("failed to delete PVC: %v", err) + } + + if pv.Spec.PersistentVolumeReclaimPolicy != corev1.PersistentVolumeReclaimDelete { + if err := clientset.CoreV1().PersistentVolumes().Delete(ctx, pv.Name, metav1.DeleteOptions{PropagationPolicy: &policy}); err != nil { + return fmt.Errorf("failed to delete PV: %v", err) + } + } + + if err := tools.PVWaitDelete(ctx, clientset, pv.Name); err != nil { + return fmt.Errorf("failed to wait for PV deletion: %v", err) + } + + if _, err := clientset.CoreV1().PersistentVolumes().Create(ctx, newPV, metav1.CreateOptions{}); err != nil { + return fmt.Errorf("failed to create PV: %v", err) + } + + if _, err := clientset.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, newPVC, metav1.CreateOptions{}); err != nil { + if _, err := clientset.CoreV1().PersistentVolumeClaims(namespace).Update(ctx, newPVC, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("failed to create/update PVC: %v", err) + } + } + + return nil +} + +func renamePVC( + ctx context.Context, + clientset *clientkubernetes.Clientset, + namespace string, + pvc *corev1.PersistentVolumeClaim, + pv *corev1.PersistentVolume, + newName string, +) error { + newPVC := pvc.DeepCopy() + newPVC.ObjectMeta.Name = newName + newPVC.ObjectMeta.UID = "" + newPVC.ObjectMeta.ResourceVersion = "" + newPVC.Status = corev1.PersistentVolumeClaimStatus{} + + patch := []byte(`{"spec":{"persistentVolumeReclaimPolicy":"` + corev1.PersistentVolumeReclaimRetain + `"}}`) + + if pv.Spec.PersistentVolumeReclaimPolicy == corev1.PersistentVolumeReclaimDelete { + if _, err := clientset.CoreV1().PersistentVolumes().Patch(ctx, pvc.Spec.VolumeName, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("failed to patch PersistentVolumes: %v", err) + } + } + + policy := metav1.DeletePropagationForeground + if err := clientset.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, pvc.Name, metav1.DeleteOptions{PropagationPolicy: &policy}); err != nil { + return fmt.Errorf("failed to delete PersistentVolumeClaims: %v", err) + } + + patch = []byte(`{"spec":{"claimRef":null}}`) + + if _, err := clientset.CoreV1().PersistentVolumes().Patch(ctx, pvc.Spec.VolumeName, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("failed to patch PersistentVolumes: %v", err) + } + + if _, err := clientset.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, newPVC, metav1.CreateOptions{}); err != nil { + if _, err := clientset.CoreV1().PersistentVolumeClaims(namespace).Update(ctx, newPVC, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("failed to create/update PersistentVolumeClaims: %v", err) + } + } + + return nil +} diff --git a/go.mod b/go.mod index 29d6d1f..a3d45e9 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,8 @@ require ( github.com/kubernetes-csi/csi-lib-utils v0.17.0 github.com/sergelogvinov/proxmox-cloud-controller-manager v0.3.0 github.com/siderolabs/go-blockdevice v0.4.7 + github.com/sirupsen/logrus v1.9.0 + github.com/spf13/cobra v1.8.0 github.com/stretchr/testify v1.8.4 google.golang.org/grpc v1.61.0 k8s.io/api v0.29.1 @@ -35,6 +37,7 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.5.0 // indirect github.com/imdario/mergo v0.3.16 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect diff --git a/go.sum b/go.sum index 871826e..7519c84 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,7 @@ github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 h1:7Ip0wMmLHLRJdrloD github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/container-storage-interface/spec v1.9.0 h1:zKtX4STsq31Knz3gciCYCi1SXtO2HJDecIjDVboYavY= github.com/container-storage-interface/spec v1.9.0/go.mod h1:ZfDu+3ZRyeVqxZM0Ds19MVLkN2d1XJ5MAfi1L3VjlT0= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -42,6 +43,8 @@ github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww= github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -79,6 +82,7 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergelogvinov/proxmox-cloud-controller-manager v0.3.0 h1:2TurTnjawIx5j3YIRw0PEBbxM7YpLwlVBXfnnnpYH5s= github.com/sergelogvinov/proxmox-cloud-controller-manager v0.3.0/go.mod h1:SILDj23jkQGVPtWCWdadd3E6VDJlWnN+P4rgKGrg1j8= github.com/siderolabs/go-blockdevice v0.4.7 h1:2bk4WpEEflGxjrNwp57ye24Pr+cYgAiAeNMWiQOuWbQ= @@ -87,6 +91,10 @@ github.com/siderolabs/go-cmd v0.1.1 h1:nTouZUSxLeiiEe7hFexSVvaTsY/3O8k1s08BxPRrs github.com/siderolabs/go-cmd v0.1.1/go.mod h1:6hY0JG34LxEEwYE8aH2iIHkHX/ir12VRLqfwAf2yJIY= github.com/siderolabs/go-retry v0.3.2 h1:FzWslFm4y8RY1wU0gIskm0oZHOpsSibZqlR8N8/k4Eo= github.com/siderolabs/go-retry v0.3.2/go.mod h1:Ac8HIh0nAYDQm04FGZHNofVAXteyd4xR9oujTRrtvK0= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -95,6 +103,7 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/objx v0.5.1 h1:4VhoImhV/Bm0ToFkXFi8hXNXwpDRZ/ynw3amt82mzq0= github.com/stretchr/objx v0.5.1/go.mod h1:/iHQpkQwBD6DLUmQ4pE+s1TXdob1mORJ4/UFdrifcy0= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -130,6 +139,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/pkg/csi/driver.go b/pkg/csi/driver.go index 39c176c..3b47c25 100644 --- a/pkg/csi/driver.go +++ b/pkg/csi/driver.go @@ -23,7 +23,7 @@ const ( // DriverVersion is the version of the CSI driver DriverVersion = "0.4.0" // DriverSpecVersion CSI spec version - DriverSpecVersion = "1.8.0" + DriverSpecVersion = "1.9.0" // StorageIDKey is the ID of the Proxmox storage StorageIDKey = "storage" diff --git a/pkg/log/log.go b/pkg/log/log.go new file mode 100644 index 0000000..18aad02 --- /dev/null +++ b/pkg/log/log.go @@ -0,0 +1,82 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package log is cli logger configurator. +package log + +import ( + "os" + + log "github.com/sirupsen/logrus" +) + +const ( + // LevelTrace is the log level for tracing + LevelTrace = "trace" + // LevelDebug is the log level for debugging + LevelDebug = "debug" + // LevelInfo is the log level for informational messages + LevelInfo = "info" + // LevelWarn is the log level for warnings + LevelWarn = "warn" + // LevelError is the log level for errors + LevelError = "error" + // LevelFatal is the log level for fatal errors + LevelFatal = "fatal" + // LevelPanic is the log level for panics + LevelPanic = "panic" +) + +// Levels is a slice of all log levels +var Levels = []string{ + LevelTrace, LevelDebug, LevelInfo, LevelWarn, + LevelError, LevelFatal, LevelPanic, +} + +// Configure configures the logger. +func Configure(entry *log.Entry, level string) { + logger := entry.Logger + logger.SetOutput(os.Stdout) + + logger.SetFormatter(&log.TextFormatter{ + // DisableColors: true, + // FullTimestamp: true, + DisableTimestamp: true, + DisableLevelTruncation: true, + }) + logger.SetLevel(logLevel(level)) +} + +func logLevel(level string) log.Level { + switch level { + case LevelTrace: + return log.TraceLevel + case LevelDebug: + return log.DebugLevel + case LevelInfo: + return log.InfoLevel + case LevelWarn: + return log.WarnLevel + case LevelError: + return log.ErrorLevel + case LevelFatal: + return log.FatalLevel + case LevelPanic: + return log.PanicLevel + } + + return log.DebugLevel +} diff --git a/pkg/proxmox/volume.go b/pkg/proxmox/volume.go new file mode 100644 index 0000000..ae285bd --- /dev/null +++ b/pkg/proxmox/volume.go @@ -0,0 +1,100 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package proxmox implements tools to work with Proxmox VM. +package proxmox + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + pxapi "github.com/Telmate/proxmox-api-go/proxmox" + + volume "github.com/sergelogvinov/proxmox-csi-plugin/pkg/volume" +) + +// WaitForVolumeDetach waits for the volume to be detached from the VM. +func WaitForVolumeDetach(client *pxapi.Client, vmName string, disk string) error { + if vmName == "" { + return nil + } + + vmr, err := client.GetVmRefsByName(vmName) + if err != nil || len(vmr) == 0 { + return fmt.Errorf("failed to get vmID") + } + + for { + time.Sleep(5 * time.Second) + + vmConfig, err := client.GetVmConfig(vmr[0]) + if err != nil { + return fmt.Errorf("failed to get vm config: %v", err) + } + + found := false + + for lun := 1; lun < 30; lun++ { + device := fmt.Sprintf("scsi%d", lun) + + if vmConfig[device] != nil && strings.Contains(vmConfig[device].(string), disk) { + found = true + + break + } + } + + if !found { + return nil + } + } +} + +// MoveQemuDisk moves the volume from one node to another. +func MoveQemuDisk(cluster *pxapi.Client, vol *volume.Volume, node string, taskTimeout int) error { + vmParams := map[string]interface{}{ + "node": vol.Node(), + "target": vol.Disk(), + "target_node": node, + "volume": vol.Disk(), + } + + oldTimeout := cluster.TaskTimeout + cluster.TaskTimeout = taskTimeout + + // POST https://pve.proxmox.com/pve-docs/api-viewer/index.html#/nodes/{node}/storage/{storage}/content/{volume} + // Copy a volume. This is experimental code - do not use. + resp, err := cluster.CreateItemReturnStatus(vmParams, "/nodes/"+vol.Node()+"/storage/"+vol.Storage()+"/content/"+vol.Disk()) + if err != nil { + return fmt.Errorf("failed to move pvc: %v, vmParams=%+v", err, vmParams) + } + + var taskResponse map[string]interface{} + + if err = json.Unmarshal([]byte(resp), &taskResponse); err != nil { + return fmt.Errorf("failed to parse response: %v", err) + } + + if _, err := cluster.WaitForCompletion(taskResponse); err != nil { + return fmt.Errorf("failed to wait for task completion: %v", err) + } + + cluster.TaskTimeout = oldTimeout + + return nil +} diff --git a/pkg/tools/config.go b/pkg/tools/config.go new file mode 100644 index 0000000..20b1c37 --- /dev/null +++ b/pkg/tools/config.go @@ -0,0 +1,48 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package tools implements tools to work with kubeernetes. +package tools + +import ( + "fmt" + + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +// BuildConfig returns a kubernetes client configuration and namespace. +func BuildConfig(kubeconfig, namespace string) (k *rest.Config, ns string, err error) { + clientConfigLoadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + + if kubeconfig != "" { + clientConfigLoadingRules.ExplicitPath = kubeconfig + } + + config := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + clientConfigLoadingRules, &clientcmd.ConfigOverrides{}) + + if namespace == "" { + namespace, _, err = config.Namespace() + if err != nil { + return nil, "", fmt.Errorf("failed to get namespace from kubeconfig: %w", err) + } + } + + k, err = config.ClientConfig() + + return k, namespace, err +} diff --git a/pkg/tools/nodes.go b/pkg/tools/nodes.go new file mode 100644 index 0000000..04df285 --- /dev/null +++ b/pkg/tools/nodes.go @@ -0,0 +1,86 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tools + +import ( + "context" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + clientkubernetes "k8s.io/client-go/kubernetes" +) + +// CSINodes returns a list of nodes that have the specified CSI driver name. +func CSINodes(ctx context.Context, kclient *clientkubernetes.Clientset, csiDriverName string) ([]string, error) { + nodes := []string{} + + csinodes, err := kclient.StorageV1().CSINodes().List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to list CSINodes: %v", err) + } + + for _, csinode := range csinodes.Items { + for _, driver := range csinode.Spec.Drivers { + if driver.Name == csiDriverName { + nodes = append(nodes, driver.NodeID) + + break + } + } + } + + return nodes, nil +} + +// CondonNodes condones the specified nodes. +func CondonNodes(ctx context.Context, kclient *clientkubernetes.Clientset, nodes []string) ([]string, error) { + cordonedNodes := []string{} + patch := []byte(`{"spec":{"unschedulable":true}}`) + + for _, node := range nodes { + nodeStatus, err := kclient.CoreV1().Nodes().Get(ctx, node, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get node status: %v", err) + } + + if !nodeStatus.Spec.Unschedulable { + _, err = kclient.CoreV1().Nodes().Patch(ctx, node, types.MergePatchType, patch, metav1.PatchOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to cordon node: %v", err) + } + + cordonedNodes = append(cordonedNodes, node) + } + } + + return cordonedNodes, nil +} + +// UncondonNodes uncondones the specified nodes. +func UncondonNodes(ctx context.Context, kclient *clientkubernetes.Clientset, nodes []string) error { + patch := []byte(`{"spec":{"unschedulable":false}}`) + + for _, node := range nodes { + _, err := kclient.CoreV1().Nodes().Patch(ctx, node, types.MergePatchType, patch, metav1.PatchOptions{}) + if err != nil { + return fmt.Errorf("failed to uncordon node: %v", err) + } + } + + return nil +} diff --git a/pkg/tools/pv.go b/pkg/tools/pv.go new file mode 100644 index 0000000..2b50678 --- /dev/null +++ b/pkg/tools/pv.go @@ -0,0 +1,101 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tools + +import ( + "context" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" + clientkubernetes "k8s.io/client-go/kubernetes" +) + +// PVCResources returns the PersistentVolumeClaim and PersistentVolume resources. +func PVCResources(ctx context.Context, clientset *clientkubernetes.Clientset, namespace, pvcName string) (*corev1.PersistentVolumeClaim, *corev1.PersistentVolume, error) { + pvc, err := clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil { + return nil, nil, fmt.Errorf("failed to get PersistentVolumeClaims: %v", err) + } + + pv, err := clientset.CoreV1().PersistentVolumes().Get(ctx, pvc.Spec.VolumeName, metav1.GetOptions{}) + if err != nil { + return nil, nil, fmt.Errorf("failed to get PersistentVolumes: %v", err) + } + + return pvc, pv, nil +} + +// PVCPodUsage returns the list of pods and the node that are using the specified PersistentVolumeClaim. +func PVCPodUsage(ctx context.Context, clientset *clientkubernetes.Clientset, namespace, pvcName string) (pods []string, node string, err error) { + podList, err := clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, "", fmt.Errorf("failed to list pods: %v", err) + } + + for _, pod := range podList.Items { + if pod.Status.Phase == corev1.PodRunning { + for _, volume := range pod.Spec.Volumes { + if volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.ClaimName == pvcName { + pods = append(pods, pod.Name) + node = pod.Spec.NodeName + + break + } + } + } + } + + return pods, node, nil +} + +// PVWaitDelete waits for the specified PersistentVolume to be deleted. +func PVWaitDelete(ctx context.Context, clientset *clientkubernetes.Clientset, pvName string) error { + _, err := clientset.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) + if err != nil { + return nil //nolint: nilerr + } + + watcher, err := clientset.CoreV1().PersistentVolumes().Watch(ctx, metav1.ListOptions{ + FieldSelector: "metadata.name=" + pvName, + }) + if err != nil { + return err + } + + defer watcher.Stop() + + timeout := time.After(10 * time.Minute) + + for { + select { + case event, ok := <-watcher.ResultChan(): + if !ok { + return fmt.Errorf("watch channel closed unexpectedly") + } + + if event.Type == watch.Deleted { + return nil + } + + case <-timeout: + return fmt.Errorf("timeout waiting for PersistentVolume %s to be deleted", pvName) + } + } +}