From be088413c452ef4232dd49613ee245abb0db0b9b Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Wed, 20 Sep 2023 16:30:14 +0300 Subject: [PATCH 01/66] use golangci-lint linter, fix multiple issues --- .github/workflows/build.yaml | 97 +++++---- .gitignore | 44 +++- .golangci.yaml | 146 +++++++++++++ Dockerfile | 57 +++-- Makefile | 128 ----------- build-all-and-push.sh | 14 -- main.go => cmd/main.go | 52 +++-- go.mod | 81 ++++--- go.sum | 411 ++++++++++++++++++++++------------- makefile | 74 +++++++ pkg/client/run.go | 31 --- pkg/config/config.go | 18 +- pkg/controller/controller.go | 196 ++++++++++------- pkg/kipcompute/compute.go | 254 ++++++++-------------- pkg/utils/k8sutil.go | 46 ++-- 15 files changed, 913 insertions(+), 736 deletions(-) create mode 100644 .golangci.yaml delete mode 100644 Makefile delete mode 100755 build-all-and-push.sh rename main.go => cmd/main.go (62%) create mode 100644 makefile delete mode 100644 pkg/client/run.go diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 3f07e4c..1347e81 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -18,53 +18,47 @@ on: jobs: - docker-build: + validate: runs-on: ubuntu-latest if: ${{ !contains(github.event.head_commit.message,'[skip ci]') }} + container: golang:1.21-alpine + steps: + - name: checkout + uses: actions/checkout@v4 + - name: test + shell: sh + env: + CGO_ENABLED: 0 + run: | + apk --update add ca-certificates tzdata make git bash + make lint + make test-json + + docker-build: + + runs-on: ubuntu-latest + needs: validate + if: ${{ !contains(github.event.head_commit.message,'[skip ci]') && github.event_name != 'pull_request' }} steps: - - uses: actions/checkout@v3 + - name: checkout + uses: actions/checkout@v4 + + - name: get short sha + id: short_sha + run: echo ::set-output name=sha::$(git rev-parse --short HEAD) + + - name: get version + id: version + run: echo ::set-output name=version::$([[ -z "${{ github.event.pull_request.number }}" ]] && echo "sha-${{ steps.short_sha.outputs.sha }}" || echo "pr-${{ github.event.pull_request.number }}") - - name: Set up QEMU + - name: set up QEMU uses: docker/setup-qemu-action@v3 - name: set up Docker buildx id: buildx uses: docker/setup-buildx-action@v3 - with: - version: latest - - - name: builder instance name - run: echo ${{ steps.buildx.outputs.name }} - - - name: available platforms - run: echo ${{ steps.buildx.outputs.platforms }} - - - name: decide on tag - id: prep - run: | - DOCKER_IMAGE=${{ github.repository }} - VERSION=noop - if [[ $GITHUB_REF == refs/tags/* ]]; then - VERSION=${GITHUB_REF#refs/tags/} - elif [[ $GITHUB_REF == refs/heads/* ]]; then - VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g') - if [ "${{ github.event.repository.default_branch }}" = "$VERSION" ]; then - VERSION=edge - fi - elif [[ $GITHUB_REF == refs/pull/* ]]; then - VERSION=pr-${{ github.event.number }} - fi - TAGS="${DOCKER_IMAGE}:${VERSION}" - if [[ $VERSION =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then - MINOR=${VERSION%.*} - MAJOR=${MINOR%.*} - TAGS="$TAGS,${DOCKER_IMAGE}:${MINOR},${DOCKER_IMAGE}:${MAJOR},${DOCKER_IMAGE}:latest" - elif [ "${{ github.event_name }}" = "push" ]; then - TAGS="$TAGS,${DOCKER_IMAGE}:sha-${GITHUB_SHA::8}" - fi - echo "tags=${TAGS}" >> $GITHUB_OUTPUT - name: login to DockerHub if: ${{ github.event_name != 'pull_request' }} @@ -73,13 +67,32 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: prepare meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ github.repository }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=sha + labels: | + github.run.id=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + org.opencontainers.image.title=kubeIP + org.opencontainers.image.description=kubeIP controller + org.opencontainers.image.vendor=DoiT International + - name: build and push - id: docker_build uses: docker/build-push-action@v5 with: - push: ${{ github.event_name != 'pull_request' }} + build-args: | + VERSION=${{ steps.version.outputs.version }} + COMMIT=${{ steps.short_sha.outputs.sha }} + BRANCH=${{ github.ref_name }} + push: true platforms: linux/amd64,linux/arm64 - tags: ${{ steps.prep.outputs.tags }} - - - name: image digest - run: echo ${{ steps.docker_build.outputs.digest }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.gitignore b/.gitignore index 1c9b6ff..5f6b96d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,19 +1,39 @@ -# code coverage -.cover +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib +.bin + +# Test binary, built with `go test -c` +*.test +.run + +# Local environment +.env +.in +.out -# git repo -.git +# Output of the go coverage tool, specifically when used with LiteIDE +*.out -# IDE customization +# IDE support files .idea .vscode -# binaries -.bin +# MacOS file system metadata +.DS_Store -# env customization -.env -.in +# local credentials +.credentials -# Test binary, build with `go test -c` -*.test +# Keep out some temporary example code and temprary file +example_code/ +older/ +temp/ + +# Dependency directories (remove the comment below to include it) +vendor/ + +tmp/ \ No newline at end of file diff --git a/.golangci.yaml b/.golangci.yaml new file mode 100644 index 0000000..7602ea2 --- /dev/null +++ b/.golangci.yaml @@ -0,0 +1,146 @@ +run: + # which dirs to skip + skip-dirs: + - mocks + # Timeout for analysis, e.g. 30s, 5m. + # Default: 1m + timeout: 5m + # Exit code when at least one issue was found. + # Default: 1 + issues-exit-code: 2 + # Include test files or not. + # Default: true + tests: false + # allow parallel run + allow-parallel-runners: true + +linters-settings: + govet: + check-shadowing: true + gocyclo: + min-complexity: 15 + maligned: + suggest-new: true + dupl: + threshold: 100 + goconst: + min-len: 2 + min-occurrences: 2 + misspell: + locale: US + ignore-words: + - "cancelled" + goimports: + local-prefixes: github.com/golangci/golangci-lint + gocritic: + enabled-tags: + - diagnostic + - experimental + - opinionated + - performance + - style + disabled-checks: + - dupImport # https://github.com/go-critic/go-critic/issues/845 + - ifElseChain + - octalLiteral + - rangeValCopy + - unnamedResult + - whyNoLint + - wrapperFunc + funlen: + lines: 105 + statements: 50 + +linters: + # please, do not use `enable-all`: it's deprecated and will be removed soon. + # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint + disable-all: true + enable: + - asciicheck + - bidichk + - bodyclose + # - containedctx + # - contextcheck disabled because of generics + - dupword + - decorder + # - depguard + - dogsled + - dupl + - durationcheck + - errcheck + - errchkjson + - errname + - errorlint + - exhaustive + # - exhaustivestruct TODO: check how to fix it + - exportloopref + # - forbidigo TODO: configure forbidden code patterns + # - forcetypeassert + - funlen + - gci + # - gochecknoglobals TODO: remove globals from code + # - gochecknoinits TODO: remove main.init + - gocognit + - goconst + - gocritic + - gocyclo + # - godox + - goerr113 + - gofmt + - goimports + - gomnd + # - gomoddirectives + - gosec + - gosimple + - govet + - goprintffuncname + - grouper + - importas + # - ireturn TODO: not sure if it is a good linter + - ineffassign + - interfacebloat + - loggercheck + - maintidx + - makezero + - misspell + - nakedret + # - nestif + - nilerr + - nilnil + # - noctx + - nolintlint + - prealloc + - predeclared + - promlinter + - reassign + - revive + # - rowserrcheck disabled because of generics + # - staticcheck doesn't work with go1.19 + # - structcheck disabled because of generics + - stylecheck + - tenv + - testableexamples + - typecheck + - unconvert + - unparam + - unused + # - varnamelen TODO: review naming + # - varcheck depricated 1.49 + # - wastedassign disabled because of generics + - whitespace + - wrapcheck + # - wsl + +issues: + exclude-rules: + - path: _test\.go + linters: + - funlen + - bodyclose + - gosec + - dupl + - gocognit + - goconst + - gocyclo + exclude: + - Using the variable on range scope `tt` in function literal diff --git a/Dockerfile b/Dockerfile index fd1a414..a806a06 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,37 +1,32 @@ -# syntax = docker/dockerfile:experimental +FROM golang:1.21-alpine AS builder -# -# ----- Go Builder Image ------ -# -FROM golang:1.20-alpine AS builder -# curl git bash -RUN apk add --no-cache curl git bash make -# -# ----- build and test ----- -# -FROM builder as build -# set working directorydoc -RUN mkdir -p /go/src/app -WORKDIR /go/src/app -# load dependency -COPY go.mod . -COPY go.sum . -RUN --mount=type=cache,target=$GOPATH/pkg/mod go mod download -# copy sources -COPY . . -# build -RUN make binary +# add CA certificates and TZ for local time +RUN apk --update add ca-certificates tzdata make git -# -# ------ release Docker image ------ -# +# Retrieve application dependencies. +# This allows the container build to reuse cached dependencies. +# Expecting to copy go.mod and if present go.sum. +COPY go.* ./ +RUN go mod download + +# Copy local code to the container image. +COPY . ./ + +# get version, commit and branch from build args +ARG VERSION +ARG COMMIT +ARG BRANCH + +# Build the binary with make (using the version, commit and branch) +RUN make build VERSION=${VERSION} COMMIT=${COMMIT} BRANCH=${BRANCH} + +# final image FROM scratch # copy CA certificates COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt -# this is the last command since it's never cached -COPY --from=build /go/src/app/.bin/github.com/doitintl/kubeip /kubeip - -ENTRYPOINT ["/kubeip"] - +# copy timezone settings +COPY --from=builder /usr/share/zoneinfo /usr/share/zoneinfo +# copy the binary to the production image from the builder stage +COPY --from=builder /app/.bin/kubeip /kubeip -# RUN cd /go/src/github.com/doitintl/kubeip && GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a --installsuffix cgo --ldflags="-s" -ldflags "-X main.version=$(git log | head -n 1 | cut -f 2 -d ' ') -X main.buildDate=$(date +%Y-%m-%d\-%H:%M)" -o /kubeip +ENTRYPOINT ["/kubeip"] \ No newline at end of file diff --git a/Makefile b/Makefile deleted file mode 100644 index aba8abf..0000000 --- a/Makefile +++ /dev/null @@ -1,128 +0,0 @@ -MODULE = $(shell env GO111MODULE=on $(GO) list -m) -DATE ?= $(shell date +%FT%T%z) -VERSION ?= $(shell git describe --tags --always --dirty --match="v*" 2> /dev/null || \ - cat $(CURDIR)/.version 2> /dev/null || echo v0) -COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null) -BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null) -PKGS = $(or $(PKG),$(shell env GO111MODULE=on $(GO) list ./...)) -TESTPKGS = $(shell env GO111MODULE=on $(GO) list -f \ - '{{ if or .TestGoFiles .XTestGoFiles }}{{ .ImportPath }}{{ end }}' \ - $(PKGS)) -BIN = $(CURDIR)/.bin -BINARY = kubeip - -GO = go -TIMEOUT = 15 -V = 0 -Q = $(if $(filter 1,$V),,@) -M = $(shell printf "\033[34;1m▶\033[0m") - -export GO111MODULE=on -export CGO_ENABLED=0 -export GOPROXY=https://proxy.golang.org - - -.PHONY: all -all: fmt lint binary - -.PHONY: binary -binary: $(BIN) ; $(info $(M) building executable...) @ ## Build program binary - $Q $(GO) build \ - -tags release \ - -ldflags '-X main.version=$(VERSION) -X main.buildDate=$(DATE)' \ - -o $(BIN)/$(basename $(MODULE)) main.go - -.PHONY: image -image: clean-image - @docker build -t "${BINARY}" -f Dockerfile . - -.PHONY: stop -stop: - @docker stop "${BINARY}" || true # Do not fail if container does not exist - -.PHONY: clean-image -clean-image: stop - @docker rmi "${BINARY}" || true # Do not fail if image does not exist - - -# Tools - -$(BIN): - @mkdir -p $@ -$(BIN)/%: | $(BIN) ; $(info $(M) building $(PACKAGE)...) - $Q tmp=$$(mktemp -d); \ - env GO111MODULE=off GOPATH=$$tmp GOBIN=$(BIN) $(GO) get $(PACKAGE) \ - || ret=$$?; \ - rm -rf $$tmp ; exit $$ret - -GOLINT = $(BIN)/golint -$(BIN)/golint: PACKAGE=golang.org/x/lint/golint - -GOCOV = $(BIN)/gocov -$(BIN)/gocov: PACKAGE=github.com/axw/gocov/... - -GOCOVXML = $(BIN)/gocov-xml -$(BIN)/gocov-xml: PACKAGE=github.com/AlekSi/gocov-xml - -GO2XUNIT = $(BIN)/go2xunit -$(BIN)/go2xunit: PACKAGE=github.com/tebeka/go2xunit - -# Tests - -TEST_TARGETS := test-default test-bench test-short test-verbose test-race -.PHONY: $(TEST_TARGETS) test-xml check test tests -test-bench: ARGS=-run=__absolutelynothing__ -bench=. ## Run benchmarks -test-short: ARGS=-short ## Run only short tests -test-verbose: ARGS=-v ## Run tests in verbose mode with coverage reporting -test-race: ARGS=-race ## Run tests with race detector -$(TEST_TARGETS): NAME=$(MAKECMDGOALS:test-%=%) -$(TEST_TARGETS): test -check test tests: fmt lint ; $(info $(M) running $(NAME:%=% )tests...) @ ## Run tests - $Q $(GO) test -timeout $(TIMEOUT)s $(ARGS) $(TESTPKGS) - -test-xml: fmt lint | $(GO2XUNIT) ; $(info $(M) running xUnit tests...) @ ## Run tests with xUnit output - $Q mkdir -p test - $Q 2>&1 $(GO) test -timeout $(TIMEOUT)s -v $(TESTPKGS) | tee test/tests.output - $(GO2XUNIT) -fail -input test/tests.output -output test/tests.xml - -COVERAGE_MODE = atomic -COVERAGE_PROFILE = $(COVERAGE_DIR)/profile.out -COVERAGE_XML = $(COVERAGE_DIR)/coverage.xml -COVERAGE_HTML = $(COVERAGE_DIR)/index.html -.PHONY: test-coverage test-coverage-tools -test-coverage-tools: | $(GOCOV) $(GOCOVXML) -test-coverage: COVERAGE_DIR := $(CURDIR)/test/coverage.$(shell date -u +"%Y-%m-%dT%H:%M:%SZ") -test-coverage: fmt lint test-coverage-tools ; $(info $(M) running coverage tests...) @ ## Run coverage tests - $Q mkdir -p $(COVERAGE_DIR) - $Q $(GO) test \ - -coverpkg=$$($(GO) list -f '{{ join .Deps "\n" }}' $(TESTPKGS) | \ - grep '^$(MODULE)/' | \ - tr '\n' ',' | sed 's/,$$//') \ - -covermode=$(COVERAGE_MODE) \ - -coverprofile="$(COVERAGE_PROFILE)" $(TESTPKGS) - $Q $(GO) tool cover -html=$(COVERAGE_PROFILE) -o $(COVERAGE_HTML) - $Q $(GOCOV) convert $(COVERAGE_PROFILE) | $(GOCOVXML) > $(COVERAGE_XML) - -.PHONY: lint -lint: | $(GOLINT) ; $(info $(M) running golint...) @ ## Run golint - $Q $(GOLINT) -set_exit_status $(PKGS) - -.PHONY: fmt -fmt: ; $(info $(M) running gofmt...) @ ## Run gofmt on all source files - $Q $(GO) fmt $(PKGS) - -# Misc - -.PHONY: clean -clean: ; $(info $(M) cleaning...) @ ## Cleanup everything - @rm -rf $(BIN) - @rm -rf test/tests.* test/coverage.* - -.PHONY: help -help: - @grep -E '^[ a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ - awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-15s\033[0m %s\n", $$1, $$2}' - -.PHONY: version -version: - @echo $(VERSION) diff --git a/build-all-and-push.sh b/build-all-and-push.sh deleted file mode 100755 index aa37687..0000000 --- a/build-all-and-push.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -export DOCKER_BUILDKIT=1 -# BUILD -echo "Building" -make -# Make Image -echo "Creating Image" -make image -# Re-tagging docker file -echo "Tagging" -docker tag kubeip:latest ${REGISTRY}/kubeip:latest -# Pushing image -echo "Pushing Image" -gcloud docker -- push ${REGISTRY}kubeip:latest diff --git a/main.go b/cmd/main.go similarity index 62% rename from main.go rename to cmd/main.go index 0d9e8fb..7da6c80 100644 --- a/main.go +++ b/cmd/main.go @@ -1,4 +1,4 @@ -// Copyright © 2021 DoiT International +// Copyright © 2023 DoiT International // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -17,43 +17,47 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. - package main import ( - c "github.com/doitintl/kubeip/pkg/client" - cfg "github.com/doitintl/kubeip/pkg/config" + "github.com/doitintl/kubeip/pkg/config" + "github.com/doitintl/kubeip/pkg/controller" "github.com/doitintl/kubeip/pkg/kipcompute" "github.com/sirupsen/logrus" ) -var config *cfg.Config -var version string -var buildDate string +var ( + version string + buildDate string + gitCommit string + gitBranch string +) func main() { - config, _ = cfg.NewConfig() - logrus.Info(config) + logger := logrus.New() + cfg := config.NewConfig() + logger.Info(cfg) + cluster, err := kipcompute.ClusterName() if err != nil { - logrus.Fatal(err) - panic(err) + logger.WithError(err).Fatal("Failed to get cluster name") } - projectID, err := kipcompute.ProjectName() + + project, err := kipcompute.ProjectName() if err != nil { - logrus.Fatal(err) - panic(err) + logger.WithError(err).Fatal("Failed to get project name") } - logrus.Info(config.AdditionalNodePools) - logrus.WithFields(logrus.Fields{ - "Cluster name": cluster, - "Project name": projectID, - "Version": version, - "Build Date": buildDate, + + logger.WithFields(logrus.Fields{ + "Cluster": cluster, + "Project": project, + "Version": version, + "Build Date": buildDate, + "Git Commit": gitCommit, + "Git Branch": gitBranch, }).Info("kubeIP is starting") - err = c.Run(config) - if err != nil { - logrus.Fatal(err) - panic(err) + + if err = controller.Start(logger, project, cluster, cfg); err != nil { + logrus.WithError(err).Fatal("Failed to start kubeIP controller") } } diff --git a/go.mod b/go.mod index c2c47ee..3c7ec69 100644 --- a/go.mod +++ b/go.mod @@ -1,58 +1,73 @@ module github.com/doitintl/kubeip -go 1.20 +go 1.21 require ( cloud.google.com/go/compute/metadata v0.2.3 - github.com/sirupsen/logrus v1.8.1 - github.com/spf13/viper v1.0.2 - golang.org/x/net v0.7.0 - golang.org/x/oauth2 v0.4.0 - golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac - google.golang.org/api v0.103.0 - k8s.io/api v0.22.2 - k8s.io/apimachinery v0.22.2 - k8s.io/client-go v0.22.2 + github.com/pkg/errors v0.9.1 + github.com/sirupsen/logrus v1.9.3 + github.com/spf13/viper v1.16.0 + golang.org/x/net v0.15.0 + golang.org/x/oauth2 v0.12.0 + golang.org/x/time v0.3.0 + google.golang.org/api v0.142.0 + k8s.io/api v0.28.2 + k8s.io/apimachinery v0.28.2 + k8s.io/client-go v0.28.2 ) require ( - cloud.google.com/go/compute v1.15.1 // indirect + cloud.google.com/go/compute v1.23.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/fsnotify/fsnotify v1.4.9 // indirect - github.com/go-logr/logr v0.4.0 // indirect + github.com/emicklei/go-restful/v3 v3.9.0 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/go-logr/logr v1.2.4 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/uuid v1.3.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect - github.com/googleapis/gax-go/v2 v2.7.0 // indirect - github.com/googleapis/gnostic v0.5.5 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/google/uuid v1.3.1 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect + github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/imdario/mergo v0.3.6 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/magiconair/properties v1.8.5 // indirect - github.com/mitchellh/mapstructure v1.4.2 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pelletier/go-toml v1.9.4 // indirect - github.com/spf13/afero v1.6.0 // indirect - github.com/spf13/cast v1.4.1 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/spf13/afero v1.9.5 // indirect + github.com/spf13/cast v1.5.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.4.2 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/term v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/crypto v0.13.0 // indirect + golang.org/x/sys v0.12.0 // indirect + golang.org/x/term v0.12.0 // indirect + golang.org/x/text v0.13.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect - google.golang.org/grpc v1.53.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230913181813-007df8e322eb // indirect + google.golang.org/grpc v1.57.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/klog/v2 v2.9.0 // indirect - k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect - sigs.k8s.io/yaml v1.2.0 // indirect + k8s.io/klog/v2 v2.100.1 // indirect + k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect + k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index c497483..1048f06 100644 --- a/go.sum +++ b/go.sum @@ -3,75 +3,83 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.107.0 h1:qkj22L7bgkl6vIeZDlOY2po43Mx/TIa2Wsa7VR+PEww= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/compute v1.15.1 h1:7UGq3QknM33pw5xATlpzeoomNxsacIVvTqTTvbfajmE= -cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= +cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= +github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= -github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -85,11 +93,14 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -99,55 +110,66 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= -github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= +github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= -github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= -github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= -github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= -github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= +github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -156,63 +178,53 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= -github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo= -github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= -github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.9.4 h1:xR7vG4IXt5RWx6FfIjyAtsoMAtnc3C/rFXBBd2AjZwE= +github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLywzIhbKM= +github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= +github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= +github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= +github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= -github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= +github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= +github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= +github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.0.2 h1:Ncr3ZIuJn322w2k1qmzXDnkLAdQMlJqBa9kfAH+irso= -github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc= +github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -223,24 +235,32 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= +github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -263,6 +283,7 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -271,9 +292,10 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -282,8 +304,8 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -291,28 +313,46 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= -golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= +golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -321,11 +361,7 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -335,31 +371,44 @@ golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -389,14 +438,32 @@ golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= +golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -406,14 +473,24 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.103.0 h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ= -google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.142.0 h1:mf+7EJ94fi5ZcnpPy+m0Yv2dkz8bKm+UL0snTCuwXlY= +google.golang.org/api v0.142.0/go.mod h1:zJAN5o6HRqR7O+9qJUFOWrZkYE66RH+efPBdTLA4xBA= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -433,11 +510,31 @@ google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 h1:L6iMMGrtzgHsWofoFcihmDEMYeDR9KN/ThbPWGrh++g= +google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8= +google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 h1:nIgk/EEq3/YlnmVVXVnm14rC2oxgs1o0ong4sD/rd44= +google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5/go.mod h1:5DZzOUPCLYL3mNkQ0ms0F3EuUNZ7py1Bqeq6sxzI7/Q= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230913181813-007df8e322eb h1:Isk1sSH7bovx8Rti2wZK0UZF6oraBDK74uoyLEEVFN0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230913181813-007df8e322eb/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -446,9 +543,16 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= -google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -461,28 +565,22 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -491,24 +589,25 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.22.2 h1:M8ZzAD0V6725Fjg53fKeTJxGsJvRbk4TEm/fexHMtfw= -k8s.io/api v0.22.2/go.mod h1:y3ydYpLJAaDI+BbSe2xmGcqxiWHmWjkEeIbiwHvnPR8= -k8s.io/apimachinery v0.22.2 h1:ejz6y/zNma8clPVfNDLnPbleBo6MpoFy/HBiBqCouVk= -k8s.io/apimachinery v0.22.2/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= -k8s.io/client-go v0.22.2 h1:DaSQgs02aCC1QcwUdkKZWOeaVsQjYvWv8ZazcZ6JcHc= -k8s.io/client-go v0.22.2/go.mod h1:sAlhrkVDf50ZHx6z4K0S40wISNTarf1r800F+RlCF6U= -k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= -k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a h1:8dYfu/Fc9Gz2rNJKB9IQRGgQOh2clmRzNIPPY1xLY5g= -k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.28.2 h1:9mpl5mOb6vXZvqbQmankOfPIGiudghwCoLl1EYfUZbw= +k8s.io/api v0.28.2/go.mod h1:RVnJBsjU8tcMq7C3iaRSGMeaKt2TWEUXcpIt/90fjEg= +k8s.io/apimachinery v0.28.2 h1:KCOJLrc6gu+wV1BYgwik4AF4vXOlVJPdiqn0yAWWwXQ= +k8s.io/apimachinery v0.28.2/go.mod h1:RdzF87y/ngqk9H4z3EL2Rppv5jj95vGS/HaFXrLDApU= +k8s.io/client-go v0.28.2 h1:DNoYI1vGq0slMBN/SWKMZMw0Rq+0EQW6/AK4v9+3VeY= +k8s.io/client-go v0.28.2/go.mod h1:sMkApowspLuc7omj1FOSUxSoqjr+d5Q0Yc0LOFnYFJY= +k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= +k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= +k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/makefile b/makefile new file mode 100644 index 0000000..7b6673a --- /dev/null +++ b/makefile @@ -0,0 +1,74 @@ +# Go parameters +GOCMD=go +GOBUILD=$(GOCMD) build +GORUN=$(GOCMD) run +GOCLEAN=$(GOCMD) clean +GOTEST=$(GOCMD) test +GOGET=$(GOCMD) get +GOTOOL=$(GOCMD) tool +GOLINT=golangci-lint +LINT_CONFIG = $(CURDIR)/.golangci.yaml + +BIN=$(CURDIR)/.bin +BINARY_NAME=kubeip + +DATE ?= $(shell date +%FT%T%z) + +# get version from environment variable if set or use git describe (match SemVer) +VERSION := $(if $(VERSION),$(VERSION),$(shell git describe --tags --always --dirty --match="[0-9]*.[0-9]*.[0-9]*" 2> /dev/null || \ + cat $(CURDIR)/.version 2> /dev/null || echo v0)) + +# get commit from environment variable if set or use git commit +COMMIT := $(if $(COMMIT),$(COMMIT),$(shell git rev-parse --short HEAD 2>/dev/null)) +# get branch from environment variable if set or use git branch +BRANCH := $(if $(BRANCH),$(BRANCH),$(shell git rev-parse --abbrev-ref HEAD 2>/dev/null)) + +Q = $(if $(filter 1,$V),,@) +M = $(shell printf "\033[34;1m▶\033[0m") + +export CGO_ENABLED=0 + +# main task +all: lint test build ; $(info $(M) build, test and deploy ...) @ ## release cycle + +# Tools +setup-lint: + $(GOCMD) install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.54.2 + +# Tasks + +build: ; $(info $(M) building binary ...) @ ## build with local Go SDK + $(GOBUILD) -v \ + -ldflags '-X main.version=$(VERSION) -X main.buildDate=$(DATE) -X main.gitCommit=$(COMMIT) -X main.gitBranch=$(BRANCH)' \ + -o $(BIN)/$(BINARY_NAME) ./cmd/. + +lint: setup-lint; $(info $(M) running golangci-lint ...) @ ## run golangci-lint linters + # updating path since golangci-lint is looking for go binary and this may lead to + # conflict when multiple go versions are installed + $Q $(GOLINT) run -v -c $(LINT_CONFIG) --out-format checkstyle ./... > golangci-lint.out + +test: ; $(info $(M) running test ...) @ ## run tests with coverage + $Q $(GOCMD) fmt ./... + $Q $(GOTEST) -v -cover ./... -coverprofile=coverage.out + $Q $(GOTOOL) cover -func=coverage.out + +test-json: ; $(info $(M) running test output JSON ...) @ ## run tests with JSON report and coverage + $Q $(GOTEST) -v -cover ./... -coverprofile=coverage.out -json > test-report.out + $Q $(GOTOOL) cover -func=coverage.out + +precommit: lint test ; $(info $(M) test and lint ...) @ ## release cycle: test > lint + +testview: ; $(info $(M) generating coverage report ...) @ ## generate HTML coverage report + $(GOTOOL) cover -html=coverage.out + +clean: ; $(info $(M) cleaning...) @ ## cleanup everything + $Q $(GOCLEAN) + @rm -rf $(BIN) + @rm -rf test/tests.* test/coverage.* + +run: ; $(info $(M) running ...) @ ## run locally + $Q $(GORUN) -v cmd/main.go + +help: ## display help + @grep -E '^[ a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ + awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-15s\033[0m %s\n", $$1, $$2}' diff --git a/pkg/client/run.go b/pkg/client/run.go deleted file mode 100644 index ee670cc..0000000 --- a/pkg/client/run.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright © 2021 DoiT International -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package client - -import ( - cfg "github.com/doitintl/kubeip/pkg/config" - "github.com/doitintl/kubeip/pkg/controller" -) - -// Run start kubeip controller -func Run(config *cfg.Config) error { - return controller.Start(config) -} diff --git a/pkg/config/config.go b/pkg/config/config.go index 7ed8000..2ba0207 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1,4 +1,4 @@ -// Copyright © 2021 DoiT International +// Copyright © 2023 DoiT International // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -27,8 +27,13 @@ import ( "github.com/spf13/viper" ) +const ( + defaultTicker = 5 * time.Minute +) + // Config kubeip configuration type Config struct { + KubeConfigPath string LabelKey string LabelValue string NodePool string @@ -44,13 +49,13 @@ type Config struct { } func setConfigDefaults() { + viper.SetDefault("KubeConfigPath", "") viper.SetDefault("LabelKey", "kubeip") viper.SetDefault("LabelValue", "reserved") viper.SetDefault("NodePool", "default-pool") viper.SetDefault("ForceAssignment", true) - viper.SetDefault("ForceAssignment", true) viper.SetDefault("AdditionalNodePools", "") - viper.SetDefault("Ticker", 5) + viper.SetDefault("Ticker", defaultTicker) viper.SetDefault("AllNodePools", false) viper.SetDefault("OrderByLabelKey", "priority") viper.SetDefault("OrderByDesc", true) @@ -60,11 +65,12 @@ func setConfigDefaults() { } // NewConfig initialize kubeip configuration -func NewConfig() (*Config, error) { - var AdditionalNodePools []string +func NewConfig() *Config { viper.SetEnvPrefix("kubeip") viper.AutomaticEnv() setConfigDefaults() + + var AdditionalNodePools []string AdditionalNodePoolsStr := viper.GetString("additionalnodepools") if len(AdditionalNodePoolsStr) > 0 { AdditionalNodePools = strings.Split(AdditionalNodePoolsStr, ",") @@ -84,5 +90,5 @@ func NewConfig() (*Config, error) { ClearLabels: viper.GetBool("clearlabels"), DryRun: viper.GetBool("dryrun"), } - return &c, nil + return &c } diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 4d77866..3f65312 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -21,21 +21,21 @@ package controller import ( - "errors" "fmt" - "golang.org/x/time/rate" "os" "os/signal" "strings" "syscall" "time" - cfg "github.com/doitintl/kubeip/pkg/config" + "github.com/doitintl/kubeip/pkg/config" "github.com/doitintl/kubeip/pkg/kipcompute" "github.com/doitintl/kubeip/pkg/types" "github.com/doitintl/kubeip/pkg/utils" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/net/context" + "golang.org/x/time/rate" api_v1 "k8s.io/api/core/v1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -45,9 +45,25 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/util/workqueue" ) +const ( + nodeResource = "node" + createEvent = "create" + deleteEvent = "delete" + maxInstances = 100 + rateLimit = 10 + burstTokens = 100 + baseDaley = time.Second + maxDelay = 100 * time.Second +) + +var ( + errTimeout = errors.New("timeout error") +) + // AddressInstanceTuple object type AddressInstanceTuple struct { address string @@ -56,14 +72,14 @@ type AddressInstanceTuple struct { // Controller object type Controller struct { - logger *logrus.Entry + logger logrus.FieldLogger clientset kubernetes.Interface queue workqueue.RateLimitingInterface informer cache.SharedIndexInformer instance chan<- types.Instance projectID string clusterName string - config *cfg.Config + config *config.Config ticker *time.Ticker processing bool } @@ -75,56 +91,91 @@ type Event struct { resourceType string } -var serverStartTime time.Time +var ( + serverStartTime time.Time + errEmptyPath = errors.New("empty path") +) + +const ( + maxRetries = 5 + prefix = "kube-system/kube-proxy-" +) + +func kubeConfigFromPath(kubepath string) (*rest.Config, error) { + if kubepath == "" { + return nil, errEmptyPath + } + + data, err := os.ReadFile(kubepath) + if err != nil { + return nil, errors.Wrapf(err, "reading kubeconfig at %s", kubepath) + } -const maxRetries = 5 + cfg, err := clientcmd.RESTConfigFromKubeConfig(data) + if err != nil { + return nil, errors.Wrapf(err, "building rest config from kubeconfig at %s", kubepath) + } -const prefix = "kube-system/kube-proxy-" + return cfg, nil +} + +func retrieveKubeConfig(log logrus.FieldLogger, cfg *config.Config) (*rest.Config, error) { + kubeconfig, err := kubeConfigFromPath(cfg.KubeConfigPath) + if err != nil && !errors.Is(err, errEmptyPath) { + return nil, errors.Wrap(err, "retrieving kube config from path") + } + + if kubeconfig != nil { + log.Debug("using kube config from env variables") + return kubeconfig, nil + } + + inClusterConfig, err := rest.InClusterConfig() + if err != nil { + return nil, errors.Wrap(err, "retrieving in cluster kube config") + } + log.Debug("using in cluster kube config") + return inClusterConfig, nil +} // Start kubeip controller -func Start(config *cfg.Config) error { - var kubeClient kubernetes.Interface - _, err := rest.InClusterConfig() +func Start(log logrus.FieldLogger, project, cluster string, cfg *config.Config) error { + restconfig, err := retrieveKubeConfig(log, cfg) if err != nil { - logrus.Fatal(err) - } else { - kubeClient = utils.GetClient() + return errors.Wrap(err, "retrieving kube config") + } + kubeClient, err := kubernetes.NewForConfig(restconfig) + if err != nil { + return errors.Wrap(err, "initializing kubernetes client") } informer := cache.NewSharedIndexInformer( &cache.ListWatch{ ListFunc: func(options meta_v1.ListOptions) (runtime.Object, error) { - return kubeClient.CoreV1().Pods(meta_v1.NamespaceAll).List(context.Background(), options) + return kubeClient.CoreV1().Pods(meta_v1.NamespaceAll).List(context.Background(), options) //nolint:wrapcheck }, WatchFunc: func(options meta_v1.ListOptions) (watch.Interface, error) { - return kubeClient.CoreV1().Pods(meta_v1.NamespaceAll).Watch(context.Background(), options) + return kubeClient.CoreV1().Pods(meta_v1.NamespaceAll).Watch(context.Background(), options) //nolint:wrapcheck }, }, &api_v1.Pod{}, - 0, //Skip resync + 0, // Skip resync cache.Indexers{}, ) - c := newResourceController(kubeClient, informer, "node") - c.projectID, err = kipcompute.ProjectName() + ctrl, err := newResourceController(log, project, cluster, kubeClient, informer) if err != nil { - logrus.Fatal(err) - return err + return errors.Wrap(err, "creating resource controller") } - c.clusterName, err = kipcompute.ClusterName() - if err != nil { - logrus.Fatal(err) - return err - } - c.config = config - c.ticker = time.NewTicker(c.config.Ticker * time.Minute) + ctrl.config = cfg + ctrl.ticker = time.NewTicker(ctrl.config.Ticker) stopCh := make(chan struct{}) defer close(stopCh) - //TODO Set size - instance := make(chan types.Instance, 100) - c.instance = instance - go c.Run(stopCh) - go c.forceAssignment() - kipcompute.Kubeip(instance, c.config) + // TODO Set size + instance := make(chan types.Instance, maxInstances) + ctrl.instance = instance + go ctrl.Run(stopCh) + go ctrl.forceAssignment() + kipcompute.Kubeip(instance, cfg) sigterm := make(chan os.Signal, 1) signal.Notify(sigterm, syscall.SIGTERM) signal.Notify(sigterm, syscall.SIGINT) @@ -133,40 +184,40 @@ func Start(config *cfg.Config) error { return nil } -func newResourceController(client kubernetes.Interface, informer cache.SharedIndexInformer, resourceType string) *Controller { +func newResourceController(log logrus.FieldLogger, project, cluster string, client kubernetes.Interface, informer cache.SharedIndexInformer) (*Controller, error) { queue := workqueue.NewRateLimitingQueue(workqueue.NewMaxOfRateLimiter( - workqueue.NewItemExponentialFailureRateLimiter(1*time.Second, 100*time.Second), + workqueue.NewItemExponentialFailureRateLimiter(baseDaley, maxDelay), // 10 qps, 100 bucket size. This is only for retry speed, and it's only the overall factor (not per item) - &workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)}, + &workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(rateLimit), burstTokens)}, )) - var newEvent Event - var err error - informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + + _, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { - newEvent.key, err = cache.MetaNamespaceKeyFunc(obj) - newEvent.eventType = "create" - newEvent.resourceType = resourceType + key, err := cache.MetaNamespaceKeyFunc(obj) if err == nil { - queue.Add(newEvent) + queue.Add(&Event{key, createEvent, nodeResource}) } }, DeleteFunc: func(obj interface{}) { - newEvent.key, err = cache.MetaNamespaceKeyFunc(obj) - newEvent.eventType = "delete" - newEvent.resourceType = resourceType + key, err := cache.MetaNamespaceKeyFunc(obj) if err == nil { - queue.Add(newEvent) + queue.Add(&Event{key, deleteEvent, nodeResource}) } }, }) + if err != nil { + return nil, errors.Wrap(err, "adding event handler") + } return &Controller{ - logger: logrus.WithField("pkg", "kubeip-"+resourceType), - clientset: client, - informer: informer, - queue: queue, - processing: false, - } + logger: log.WithField("pkg", "kubeip-node"), + projectID: project, + clusterName: cluster, + clientset: client, + informer: informer, + queue: queue, + processing: false, + }, nil } // Run starts the kubeip controller @@ -180,7 +231,7 @@ func (c *Controller) Run(stopCh <-chan struct{}) { go c.informer.Run(stopCh) if !cache.WaitForCacheSync(stopCh, c.HasSynced) { - utilruntime.HandleError(fmt.Errorf("timed out waiting for caches to sync")) + utilruntime.HandleError(errTimeout) return } @@ -201,7 +252,6 @@ func (c *Controller) LastSyncResourceVersion() string { func (c *Controller) runWorker() { for c.processNextItem() { - // continue looping } } @@ -246,7 +296,7 @@ func (c *Controller) isNodePoolMonitored(pool string) bool { func (c *Controller) processItem(newEvent Event) error { obj, _, err := c.informer.GetIndexer().GetByKey(newEvent.key) if err != nil { - return fmt.Errorf("error fetching object with key %s from store: %v", newEvent.key, err) + return errors.Wrapf(err, "getting object from informer by key %s", newEvent.key) } // get object's metadata objectMeta := utils.GetObjectMetaData(obj) @@ -305,12 +355,11 @@ func (c *Controller) processItem(newEvent Event) error { return nil } } - } return nil } -func isNodeReady(node api_v1.Node) bool { +func isNodeReady(node *api_v1.Node) bool { for _, condition := range node.Status.Conditions { if condition.Type == api_v1.NodeReady { // If the node is unknown we assume that it is ready, we do not want to do IP changes so rapidly. @@ -320,38 +369,36 @@ func isNodeReady(node api_v1.Node) bool { return false } -func (c *Controller) processAllNodes(shouldCheckOptimalIPAssignment bool) { +func (c *Controller) processAllNodes(shouldCheckOptimalIPAssignment bool) { //nolint:funlen,gocognit,gocyclo kubeClient := utils.GetClient() logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Info("Collecting Node List...") nodelist, _ := kubeClient.CoreV1().Nodes().List(context.Background(), meta_v1.ListOptions{}) - var pool string - var ok bool - var nodesOfInterest []types.Instance + nodesOfInterest := make([]types.Instance, 0, len(nodelist.Items)) - for _, node := range nodelist.Items { - labels := node.GetLabels() - if pool, ok = labels["cloud.google.com/gke-nodepool"]; ok { + for node := range nodelist.Items { + var inst types.Instance + labels := nodelist.Items[node].GetLabels() + if pool, ok := labels["cloud.google.com/gke-nodepool"]; ok { if !c.isNodePoolMonitored(pool) { continue } + inst.Pool = pool } else { logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Info("Did not found node pool") continue } - var inst types.Instance + if nodeZone, ok := labels["failure-domain.beta.kubernetes.io/zone"]; ok { inst.Zone = nodeZone } else { logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Info("Did not find zone") continue } - inst.ProjectID = c.projectID - inst.Name = node.GetName() - inst.Pool = pool + inst.Name = nodelist.Items[node].GetName() // If node is not ready we will basically remove the node IP just in case - if !isNodeReady(node) { + if !isNodeReady(&nodelist.Items[node]) { logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Infof("Node %s in zone %s is not ready, removing IP so we can reuse it. ", inst.Name, inst.Zone) // Delete the IP we will re-assign this err := kipcompute.DeleteIP(c.projectID, inst.Zone, inst.Name, c.config) @@ -362,7 +409,6 @@ func (c *Controller) processAllNodes(shouldCheckOptimalIPAssignment bool) { } continue } - nodesOfInterest = append(nodesOfInterest, inst) } @@ -421,7 +467,6 @@ func (c *Controller) processAllNodes(shouldCheckOptimalIPAssignment bool) { if len(toRemove) > 0 { logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Infof("Found %d ips %s in region %s which are not part of the top most addresses %s", len(toRemove), toRemove, region, topMostAddresses) for _, remove := range toRemove { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Infof("Instance %s in project %s in region %s uses suboptimal IP %s... Removing so we reassign", remove.instance.Name, c.projectID, region, toRemove) // Delete the IP we will re-assign this err := kipcompute.DeleteIP(c.projectID, remove.instance.Zone, remove.instance.Name, c.config) @@ -467,7 +512,8 @@ func (c *Controller) forceAssignment() { } func (c *Controller) assignMissingTags() { - nodePools := append(c.config.AdditionalNodePools, c.config.NodePool) + nodePools := c.config.AdditionalNodePools + nodePools = append(nodePools, c.config.NodePool) kubeClient := utils.GetClient() @@ -479,7 +525,6 @@ func (c *Controller) assignMissingTags() { if err != nil { logrus.Error(err) continue - } for _, node := range nodelist.Items { labels := node.GetLabels() @@ -490,7 +535,6 @@ func (c *Controller) assignMissingTags() { } logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "assignMissingTags"}).Infof("Found node without tag %s", node.GetName()) kipcompute.AddTagIfMissing(c.projectID, node.GetName(), nodeZone, c.config) - } else { continue } diff --git a/pkg/kipcompute/compute.go b/pkg/kipcompute/compute.go index 528bbd1..1b04aa6 100644 --- a/pkg/kipcompute/compute.go +++ b/pkg/kipcompute/compute.go @@ -21,7 +21,6 @@ package kipcompute import ( - "errors" "fmt" "math" "sort" @@ -30,69 +29,69 @@ import ( "time" "cloud.google.com/go/compute/metadata" - cfg "github.com/doitintl/kubeip/pkg/config" + "github.com/doitintl/kubeip/pkg/config" "github.com/doitintl/kubeip/pkg/types" "github.com/doitintl/kubeip/pkg/utils" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/net/context" - "golang.org/x/oauth2/google" - "google.golang.org/api/compute/v0.beta" - "google.golang.org/api/container/v1" + "google.golang.org/api/compute/v0.beta" //nolint:goimports +) + +const ( + waitTime = 2 * time.Second ) // ClusterName get GKE cluster name from metadata func ClusterName() (string, error) { - return metadata.InstanceAttributeValue("cluster-name") + return metadata.InstanceAttributeValue("cluster-name") //nolint:wrapcheck } // ProjectName get GCP project name from metadata func ProjectName() (string, error) { - return metadata.ProjectID() + return metadata.ProjectID() //nolint:wrapcheck } -func getPriorityOrder(address *compute.Address, config *cfg.Config) int { +func getPriorityOrder(address *compute.Address, cfg *config.Config) int { var defaultValue int - if config.OrderByDesc { + if cfg.OrderByDesc { defaultValue = math.MinInt } else { defaultValue = math.MaxInt } - strVal, ok := address.Labels[config.OrderByLabelKey] + strVal, ok := address.Labels[cfg.OrderByLabelKey] if ok { intVal, err := strconv.Atoi(strVal) if err != nil { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "getPriorityOrder"}).Errorf("Address %s has errors. Failed to convert order by label value %s with value %s to integer", address.Name, config.OrderByLabelKey, strVal, err) + logrus.WithFields(logrus.Fields{ + "pkg": "kubeip", + "function": "getPriorityOrder", + }).WithError(err). + Errorf("Address %s has errors. Failed to convert order by label value %s with value %s to integer", address.Name, cfg.OrderByLabelKey, strVal) return defaultValue } return intVal - } return defaultValue } // GetAllAddresses retrieves all addresses matching the query. -func GetAllAddresses(projectID string, region string, filterJustReserved bool, config *cfg.Config) (*compute.AddressList, error) { - return getAllAddresses(projectID, region, config.NodePool, filterJustReserved, config) +func GetAllAddresses(projectID, region string, filterJustReserved bool, cfg *config.Config) (*compute.AddressList, error) { + return getAllAddresses(projectID, region, cfg.NodePool, filterJustReserved, cfg) } -func getAllAddresses(projectID string, region string, pool string, filterJustReserved bool, config *cfg.Config) (*compute.AddressList, error) { - hc, err := google.DefaultClient(context.Background(), container.CloudPlatformScope) - if err != nil { - logrus.Error(err) - return nil, err - } - computeService, err := compute.New(hc) +func getAllAddresses(projectID, region, pool string, filterJustReserved bool, cfg *config.Config) (*compute.AddressList, error) { + computeService, err := compute.NewService(context.Background()) if err != nil { - logrus.Error(err) - return nil, err + return nil, errors.Wrap(err, "failed to get create compute service") } var filter string - if config.AllNodePools || strings.EqualFold(pool, config.NodePool) { - filter = "(labels." + config.LabelKey + "=" + config.LabelValue + ")" + " AND (-labels." + config.LabelKey + "-node-pool:*)" + if cfg.AllNodePools || strings.EqualFold(pool, cfg.NodePool) { + filter = "(labels." + cfg.LabelKey + "=" + cfg.LabelValue + ")" + " AND (-labels." + cfg.LabelKey + "-node-pool:*)" } else { - filter = "(labels." + config.LabelKey + "=" + config.LabelValue + ")" + " AND " + "(labels." + config.LabelKey + "-node-pool=" + pool + ")" + filter = "(labels." + cfg.LabelKey + "=" + cfg.LabelValue + ")" + " AND " + "(labels." + cfg.LabelKey + "-node-pool=" + pool + ")" } var computedFilter string @@ -106,17 +105,16 @@ func getAllAddresses(projectID string, region string, pool string, filterJustRes addresses, err = computeService.Addresses.List(projectID, region).Filter(computedFilter).Do() if err != nil { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "getAllAddresses"}).Errorf("Failed to list IP addresses: %q", err) - return nil, err + return nil, errors.Wrap(err, "failed to list addresses") } // Right now the SDK does not support filter and order together, so we do it programmatically. sort.SliceStable(addresses.Items, func(i, j int) bool { address1 := addresses.Items[i] address2 := addresses.Items[j] - val1 := getPriorityOrder(address1, config) - val2 := getPriorityOrder(address2, config) - if config.OrderByDesc { + val1 := getPriorityOrder(address1, cfg) + val2 := getPriorityOrder(address2, cfg) + if cfg.OrderByDesc { return val1 > val2 } return val1 < val2 @@ -125,74 +123,56 @@ func getAllAddresses(projectID string, region string, pool string, filterJustRes return addresses, nil } -func findFreeAddress(projectID string, region string, pool string, config *cfg.Config) (types.IPAddress, error) { - addresses, err := getAllAddresses(projectID, region, pool, true, config) +func findFreeAddress(projectID, region, pool string, cfg *config.Config) (*types.IPAddress, error) { + addresses, err := getAllAddresses(projectID, region, pool, true, cfg) if err != nil { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "findFreeAddress"}).Errorf("Failed to list IP addresses in region %s: %q", region, err) - return types.IPAddress{IP: "", Labels: map[string]string{}}, err + return nil, err } if len(addresses.Items) != 0 { address := addresses.Items[0] - return types.IPAddress{IP: address.Address, Labels: address.Labels}, nil + return &types.IPAddress{IP: address.Address, Labels: address.Labels}, nil } - return types.IPAddress{IP: "", Labels: map[string]string{}}, errors.New("no free address found") - + return nil, errors.New("no free address found") } // DeleteIP delete current IP on GKE node -func DeleteIP(projectID string, zone string, instance string, config *cfg.Config) error { - hc, err := google.DefaultClient(context.Background(), container.CloudPlatformScope) +func DeleteIP(projectID, zone, instance string, cfg *config.Config) error { + computeService, err := compute.NewService(context.Background()) if err != nil { - logrus.Fatalf("Could not get authenticated client: %v", err) - } - - computeService, err := compute.New(hc) - if err != nil { - logrus.Infof(err.Error()) - return err + return errors.Wrap(err, "failed to get create compute service") } inst, err := computeService.Instances.Get(projectID, zone, instance).Do() if err != nil { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "DeleteIP"}).Errorf("Instance not found %s zone %s: %q", instance, zone, err) - return err + return errors.Wrapf(err, "failed to get instance %s zone %s", instance, zone) } if len(inst.NetworkInterfaces) > 0 && len(inst.NetworkInterfaces[0].AccessConfigs) > 0 { accessConfigName := inst.NetworkInterfaces[0].AccessConfigs[0].Name - if config.DryRun { + if cfg.DryRun { logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "DeleteIP"}).Infof("Deleted Access Config for %s zone %s new ip %s", instance, zone, accessConfigName) } else { op, err := computeService.Instances.DeleteAccessConfig(projectID, zone, instance, accessConfigName, "nic0").Do() if err != nil { - logrus.Errorf("DeleteAccessConfig %q", err) - return err + return errors.Wrap(err, "failed to delete access config") } err = waitForCompilation(projectID, zone, op) if err != nil { - return err + return errors.Wrap(err, "failed to wait for compilation") } } - } logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "DeleteIP"}).Infof("Deleted IP for %s zone %s", instance, zone) // Delete an prior tags. - utils.TagNode(instance, types.IPAddress{IP: "0.0.0.0", Labels: map[string]string{}}, config) + utils.TagNode(instance, &types.IPAddress{IP: "0.0.0.0", Labels: map[string]string{}}, cfg) return nil } -func addIP(projectID string, zone string, instance string, pool string, addr types.IPAddress, config *cfg.Config) error { - hc, err := google.DefaultClient(context.Background(), container.CloudPlatformScope) +func addIP(projectID, zone, instance string, addr *types.IPAddress, cfg *config.Config) error { + computeService, err := compute.NewService(context.Background()) if err != nil { - logrus.Fatalf("Could not get authenticated client: %v", err) + return errors.Wrap(err, "failed to get create compute service") } - - computeService, err := compute.New(hc) - if err != nil { - logrus.Infof(err.Error()) - return err - } - - if config.DryRun { + if cfg.DryRun { logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "addIP"}).Infof("Added Access Config for %s zone %s new ip %s", instance, zone, addr.IP) } else { accessConfig := &compute.AccessConfig{ @@ -201,14 +181,14 @@ func addIP(projectID string, zone string, instance string, pool string, addr typ NatIP: addr.IP, Kind: "compute#accessConfig", } - op, err := computeService.Instances.AddAccessConfig(projectID, zone, instance, "nic0", accessConfig).Do() + var op *compute.Operation + op, err = computeService.Instances.AddAccessConfig(projectID, zone, instance, "nic0", accessConfig).Do() if err != nil { - logrus.Errorf("AddAccessConfig %q", err) - return err + return errors.Wrap(err, "failed to add access config") } err = waitForCompilation(projectID, zone, op) if err != nil { - return err + return errors.Wrap(err, "failed to wait for compilation") } } @@ -216,56 +196,46 @@ func addIP(projectID string, zone string, instance string, pool string, addr typ return nil } -func replaceIP(projectID string, zone string, instance string, pool string, config *cfg.Config) error { +func replaceIP(projectID, zone, instance, pool string, cfg *config.Config) error { region := zone[:len(zone)-2] - addr, err := findFreeAddress(projectID, region, pool, config) + addr, err := findFreeAddress(projectID, region, pool, cfg) // Check if we found address. if err != nil { - logrus.Infof(err.Error()) - return err + return errors.Wrap(err, "failed to find free address") } - err = DeleteIP(projectID, zone, instance, config) + err = DeleteIP(projectID, zone, instance, cfg) if err != nil { - logrus.Infof(err.Error()) - return err + return errors.Wrap(err, "failed to delete IP") } - err = addIP(projectID, zone, instance, pool, addr, config) + err = addIP(projectID, zone, instance, addr, cfg) if err != nil { - logrus.Infof(err.Error()) - return err + return errors.Wrap(err, "failed to add IP") } logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "replaceIP"}).Infof("Replaced IP for %s zone %s new ip %s", instance, zone, addr.IP) oldNode, err := utils.GetNodeByIP(addr.IP) if err == nil { - utils.TagNode(oldNode, types.IPAddress{IP: "0.0.0.0", Labels: map[string]string{}}, config) + utils.TagNode(oldNode, &types.IPAddress{IP: "0.0.0.0", Labels: map[string]string{}}, cfg) } - utils.TagNode(instance, addr, config) + utils.TagNode(instance, addr, cfg) return nil - } -func waitForCompilation(projectID string, zone string, operation *compute.Operation) (err error) { - hc, err := google.DefaultClient(context.Background(), container.CloudPlatformScope) - if err != nil { - logrus.Fatalf("Could not get authenticated client: %v", err) - return err - } - computeService, err := compute.New(hc) +func waitForCompilation(projectID, zone string, operation *compute.Operation) (err error) { + computeService, err := compute.NewService(context.Background()) if err != nil { - logrus.Fatalf("Could not get create compute service: %v", err) - return err + return errors.Wrap(err, "failed to get create compute service") } for { - op, err := computeService.ZoneOperations.Get(projectID, zone, operation.Name).Do() + var op *compute.Operation + op, err = computeService.ZoneOperations.Get(projectID, zone, operation.Name).Do() if err != nil { - logrus.Errorf("ZoneOperations.Get %q %s", err, operation.Name) - return err + return errors.Wrapf(err, "failed to get zone operations resource: %s", operation.Name) } - if strings.ToLower(op.Status) != "done" { - time.Sleep(2 * time.Second) + if !strings.EqualFold(op.Status, "done") { + time.Sleep(waitTime) } else { return nil } @@ -273,25 +243,17 @@ func waitForCompilation(projectID string, zone string, operation *compute.Operat } // GetAddressUsedByInstance returns the IP used by this instance or the broadcast address otherwise. -func GetAddressUsedByInstance(projectID string, instance string, zone string, config *cfg.Config) (string, error) { - hc, err := google.DefaultClient(context.Background(), container.CloudPlatformScope) +func GetAddressUsedByInstance(projectID, instance, zone string, cfg *config.Config) (string, error) { + computeService, err := compute.NewService(context.Background()) if err != nil { - logrus.Fatalf("Could not get authenticated client: %v", err) - return "", err - } - - computeService, err := compute.New(hc) - if err != nil { - logrus.Fatalf("Could not get create compute service: %v", err) - return "", err + return "", errors.Wrap(err, "failed to get create compute service") } region := zone[:len(zone)-2] - filter := "(labels." + config.LabelKey + "=" + config.LabelValue + ")" + filter := "(labels." + cfg.LabelKey + "=" + cfg.LabelValue + ")" addresses, err := computeService.Addresses.List(projectID, region).Filter(filter).Do() if err != nil { - logrus.Fatalf("Could not list addresses for instance %s: %v", instance, err) - return "", err + return "", errors.Wrap(err, "failed to list addresses") } for _, addr := range addresses.Items { @@ -304,20 +266,14 @@ func GetAddressUsedByInstance(projectID string, instance string, zone string, co } // IsInstanceUsesReservedIP test if GKE node is using reserved IP -func IsInstanceUsesReservedIP(projectID string, instance string, zone string, config *cfg.Config) bool { - ctx := context.Background() - hc, err := google.DefaultClient(ctx, container.CloudPlatformScope) +func IsInstanceUsesReservedIP(projectID, instance, zone string, cfg *config.Config) bool { + computeService, err := compute.NewService(context.Background()) if err != nil { - logrus.Error(err) - return false - } - computeService, err := compute.New(hc) - if err != nil { - logrus.Error(err) + logrus.WithError(err).Error("failed to get create compute service") return false } region := zone[:len(zone)-2] - filter := "(labels." + config.LabelKey + "=" + config.LabelValue + ")" + filter := "(labels." + cfg.LabelKey + "=" + cfg.LabelValue + ")" addresses, err := computeService.Addresses.List(projectID, region).Filter("(status=IN_USE) AND " + filter).Do() if err != nil { logrus.Error(err) @@ -333,52 +289,39 @@ func IsInstanceUsesReservedIP(projectID string, instance string, zone string, co } // Kubeip replace GKE node IP -func Kubeip(instance <-chan types.Instance, config *cfg.Config) { +func Kubeip(instance <-chan types.Instance, cfg *config.Config) { for { inst := <-instance logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "Kubeip"}).Infof("Working on %s in zone %s", inst.Name, inst.Zone) - _ = replaceIP(inst.ProjectID, inst.Zone, inst.Name, inst.Pool, config) + _ = replaceIP(inst.ProjectID, inst.Zone, inst.Name, inst.Pool, cfg) } } -func getAddressDetails(ip string, region string, projectID string, config *cfg.Config) (types.IPAddress, error) { - hc, err := google.DefaultClient(context.Background(), container.CloudPlatformScope) - if err != nil { - logrus.Error(err) - return types.IPAddress{IP: "", Labels: map[string]string{}}, err - } - computeService, err := compute.New(hc) +func getAddressDetails(ip, region, projectID string) (*types.IPAddress, error) { + computeService, err := compute.NewService(context.Background()) if err != nil { - logrus.Error(err) - return types.IPAddress{IP: "", Labels: map[string]string{}}, err + return nil, errors.Wrap(err, "failed to get create compute service") } filter := "address=" + "\"" + ip + "\"" addresses, err := computeService.Addresses.List(projectID, region).Filter(filter).Do() if err != nil { - logrus.Error(err) - return types.IPAddress{IP: "", Labels: map[string]string{}}, err + return nil, errors.Wrap(err, "failed to list addresses") } if len(addresses.Items) != 1 { address := addresses.Items[0] logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "getAddressDetails"}).Infof("Node ip is reserved %s %s", ip, fmt.Sprint(address.Labels)) - return types.IPAddress{IP: address.Address, Labels: address.Labels}, nil + return &types.IPAddress{IP: address.Address, Labels: address.Labels}, nil } - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "getAddressDetails"}).Errorf("More than one address found %s", ip) - return types.IPAddress{IP: "", Labels: map[string]string{}}, fmt.Errorf("more than one address found for ip %s", ip) + return nil, errors.New("more than one address found") } -func isAddressReserved(ip string, region string, projectID string, config *cfg.Config) bool { - hc, err := google.DefaultClient(context.Background(), container.CloudPlatformScope) +func isAddressReserved(ip, region, projectID string) bool { + computeService, err := compute.NewService(context.Background()) if err != nil { - logrus.Error(err) - return false - } - computeService, err := compute.New(hc) - if err != nil { - logrus.Error(err) + logrus.WithError(err).Error("failed to get create compute service") return false } filter := "address=" + "\"" + ip + "\"" @@ -396,14 +339,10 @@ func isAddressReserved(ip string, region string, projectID string, config *cfg.C } // AddTagIfMissing add GKE node tag if missing -func AddTagIfMissing(projectID string, instance string, zone string, config *cfg.Config) { - hc, err := google.DefaultClient(context.Background(), container.CloudPlatformScope) - if err != nil { - logrus.Fatalf("Could not get authenticated client: %v", err) - return - } - computeService, err := compute.New(hc) +func AddTagIfMissing(projectID, instance, zone string, cfg *config.Config) { + computeService, err := compute.NewService(context.Background()) if err != nil { + logrus.WithError(err).Error("failed to get create compute service") return } inst, err := computeService.Instances.Get(projectID, zone, instance).Do() @@ -411,18 +350,17 @@ func AddTagIfMissing(projectID string, instance string, zone string, config *cfg return } var ip string - for _, config := range inst.NetworkInterfaces[0].AccessConfigs { - if config.NatIP != "" { - ip = config.NatIP + for _, c := range inst.NetworkInterfaces[0].AccessConfigs { + if c.NatIP != "" { + ip = c.NatIP } } - if isAddressReserved(ip, zone[:len(zone)-2], projectID, config) { - addressDetails, err := getAddressDetails(ip, zone, projectID, config) + if isAddressReserved(ip, zone[:len(zone)-2], projectID) { + addressDetails, err := getAddressDetails(ip, zone, projectID) if err != nil { return } logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "AddTagIfMissing"}).Infof("Tagging %s", instance) - utils.TagNode(instance, addressDetails, config) + utils.TagNode(instance, addressDetails, cfg) } - } diff --git a/pkg/utils/k8sutil.go b/pkg/utils/k8sutil.go index f1a954a..effebd8 100644 --- a/pkg/utils/k8sutil.go +++ b/pkg/utils/k8sutil.go @@ -22,13 +22,12 @@ package utils import ( "bytes" - "errors" "fmt" "strings" - cfg "github.com/doitintl/kubeip/pkg/config" + "github.com/doitintl/kubeip/pkg/config" "github.com/doitintl/kubeip/pkg/types" - + "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/net/context" appsv1 "k8s.io/api/apps/v1" @@ -61,12 +60,12 @@ func Contains(s []string, e string) bool { // GetClient returns a k8s clientset to the request from inside of cluster func GetClient() kubernetes.Interface { - config, err := rest.InClusterConfig() + cfg, err := rest.InClusterConfig() if err != nil { logrus.Fatalf("Can not get kubernetes config: %v", err) } - clientset, err := kubernetes.NewForConfig(config) + clientset, err := kubernetes.NewForConfig(cfg) if err != nil { logrus.Fatalf("Can not create kubernetes client: %v", err) } @@ -76,7 +75,6 @@ func GetClient() kubernetes.Interface { // GetObjectMetaData returns metadata of a given k8s object func GetObjectMetaData(obj interface{}) metav1.ObjectMeta { - var objectMeta metav1.ObjectMeta switch object := obj.(type) { @@ -106,11 +104,11 @@ func GetObjectMetaData(obj interface{}) metav1.ObjectMeta { return objectMeta } -func clearLabels(m map[string]string, config *cfg.Config) string { +func clearLabels(m map[string]string, cfg *config.Config) string { stringBuffer := new(bytes.Buffer) for key := range m { - if !strings.EqualFold(key, config.OrderByLabelKey) && - !strings.EqualFold(key, config.LabelKey) && + if !strings.EqualFold(key, cfg.OrderByLabelKey) && + !strings.EqualFold(key, cfg.LabelKey) && !strings.Contains(key, "kubip_assigned") && !strings.Contains(key, "kubernetes") && !strings.Contains(key, "google") && @@ -121,11 +119,11 @@ func clearLabels(m map[string]string, config *cfg.Config) string { return stringBuffer.String() } -func createLabelKeyValuePairs(m map[string]string, config *cfg.Config) string { +func createLabelKeyValuePairs(m map[string]string, cfg *config.Config) string { stringBuffer := new(bytes.Buffer) for key, value := range m { - if !strings.EqualFold(key, config.OrderByLabelKey) && - !strings.EqualFold(key, config.LabelKey) && + if !strings.EqualFold(key, cfg.OrderByLabelKey) && + !strings.EqualFold(key, cfg.LabelKey) && !strings.Contains(key, "kubip_assigned") && !strings.Contains(key, "kubernetes") && !strings.Contains(key, "google") && @@ -137,34 +135,35 @@ func createLabelKeyValuePairs(m map[string]string, config *cfg.Config) string { } // TagNode tag GKE node with "kubip_assigned" label (with typo) and also copy the labels present on the address if the copyLabels flag is set to true -func TagNode(node string, ip types.IPAddress, config *cfg.Config) { +func TagNode(node string, ip *types.IPAddress, cfg *config.Config) { kubeClient := GetClient() logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "tagNode"}).Infof("Tagging node %s as %s", node, ip.IP) - dashIP := strings.Replace(ip.IP, ".", "-", 4) + // replace . with - in IP address + dashIP := strings.Replace(ip.IP, ".", "-", 4) //nolint:gomnd var labelString string - if config.CopyLabels { + if cfg.CopyLabels { var labelsToClear string - if config.ClearLabels { + if cfg.ClearLabels { result, err := kubeClient.CoreV1().Nodes().Get(context.Background(), node, metav1.GetOptions{}) if err != nil { logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "tagNode"}).Error(err) } else { logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "tagNode"}).Infof("Clear label tag for node %s with ip %s and clear tags %s", node, ip.IP, result.Labels) - createLabelKeyValuePairs(result.Labels, config) - labelsToClear = clearLabels(result.Labels, config) + createLabelKeyValuePairs(result.Labels, cfg) + labelsToClear = clearLabels(result.Labels, cfg) } } else { labelsToClear = "" } - labelString = "{" + "\"" + "kubip_assigned" + "\":\"" + dashIP + "\"" + labelsToClear + createLabelKeyValuePairs(ip.Labels, config) + "}" + labelString = "{" + "\"" + "kubip_assigned" + "\":\"" + dashIP + "\"" + labelsToClear + createLabelKeyValuePairs(ip.Labels, cfg) + "}" } else { labelString = "{" + "\"" + "kubip_assigned" + "\":\"" + dashIP + "\"" + "}" } patch := fmt.Sprintf(`{"metadata":{"labels":%v}}`, labelString) - if config.DryRun { + if cfg.DryRun { logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "tagNode"}).Infof("Tagging node %s as %s with tags %s ", node, ip.IP, labelString) } else { logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "tagNode"}).Infof("Tagging node %s as %s with tags %s ", node, ip.IP, labelString) @@ -174,24 +173,21 @@ func TagNode(node string, ip types.IPAddress, config *cfg.Config) { logrus.Error(err) } } - } // GetNodeByIP get GKE node by IP func GetNodeByIP(ip string) (string, error) { kubeClient := GetClient() - dashIP := strings.Replace(ip, ".", "-", 4) + dashIP := strings.Replace(ip, ".", "-", 4) //nolint:gomnd label := fmt.Sprintf("kubip_assigned=%v", dashIP) l, err := kubeClient.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{ LabelSelector: label, }) if err != nil { - logrus.Error(err) - return "", err + return "", errors.Wrap(err, "failed to list nodes") } if len(l.Items) == 0 { return "", errors.New("did not found matching node with IP") } return l.Items[0].GetName(), nil - } From 5dd9cf591dfbc38f019347bc2fdaabd749b7e464 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Wed, 20 Sep 2023 16:56:08 +0300 Subject: [PATCH 02/66] fix build and push --- .github/workflows/build.yaml | 4 +++- Dockerfile | 3 +++ test.sh | 29 ----------------------------- 3 files changed, 6 insertions(+), 30 deletions(-) delete mode 100644 test.sh diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 1347e81..9344f61 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -1,6 +1,7 @@ name: build on: + workflow_dispatch: push: branches: - '*' @@ -40,7 +41,8 @@ jobs: runs-on: ubuntu-latest needs: validate - if: ${{ !contains(github.event.head_commit.message,'[skip ci]') && github.event_name != 'pull_request' }} + # build only on master branch and tags + if: ${{ !contains(github.event.head_commit.message,'[skip ci]') && (github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/'))) }} steps: - name: checkout uses: actions/checkout@v4 diff --git a/Dockerfile b/Dockerfile index a806a06..f59fd02 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,9 @@ FROM golang:1.21-alpine AS builder # add CA certificates and TZ for local time RUN apk --update add ca-certificates tzdata make git +# create a working directory +WORKDIR /app + # Retrieve application dependencies. # This allows the container build to reuse cached dependencies. # Expecting to copy go.mod and if present go.sum. diff --git a/test.sh b/test.sh deleted file mode 100644 index 26a7641..0000000 --- a/test.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -xe -if [ -z "$REGION" ]; then - echo REGION not defined! - exit -fi -if [ -z "$CLUSTER" ]; then - echo CLUSTER not defined! - exit -fi - -NODES=`gcloud container node-pools describe default-pool --cluster $CLUSTER|grep initialNodeCount|awk '{print $2}'` -NEW_NODES=$(($NODES + 1)) -gcloud compute addresses create kubeip-test-1 --region $REGION -gcloud beta compute addresses update kubeip-test-1 --update-labels kubeip=reserved --region us-central1 -IP1=`gcloud compute addresses describe kubeip-test-1 --region $REGION|grep address:|awk '{print $2}'` -gcloud compute addresses create kubeip-test-2 --region $REGION -gcloud beta compute addresses update kubeip-test-2 --update-labels kubeip=reserved --region us-central1 -IP2=`gcloud compute addresses describe kubeip-test-2 --region $REGION|grep address:|awk '{print $2}'` -gcloud beta container clusters resize $CLUSTER --node-pool default-pool --size $NEW_NODES --quiet - -STATUS1=`gcloud compute addresses describe kubeip-test-1 --region $REGION|grep status|awk '{print $2}'` -STATUS2=`gcloud compute addresses describe kubeip-test-2 --region $REGION|grep status|awk '{print $2}'` -echo 'expecting one IP IN_USE and one RESERVED' -echo 'Results:' -echo $STATUS1 '--' $STATUS2 - -gcloud beta container clusters resize $CLUSTER --node-pool default-pool --size $NODES --quiet -gcloud compute addresses delete kubeip-test-1 --region $REGION --quiet -gcloud compute addresses delete kubeip-test-2 --region $REGION --quiet From 7d8fea67a89f65344b3cb916fee41317351a182d Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Wed, 20 Sep 2023 18:36:17 +0300 Subject: [PATCH 03/66] refactor main support plain text and json log --- cmd/main.go | 83 +++++++++++++++++++++--------------- go.mod | 3 +- go.sum | 14 +++--- pkg/config/config.go | 4 ++ pkg/controller/controller.go | 6 +-- 5 files changed, 66 insertions(+), 44 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 7da6c80..cda93e6 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,22 +1,3 @@ -// Copyright © 2023 DoiT International -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. package main import ( @@ -24,6 +5,7 @@ import ( "github.com/doitintl/kubeip/pkg/controller" "github.com/doitintl/kubeip/pkg/kipcompute" "github.com/sirupsen/logrus" + "sigs.k8s.io/controller-runtime/pkg/manager/signals" ) var ( @@ -33,31 +15,64 @@ var ( gitBranch string ) -func main() { +func prepareLogger(level string, json bool) *logrus.Entry { logger := logrus.New() + + // set debug log level + switch level { + case "debug", "DEBUG": + logger.SetLevel(logrus.DebugLevel) + case "info", "INFO": + logger.SetLevel(logrus.InfoLevel) + case "warning", "WARNING": + logger.SetLevel(logrus.WarnLevel) + case "error", "ERROR": + logger.SetLevel(logrus.ErrorLevel) + case "fatal", "FATAL": + logger.SetLevel(logrus.FatalLevel) + case "panic", "PANIC": + logger.SetLevel(logrus.PanicLevel) + default: + logger.SetLevel(logrus.WarnLevel) + } + + logger.SetFormatter(&logrus.TextFormatter{ + FullTimestamp: true, + }) + if json { + logger.SetFormatter(&logrus.JSONFormatter{}) + } + + log := logger.WithFields(logrus.Fields{ + "version": version, + }) + + return log +} + +func main() { + ctx := signals.SetupSignalHandler() cfg := config.NewConfig() - logger.Info(cfg) + logger := prepareLogger(cfg.LogLevel, cfg.LogJSON) + logger.WithField("config", cfg).Info("using kubeIP configuration") cluster, err := kipcompute.ClusterName() if err != nil { - logger.WithError(err).Fatal("Failed to get cluster name") + logger.WithError(err).Fatal("failed to get cluster name") } project, err := kipcompute.ProjectName() if err != nil { - logger.WithError(err).Fatal("Failed to get project name") + logger.WithError(err).Fatal("failed to get project name") } - logger.WithFields(logrus.Fields{ - "Cluster": cluster, - "Project": project, - "Version": version, - "Build Date": buildDate, - "Git Commit": gitCommit, - "Git Branch": gitBranch, - }).Info("kubeIP is starting") - - if err = controller.Start(logger, project, cluster, cfg); err != nil { - logrus.WithError(err).Fatal("Failed to start kubeIP controller") + logger = logger.WithFields(logrus.Fields{ + "cluster": cluster, + "project": project, + }) + logger.Info("starting kubeIP controller") + + if err = controller.Start(ctx, logger, project, cluster, cfg); err != nil { + logrus.WithError(err).Fatal("failed to start kubeIP controller") } } diff --git a/go.mod b/go.mod index 3c7ec69..da86141 100644 --- a/go.mod +++ b/go.mod @@ -8,12 +8,12 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.16.0 golang.org/x/net v0.15.0 - golang.org/x/oauth2 v0.12.0 golang.org/x/time v0.3.0 google.golang.org/api v0.142.0 k8s.io/api v0.28.2 k8s.io/apimachinery v0.28.2 k8s.io/client-go v0.28.2 + sigs.k8s.io/controller-runtime v0.16.2 ) require ( @@ -53,6 +53,7 @@ require ( github.com/subosito/gotenv v1.4.2 // indirect go.opencensus.io v0.24.0 // indirect golang.org/x/crypto v0.13.0 // indirect + golang.org/x/oauth2 v0.12.0 // indirect golang.org/x/sys v0.12.0 // indirect golang.org/x/term v0.12.0 // indirect golang.org/x/text v0.13.0 // indirect diff --git a/go.sum b/go.sum index 1048f06..01ed5a9 100644 --- a/go.sum +++ b/go.sum @@ -198,10 +198,10 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.9.4 h1:xR7vG4IXt5RWx6FfIjyAtsoMAtnc3C/rFXBBd2AjZwE= -github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLywzIhbKM= -github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= -github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= +github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= +github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -458,8 +458,8 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= -golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= +golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= +golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -605,6 +605,8 @@ k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/controller-runtime v0.16.2 h1:mwXAVuEk3EQf478PQwQ48zGOXvW27UJc8NHktQVuIPU= +sigs.k8s.io/controller-runtime v0.16.2/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= diff --git a/pkg/config/config.go b/pkg/config/config.go index 2ba0207..b0d67d0 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -45,6 +45,8 @@ type Config struct { OrderByDesc bool CopyLabels bool ClearLabels bool + LogLevel string + LogJSON bool DryRun bool } @@ -61,6 +63,8 @@ func setConfigDefaults() { viper.SetDefault("OrderByDesc", true) viper.SetDefault("CopyLabels", true) viper.SetDefault("ClearLabels", true) + viper.SetDefault("LogLevel", "info") + viper.SetDefault("LogJSON", false) viper.SetDefault("DryRun", false) } diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 3f65312..20f9595 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -139,12 +139,12 @@ func retrieveKubeConfig(log logrus.FieldLogger, cfg *config.Config) (*rest.Confi } // Start kubeip controller -func Start(log logrus.FieldLogger, project, cluster string, cfg *config.Config) error { - restconfig, err := retrieveKubeConfig(log, cfg) +func Start(ctx context.Context, log logrus.FieldLogger, project, cluster string, cfg *config.Config) error { + kubeConfig, err := retrieveKubeConfig(log, cfg) if err != nil { return errors.Wrap(err, "retrieving kube config") } - kubeClient, err := kubernetes.NewForConfig(restconfig) + kubeClient, err := kubernetes.NewForConfig(kubeConfig) if err != nil { return errors.Wrap(err, "initializing kubernetes client") } From 8a072f5074de59875affaea13b69110dbb31499f Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Wed, 20 Sep 2023 22:44:09 +0300 Subject: [PATCH 04/66] cleanup and fix errors --- cmd/main.go | 13 +- {pkg => internal}/config/config.go | 20 --- {pkg => internal}/controller/controller.go | 189 ++++++++------------- {pkg => internal}/kipcompute/compute.go | 26 +-- internal/types/types.go | 16 ++ {pkg => internal}/utils/k8sutil.go | 36 +--- pkg/types/types.go | 36 ---- 7 files changed, 115 insertions(+), 221 deletions(-) rename {pkg => internal}/config/config.go (66%) rename {pkg => internal}/controller/controller.go (57%) rename {pkg => internal}/kipcompute/compute.go (88%) create mode 100644 internal/types/types.go rename {pkg => internal}/utils/k8sutil.go (69%) delete mode 100644 pkg/types/types.go diff --git a/cmd/main.go b/cmd/main.go index cda93e6..d1f56ff 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,9 +1,9 @@ package main import ( - "github.com/doitintl/kubeip/pkg/config" - "github.com/doitintl/kubeip/pkg/controller" - "github.com/doitintl/kubeip/pkg/kipcompute" + "github.com/doitintl/kubeip/internal/config" + "github.com/doitintl/kubeip/internal/controller" + "github.com/doitintl/kubeip/internal/kipcompute" "github.com/sirupsen/logrus" "sigs.k8s.io/controller-runtime/pkg/manager/signals" ) @@ -67,8 +67,11 @@ func main() { } logger = logger.WithFields(logrus.Fields{ - "cluster": cluster, - "project": project, + "cluster": cluster, + "project": project, + "branch": gitBranch, + "commit": gitCommit, + "buildDate": buildDate, }) logger.Info("starting kubeIP controller") diff --git a/pkg/config/config.go b/internal/config/config.go similarity index 66% rename from pkg/config/config.go rename to internal/config/config.go index b0d67d0..5e0d759 100644 --- a/pkg/config/config.go +++ b/internal/config/config.go @@ -1,23 +1,3 @@ -// Copyright © 2023 DoiT International -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - package config import ( diff --git a/pkg/controller/controller.go b/internal/controller/controller.go similarity index 57% rename from pkg/controller/controller.go rename to internal/controller/controller.go index 20f9595..3b177d0 100644 --- a/pkg/controller/controller.go +++ b/internal/controller/controller.go @@ -1,43 +1,21 @@ -// Copyright © 2021 DoiT International -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - package controller import ( "fmt" "os" - "os/signal" "strings" - "syscall" "time" - "github.com/doitintl/kubeip/pkg/config" - "github.com/doitintl/kubeip/pkg/kipcompute" - "github.com/doitintl/kubeip/pkg/types" - "github.com/doitintl/kubeip/pkg/utils" + "github.com/doitintl/kubeip/internal/config" + "github.com/doitintl/kubeip/internal/kipcompute" + "github.com/doitintl/kubeip/internal/types" + "github.com/doitintl/kubeip/internal/utils" "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/net/context" "golang.org/x/time/rate" - api_v1 "k8s.io/api/core/v1" - meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" @@ -50,14 +28,15 @@ import ( ) const ( - nodeResource = "node" - createEvent = "create" - deleteEvent = "delete" - maxInstances = 100 - rateLimit = 10 - burstTokens = 100 - baseDaley = time.Second - maxDelay = 100 * time.Second + nodeResource = "node" + createEvent = "create" + deleteEvent = "delete" + maxInstances = 100 + rateLimit = 10 + burstTokens = 100 + baseDaley = time.Second + maxDelay = 100 * time.Second + nodeCacheSyncPeriod = 5 * time.Minute ) var ( @@ -75,7 +54,7 @@ type Controller struct { logger logrus.FieldLogger clientset kubernetes.Interface queue workqueue.RateLimitingInterface - informer cache.SharedIndexInformer + informer cache.SharedInformer instance chan<- types.Instance projectID string clusterName string @@ -148,21 +127,21 @@ func Start(ctx context.Context, log logrus.FieldLogger, project, cluster string, if err != nil { return errors.Wrap(err, "initializing kubernetes client") } - informer := cache.NewSharedIndexInformer( + // Create a new Node informer + nodeInformer := cache.NewSharedInformer( &cache.ListWatch{ - ListFunc: func(options meta_v1.ListOptions) (runtime.Object, error) { - return kubeClient.CoreV1().Pods(meta_v1.NamespaceAll).List(context.Background(), options) //nolint:wrapcheck + ListFunc: func(options metav1.ListOptions) (object runtime.Object, err error) { + return kubeClient.CoreV1().Nodes().List(context.Background(), options) //nolint:wrapcheck }, - WatchFunc: func(options meta_v1.ListOptions) (watch.Interface, error) { - return kubeClient.CoreV1().Pods(meta_v1.NamespaceAll).Watch(context.Background(), options) //nolint:wrapcheck + WatchFunc: func(options metav1.ListOptions) (retWc watch.Interface, err error) { + return kubeClient.CoreV1().Nodes().Watch(context.Background(), options) //nolint:wrapcheck }, }, - &api_v1.Pod{}, - 0, // Skip resync - cache.Indexers{}, + &corev1.Node{}, + nodeCacheSyncPeriod, ) - ctrl, err := newResourceController(log, project, cluster, kubeClient, informer) + ctrl, err := newResourceController(log, project, cluster, kubeClient, nodeInformer) if err != nil { return errors.Wrap(err, "creating resource controller") } @@ -170,21 +149,20 @@ func Start(ctx context.Context, log logrus.FieldLogger, project, cluster string, ctrl.ticker = time.NewTicker(ctrl.config.Ticker) stopCh := make(chan struct{}) defer close(stopCh) - // TODO Set size + instance := make(chan types.Instance, maxInstances) ctrl.instance = instance go ctrl.Run(stopCh) go ctrl.forceAssignment() + kipcompute.Kubeip(instance, cfg) - sigterm := make(chan os.Signal, 1) - signal.Notify(sigterm, syscall.SIGTERM) - signal.Notify(sigterm, syscall.SIGINT) - <-sigterm + // wait till context is canceled + <-ctx.Done() return nil } -func newResourceController(log logrus.FieldLogger, project, cluster string, client kubernetes.Interface, informer cache.SharedIndexInformer) (*Controller, error) { +func newResourceController(log logrus.FieldLogger, project, cluster string, client kubernetes.Interface, informer cache.SharedInformer) (*Controller, error) { queue := workqueue.NewRateLimitingQueue(workqueue.NewMaxOfRateLimiter( workqueue.NewItemExponentialFailureRateLimiter(baseDaley, maxDelay), // 10 qps, 100 bucket size. This is only for retry speed, and it's only the overall factor (not per item) @@ -210,7 +188,7 @@ func newResourceController(log logrus.FieldLogger, project, cluster string, clie } return &Controller{ - logger: log.WithField("pkg", "kubeip-node"), + logger: log.WithField("internal", "kubeip-node"), projectID: project, clusterName: cluster, clientset: client, @@ -267,11 +245,11 @@ func (c *Controller) processNextItem() bool { // No error, reset the ratelimit counters c.queue.Forget(newEvent) } else if c.queue.NumRequeues(newEvent) < maxRetries { - c.logger.Errorf("Error processing %s (will retry): %v", newEvent.(Event).key, err) + c.logger.WithError(err).Errorf("error processing %s (will retry)", newEvent.(Event).key) c.queue.AddRateLimited(newEvent) } else { // err != nil and too many retries - c.logger.Errorf("Error processing %s (giving up): %v", newEvent.(Event).key, err) + c.logger.WithError(err).Errorf("error processing %s (giving up)", newEvent.(Event).key) c.queue.Forget(newEvent) utilruntime.HandleError(err) } @@ -293,8 +271,9 @@ func (c *Controller) isNodePoolMonitored(pool string) bool { } return false } + func (c *Controller) processItem(newEvent Event) error { - obj, _, err := c.informer.GetIndexer().GetByKey(newEvent.key) + obj, _, err := c.informer.GetStore().GetByKey(newEvent.key) if err != nil { return errors.Wrapf(err, "getting object from informer by key %s", newEvent.key) } @@ -305,8 +284,6 @@ func (c *Controller) processItem(newEvent Event) error { switch newEvent.eventType { case "delete": if strings.HasPrefix(newEvent.key, prefix) { - node := newEvent.key[len(prefix):] - logrus.WithFields(logrus.Fields{"pkg": "kubeip-" + newEvent.resourceType, "function": "processItem"}).Infof("Processing removal to %v: %s ", newEvent.resourceType, node) // A node has been deleted... we need to check whether the assignment is still optimal c.forceAssignmentOnce(true) return nil @@ -318,40 +295,35 @@ func (c *Controller) processItem(newEvent Event) error { if strings.HasPrefix(newEvent.key, prefix) { kubeClient := utils.GetClient() node := newEvent.key[len(prefix):] - var options meta_v1.GetOptions + var options metav1.GetOptions options.Kind = "Node" options.APIVersion = "1" - nodeMeta, err := kubeClient.CoreV1().Nodes().Get(context.Background(), node, options) + var nodeMeta *corev1.Node + nodeMeta, err = kubeClient.CoreV1().Nodes().Get(context.Background(), node, options) if err != nil { - logrus.Infof(err.Error()) + return errors.Wrap(err, "failed to get node") } labels := nodeMeta.Labels var pool string var ok bool if pool, ok = labels["cloud.google.com/gke-nodepool"]; ok { - logrus.Infof("Node pool found %s", pool) if !c.isNodePoolMonitored(pool) { return nil } } else { - logrus.Infof("Did not found node pool. These are the labels present %s. ", labels) - return errors.New("Did not find node pool. ") + return errors.New("failed to find node pool") } var inst types.Instance if nodeZone, ok := labels["failure-domain.beta.kubernetes.io/zone"]; ok { - logrus.Infof("Zone pool found %s", nodeZone) inst.Zone = nodeZone } else { - logrus.Info("Did not find zone") - return errors.New("Did not find zone. ") + return errors.New("failed to find zone") } - logrus.WithFields(logrus.Fields{"pkg": "kubeip-" + newEvent.resourceType, "function": "processItem"}).Infof("Processing add to %v: %s ", newEvent.resourceType, node) inst.Name = node inst.ProjectID = c.projectID inst.Pool = pool c.instance <- inst - logrus.WithFields(logrus.Fields{"pkg": "kubeip-" + newEvent.resourceType, "function": "processItem"}).Infof("Processing node %s of cluster %s in zone %s", node, c.clusterName, inst.Zone) return nil } } @@ -359,20 +331,22 @@ func (c *Controller) processItem(newEvent Event) error { return nil } -func isNodeReady(node *api_v1.Node) bool { +func isNodeReady(node *corev1.Node) bool { for _, condition := range node.Status.Conditions { - if condition.Type == api_v1.NodeReady { + if condition.Type == corev1.NodeReady { // If the node is unknown we assume that it is ready, we do not want to do IP changes so rapidly. - return condition.Status == api_v1.ConditionTrue || condition.Status == api_v1.ConditionUnknown + return condition.Status == corev1.ConditionTrue || condition.Status == corev1.ConditionUnknown } } return false } -func (c *Controller) processAllNodes(shouldCheckOptimalIPAssignment bool) { //nolint:funlen,gocognit,gocyclo +func (c *Controller) processAllNodes(shouldCheckOptimalIPAssignment bool) error { //nolint:funlen,gocognit,gocyclo kubeClient := utils.GetClient() - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Info("Collecting Node List...") - nodelist, _ := kubeClient.CoreV1().Nodes().List(context.Background(), meta_v1.ListOptions{}) + nodelist, err := kubeClient.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) + if err != nil { + return errors.Wrap(err, "failed to list nodes") + } nodesOfInterest := make([]types.Instance, 0, len(nodelist.Items)) for node := range nodelist.Items { @@ -384,14 +358,14 @@ func (c *Controller) processAllNodes(shouldCheckOptimalIPAssignment bool) { //no } inst.Pool = pool } else { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Info("Did not found node pool") + c.logger.Warn("failed to find node pool") continue } if nodeZone, ok := labels["failure-domain.beta.kubernetes.io/zone"]; ok { inst.Zone = nodeZone } else { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Info("Did not find zone") + c.logger.Warn("failed to find zone") continue } inst.ProjectID = c.projectID @@ -399,13 +373,11 @@ func (c *Controller) processAllNodes(shouldCheckOptimalIPAssignment bool) { //no // If node is not ready we will basically remove the node IP just in case if !isNodeReady(&nodelist.Items[node]) { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Infof("Node %s in zone %s is not ready, removing IP so we can reuse it. ", inst.Name, inst.Zone) + c.logger.Debugf("node %s in zone %s is not ready, remove IP for reuse", inst.Name, inst.Zone) // Delete the IP we will re-assign this - err := kipcompute.DeleteIP(c.projectID, inst.Zone, inst.Name, c.config) + err = kipcompute.DeleteIP(c.projectID, inst.Zone, inst.Name, c.config) if err != nil { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}). - Errorf("Could not delete IP used by instance %s in zone %s. Aborting.", inst.Name, inst.Zone) - return + return errors.Wrap(err, "failed to delete IP") } continue } @@ -416,7 +388,7 @@ func (c *Controller) processAllNodes(shouldCheckOptimalIPAssignment bool) { //no if shouldCheckOptimalIPAssignment { // Determining the required IP per region regionsCount := make(map[string][]types.Instance) - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Infof("Collected %d Nodes of interest...calculating number of IPs required", len(nodesOfInterest)) + c.logger.Debugf("collected %d Nodes of interest...calculating number of IPs required", len(nodesOfInterest)) for _, inst := range nodesOfInterest { zone := inst.Zone region := zone[:len(zone)-2] @@ -425,12 +397,9 @@ func (c *Controller) processAllNodes(shouldCheckOptimalIPAssignment bool) { //no // Determining the most optimal nodes per region. for region, instances := range regionsCount { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Infof("Collected %d Nodes of interest...processing %d nodes instances within region %s", len(nodesOfInterest), len(instances), region) - addresses, err := kipcompute.GetAllAddresses(c.projectID, region, false, c.config) if err != nil { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Errorf("Could not retrieve addresses for project %s region %s. Aborting.", c.projectID, region) - return + return errors.Wrap(err, "failed to retrieve all addresses") } var topMostAddresses []string @@ -440,13 +409,10 @@ func (c *Controller) processAllNodes(shouldCheckOptimalIPAssignment bool) { //no // Retrieve all addresses in the region. var usedAddresses []AddressInstanceTuple - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Infof("Retrieving addresses used in project %s in region %s", c.projectID, region) for _, instance := range instances { - address, err := kipcompute.GetAddressUsedByInstance(c.projectID, instance.Name, instance.Zone, c.config) - if err != nil { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}). - Errorf("Could not retrieve address for project %s region %s instance %s. Aborting.", c.projectID, region, instance.Name) - return + address, errGet := kipcompute.GetAddressUsedByInstance(c.projectID, instance.Name, instance.Zone, c.config) + if errGet != nil { + return errors.Wrap(errGet, "failed to retrieve address") } usedAddresses = append(usedAddresses, AddressInstanceTuple{ address, @@ -455,7 +421,7 @@ func (c *Controller) processAllNodes(shouldCheckOptimalIPAssignment bool) { //no } // Perform subtraction - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Infof("Project %s in region %s should use the following IPs %s... Checking that the instances follow these assignments", c.projectID, region, topMostAddresses) + c.logger.Infof("project %s in region %s should use the following IPs %s... Checking that the instances follow these assignments", c.projectID, region, topMostAddresses) var toRemove []AddressInstanceTuple for _, usedAddress := range usedAddresses { if usedAddress.address != "0.0.0.0" && !utils.Contains(topMostAddresses, usedAddress.address) { @@ -463,17 +429,12 @@ func (c *Controller) processAllNodes(shouldCheckOptimalIPAssignment bool) { //no } } - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Infof("Found %d Addresses to remove project %s in region %s. Addresses %s", len(toRemove), c.projectID, region, toRemove) if len(toRemove) > 0 { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Infof("Found %d ips %s in region %s which are not part of the top most addresses %s", len(toRemove), toRemove, region, topMostAddresses) for _, remove := range toRemove { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Infof("Instance %s in project %s in region %s uses suboptimal IP %s... Removing so we reassign", remove.instance.Name, c.projectID, region, toRemove) // Delete the IP we will re-assign this - err := kipcompute.DeleteIP(c.projectID, remove.instance.Zone, remove.instance.Name, c.config) + err = kipcompute.DeleteIP(c.projectID, remove.instance.Zone, remove.instance.Name, c.config) if err != nil { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}). - Errorf("Could not delete IP %s used by instance %s which is suboptimal. Aborting.", remove.address, remove.instance.Name) - return + return errors.Wrap(err, "failed to delete IP") } } } @@ -482,31 +443,30 @@ func (c *Controller) processAllNodes(shouldCheckOptimalIPAssignment bool) { //no for _, inst := range nodesOfInterest { if !kipcompute.IsInstanceUsesReservedIP(c.projectID, inst.Name, inst.Zone, c.config) { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Infof("Found unassigned node %s in pool %s", inst.Name, inst.Pool) + logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "processAllNodes"}).Infof("Found unassigned node %s in pool %s", inst.Name, inst.Pool) c.instance <- inst } } + return nil } func (c *Controller) forceAssignmentOnce(shouldCheckOptimalIPAssignment bool) { if !c.processing { c.processing = true if c.config.ForceAssignment { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "forceAssignmentOnce"}).Info("Starting forceAssignmentOnce") - c.processAllNodes(shouldCheckOptimalIPAssignment) + err := c.processAllNodes(shouldCheckOptimalIPAssignment) + if err != nil { + c.logger.WithError(err).Error("failed to process all nodes") + } } c.assignMissingTags() c.processing = false - } else { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "forceAssignmentOnce"}).Info("Skipping forceAssignmentOnce ... already in progress") } } func (c *Controller) forceAssignment() { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "forceAssignment"}).Info("Processing initial force assignment check") c.forceAssignmentOnce(true) for range c.ticker.C { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "forceAssignment"}).Info("Tick received for force assignment check") c.forceAssignmentOnce(false) } } @@ -519,24 +479,15 @@ func (c *Controller) assignMissingTags() { for _, pool := range nodePools { label := fmt.Sprintf("!kubip_assigned,cloud.google.com/gke-nodepool=%s", pool) - nodelist, err := kubeClient.CoreV1().Nodes().List(context.Background(), meta_v1.ListOptions{ - LabelSelector: label, - }) + nodelist, err := kubeClient.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{LabelSelector: label}) if err != nil { - logrus.Error(err) + c.logger.Warn("failed to list nodes") continue } for _, node := range nodelist.Items { labels := node.GetLabels() if nodeZone, ok := labels["failure-domain.beta.kubernetes.io/zone"]; ok { - if err != nil { - logrus.Fatalf("Could not get authenticated client: %v", err) - continue - } - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "assignMissingTags"}).Infof("Found node without tag %s", node.GetName()) kipcompute.AddTagIfMissing(c.projectID, node.GetName(), nodeZone, c.config) - } else { - continue } } } diff --git a/pkg/kipcompute/compute.go b/internal/kipcompute/compute.go similarity index 88% rename from pkg/kipcompute/compute.go rename to internal/kipcompute/compute.go index 1b04aa6..c1d09d6 100644 --- a/pkg/kipcompute/compute.go +++ b/internal/kipcompute/compute.go @@ -29,9 +29,9 @@ import ( "time" "cloud.google.com/go/compute/metadata" - "github.com/doitintl/kubeip/pkg/config" - "github.com/doitintl/kubeip/pkg/types" - "github.com/doitintl/kubeip/pkg/utils" + "github.com/doitintl/kubeip/internal/config" + "github.com/doitintl/kubeip/internal/types" + "github.com/doitintl/kubeip/internal/utils" "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/net/context" @@ -65,7 +65,7 @@ func getPriorityOrder(address *compute.Address, cfg *config.Config) int { intVal, err := strconv.Atoi(strVal) if err != nil { logrus.WithFields(logrus.Fields{ - "pkg": "kubeip", + "internal": "kubeip", "function": "getPriorityOrder", }).WithError(err). Errorf("Address %s has errors. Failed to convert order by label value %s with value %s to integer", address.Name, cfg.OrderByLabelKey, strVal) @@ -149,7 +149,7 @@ func DeleteIP(projectID, zone, instance string, cfg *config.Config) error { if len(inst.NetworkInterfaces) > 0 && len(inst.NetworkInterfaces[0].AccessConfigs) > 0 { accessConfigName := inst.NetworkInterfaces[0].AccessConfigs[0].Name if cfg.DryRun { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "DeleteIP"}).Infof("Deleted Access Config for %s zone %s new ip %s", instance, zone, accessConfigName) + logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "DeleteIP"}).Infof("Deleted Access Config for %s zone %s new ip %s", instance, zone, accessConfigName) } else { op, err := computeService.Instances.DeleteAccessConfig(projectID, zone, instance, accessConfigName, "nic0").Do() if err != nil { @@ -161,7 +161,7 @@ func DeleteIP(projectID, zone, instance string, cfg *config.Config) error { } } } - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "DeleteIP"}).Infof("Deleted IP for %s zone %s", instance, zone) + logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "DeleteIP"}).Infof("Deleted IP for %s zone %s", instance, zone) // Delete an prior tags. utils.TagNode(instance, &types.IPAddress{IP: "0.0.0.0", Labels: map[string]string{}}, cfg) return nil @@ -173,7 +173,7 @@ func addIP(projectID, zone, instance string, addr *types.IPAddress, cfg *config. return errors.Wrap(err, "failed to get create compute service") } if cfg.DryRun { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "addIP"}).Infof("Added Access Config for %s zone %s new ip %s", instance, zone, addr.IP) + logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "addIP"}).Infof("Added Access Config for %s zone %s new ip %s", instance, zone, addr.IP) } else { accessConfig := &compute.AccessConfig{ Name: "External NAT", @@ -192,7 +192,7 @@ func addIP(projectID, zone, instance string, addr *types.IPAddress, cfg *config. } } - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "addIP"}).Infof("Added IP for %s zone %s new ip %s", instance, zone, addr.IP) + logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "addIP"}).Infof("Added IP for %s zone %s new ip %s", instance, zone, addr.IP) return nil } @@ -214,7 +214,7 @@ func replaceIP(projectID, zone, instance, pool string, cfg *config.Config) error return errors.Wrap(err, "failed to add IP") } - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "replaceIP"}).Infof("Replaced IP for %s zone %s new ip %s", instance, zone, addr.IP) + logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "replaceIP"}).Infof("Replaced IP for %s zone %s new ip %s", instance, zone, addr.IP) oldNode, err := utils.GetNodeByIP(addr.IP) if err == nil { utils.TagNode(oldNode, &types.IPAddress{IP: "0.0.0.0", Labels: map[string]string{}}, cfg) @@ -292,7 +292,7 @@ func IsInstanceUsesReservedIP(projectID, instance, zone string, cfg *config.Conf func Kubeip(instance <-chan types.Instance, cfg *config.Config) { for { inst := <-instance - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "Kubeip"}).Infof("Working on %s in zone %s", inst.Name, inst.Zone) + logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "Kubeip"}).Infof("Working on %s in zone %s", inst.Name, inst.Zone) _ = replaceIP(inst.ProjectID, inst.Zone, inst.Name, inst.Pool, cfg) } } @@ -311,7 +311,7 @@ func getAddressDetails(ip, region, projectID string) (*types.IPAddress, error) { if len(addresses.Items) != 1 { address := addresses.Items[0] - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "getAddressDetails"}).Infof("Node ip is reserved %s %s", ip, fmt.Sprint(address.Labels)) + logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "getAddressDetails"}).Infof("Node ip is reserved %s %s", ip, fmt.Sprint(address.Labels)) return &types.IPAddress{IP: address.Address, Labels: address.Labels}, nil } @@ -332,7 +332,7 @@ func isAddressReserved(ip, region, projectID string) bool { } if len(addresses.Items) != 0 { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "isAddressReserved"}).Infof("Node ip is reserved %s", ip) + logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "isAddressReserved"}).Infof("Node ip is reserved %s", ip) return true } return false @@ -360,7 +360,7 @@ func AddTagIfMissing(projectID, instance, zone string, cfg *config.Config) { if err != nil { return } - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "AddTagIfMissing"}).Infof("Tagging %s", instance) + logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "AddTagIfMissing"}).Infof("Tagging %s", instance) utils.TagNode(instance, addressDetails, cfg) } } diff --git a/internal/types/types.go b/internal/types/types.go new file mode 100644 index 0000000..b506a76 --- /dev/null +++ b/internal/types/types.go @@ -0,0 +1,16 @@ +package types + +// Instance GKE Instance VM +type Instance struct { + ProjectID string + Name string + Zone string + Pool string +} + +// IPAddress GKE IP +type IPAddress struct { + IP string + Name string + Labels map[string]string +} diff --git a/pkg/utils/k8sutil.go b/internal/utils/k8sutil.go similarity index 69% rename from pkg/utils/k8sutil.go rename to internal/utils/k8sutil.go index effebd8..6f73a57 100644 --- a/pkg/utils/k8sutil.go +++ b/internal/utils/k8sutil.go @@ -1,23 +1,3 @@ -// Copyright © 2021 DoiT International -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - package utils import ( @@ -25,8 +5,8 @@ import ( "fmt" "strings" - "github.com/doitintl/kubeip/pkg/config" - "github.com/doitintl/kubeip/pkg/types" + "github.com/doitintl/kubeip/internal/config" + "github.com/doitintl/kubeip/internal/types" "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/net/context" @@ -137,7 +117,7 @@ func createLabelKeyValuePairs(m map[string]string, cfg *config.Config) string { // TagNode tag GKE node with "kubip_assigned" label (with typo) and also copy the labels present on the address if the copyLabels flag is set to true func TagNode(node string, ip *types.IPAddress, cfg *config.Config) { kubeClient := GetClient() - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "tagNode"}).Infof("Tagging node %s as %s", node, ip.IP) + logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "tagNode"}).Infof("Tagging node %s as %s", node, ip.IP) // replace . with - in IP address dashIP := strings.Replace(ip.IP, ".", "-", 4) //nolint:gomnd var labelString string @@ -147,9 +127,9 @@ func TagNode(node string, ip *types.IPAddress, cfg *config.Config) { if cfg.ClearLabels { result, err := kubeClient.CoreV1().Nodes().Get(context.Background(), node, metav1.GetOptions{}) if err != nil { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "tagNode"}).Error(err) + logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "tagNode"}).Error(err) } else { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "tagNode"}).Infof("Clear label tag for node %s with ip %s and clear tags %s", node, ip.IP, result.Labels) + logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "tagNode"}).Infof("Clear label tag for node %s with ip %s and clear tags %s", node, ip.IP, result.Labels) createLabelKeyValuePairs(result.Labels, cfg) labelsToClear = clearLabels(result.Labels, cfg) } @@ -164,12 +144,12 @@ func TagNode(node string, ip *types.IPAddress, cfg *config.Config) { patch := fmt.Sprintf(`{"metadata":{"labels":%v}}`, labelString) if cfg.DryRun { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "tagNode"}).Infof("Tagging node %s as %s with tags %s ", node, ip.IP, labelString) + logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "tagNode"}).Infof("Tagging node %s as %s with tags %s ", node, ip.IP, labelString) } else { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "tagNode"}).Infof("Tagging node %s as %s with tags %s ", node, ip.IP, labelString) + logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "tagNode"}).Infof("Tagging node %s as %s with tags %s ", node, ip.IP, labelString) _, err := kubeClient.CoreV1().Nodes().Patch(context.Background(), node, typesv1.MergePatchType, []byte(patch), metav1.PatchOptions{}) if err != nil { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "tagNode"}).Infof("Error occurred while tagging node %s as %s with tags %s ", node, ip.IP, labelString) + logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "tagNode"}).Infof("Error occurred while tagging node %s as %s with tags %s ", node, ip.IP, labelString) logrus.Error(err) } } diff --git a/pkg/types/types.go b/pkg/types/types.go deleted file mode 100644 index 54068de..0000000 --- a/pkg/types/types.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright © 2021 DoiT International -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package types - -// Instance GKE Instance VM -type Instance struct { - ProjectID string - Name string - Zone string - Pool string -} - -// IPAddress GKE IP -type IPAddress struct { - IP string - Name string - Labels map[string]string -} From 5cf69686843a856e27146f7e4213af2fa0da15b0 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Wed, 20 Sep 2023 22:50:01 +0300 Subject: [PATCH 05/66] get rid of global logrus --- internal/controller/controller.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/controller/controller.go b/internal/controller/controller.go index 3b177d0..9e3d706 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -443,7 +443,10 @@ func (c *Controller) processAllNodes(shouldCheckOptimalIPAssignment bool) error for _, inst := range nodesOfInterest { if !kipcompute.IsInstanceUsesReservedIP(c.projectID, inst.Name, inst.Zone, c.config) { - logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "processAllNodes"}).Infof("Found unassigned node %s in pool %s", inst.Name, inst.Pool) + c.logger.WithFields(logrus.Fields{ + "instance": inst.Name, + "pool": inst.Pool, + }).Debugf("found unassigned node in pool") c.instance <- inst } } From 98ac15cf13408b3f1c7c376d90082c635ecc339f Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Wed, 20 Sep 2023 23:01:13 +0300 Subject: [PATCH 06/66] get rid of global logrus --- internal/controller/controller.go | 9 ++------- internal/kipcompute/compute.go | 28 ++-------------------------- 2 files changed, 4 insertions(+), 33 deletions(-) diff --git a/internal/controller/controller.go b/internal/controller/controller.go index 9e3d706..d1d2496 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -203,18 +203,14 @@ func (c *Controller) Run(stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer c.queue.ShutDown() - c.logger.Info("Starting kubeip controller") serverStartTime = time.Now().Local() - go c.informer.Run(stopCh) - if !cache.WaitForCacheSync(stopCh, c.HasSynced) { utilruntime.HandleError(errTimeout) return } c.logger.Info("kubeip controller synced and ready") - wait.Until(c.runWorker, time.Second, stopCh) } @@ -420,15 +416,13 @@ func (c *Controller) processAllNodes(shouldCheckOptimalIPAssignment bool) error }) } - // Perform subtraction - c.logger.Infof("project %s in region %s should use the following IPs %s... Checking that the instances follow these assignments", c.projectID, region, topMostAddresses) + // Remove all addresses that are not in the top most addresses. var toRemove []AddressInstanceTuple for _, usedAddress := range usedAddresses { if usedAddress.address != "0.0.0.0" && !utils.Contains(topMostAddresses, usedAddress.address) { toRemove = append(toRemove, usedAddress) } } - if len(toRemove) > 0 { for _, remove := range toRemove { // Delete the IP we will re-assign this @@ -441,6 +435,7 @@ func (c *Controller) processAllNodes(shouldCheckOptimalIPAssignment bool) error } } + // Find all nodes that do not have an IP assigned and send them to the channel for assignment. for _, inst := range nodesOfInterest { if !kipcompute.IsInstanceUsesReservedIP(c.projectID, inst.Name, inst.Zone, c.config) { c.logger.WithFields(logrus.Fields{ diff --git a/internal/kipcompute/compute.go b/internal/kipcompute/compute.go index c1d09d6..7787c58 100644 --- a/internal/kipcompute/compute.go +++ b/internal/kipcompute/compute.go @@ -21,7 +21,6 @@ package kipcompute import ( - "fmt" "math" "sort" "strconv" @@ -33,7 +32,6 @@ import ( "github.com/doitintl/kubeip/internal/types" "github.com/doitintl/kubeip/internal/utils" "github.com/pkg/errors" - "github.com/sirupsen/logrus" "golang.org/x/net/context" "google.golang.org/api/compute/v0.beta" //nolint:goimports ) @@ -64,11 +62,6 @@ func getPriorityOrder(address *compute.Address, cfg *config.Config) int { if ok { intVal, err := strconv.Atoi(strVal) if err != nil { - logrus.WithFields(logrus.Fields{ - "internal": "kubeip", - "function": "getPriorityOrder", - }).WithError(err). - Errorf("Address %s has errors. Failed to convert order by label value %s with value %s to integer", address.Name, cfg.OrderByLabelKey, strVal) return defaultValue } return intVal @@ -148,9 +141,7 @@ func DeleteIP(projectID, zone, instance string, cfg *config.Config) error { } if len(inst.NetworkInterfaces) > 0 && len(inst.NetworkInterfaces[0].AccessConfigs) > 0 { accessConfigName := inst.NetworkInterfaces[0].AccessConfigs[0].Name - if cfg.DryRun { - logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "DeleteIP"}).Infof("Deleted Access Config for %s zone %s new ip %s", instance, zone, accessConfigName) - } else { + if !cfg.DryRun { op, err := computeService.Instances.DeleteAccessConfig(projectID, zone, instance, accessConfigName, "nic0").Do() if err != nil { return errors.Wrap(err, "failed to delete access config") @@ -161,7 +152,6 @@ func DeleteIP(projectID, zone, instance string, cfg *config.Config) error { } } } - logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "DeleteIP"}).Infof("Deleted IP for %s zone %s", instance, zone) // Delete an prior tags. utils.TagNode(instance, &types.IPAddress{IP: "0.0.0.0", Labels: map[string]string{}}, cfg) return nil @@ -172,9 +162,7 @@ func addIP(projectID, zone, instance string, addr *types.IPAddress, cfg *config. if err != nil { return errors.Wrap(err, "failed to get create compute service") } - if cfg.DryRun { - logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "addIP"}).Infof("Added Access Config for %s zone %s new ip %s", instance, zone, addr.IP) - } else { + if !cfg.DryRun { accessConfig := &compute.AccessConfig{ Name: "External NAT", Type: "ONE_TO_ONE_NAT", @@ -191,8 +179,6 @@ func addIP(projectID, zone, instance string, addr *types.IPAddress, cfg *config. return errors.Wrap(err, "failed to wait for compilation") } } - - logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "addIP"}).Infof("Added IP for %s zone %s new ip %s", instance, zone, addr.IP) return nil } @@ -214,7 +200,6 @@ func replaceIP(projectID, zone, instance, pool string, cfg *config.Config) error return errors.Wrap(err, "failed to add IP") } - logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "replaceIP"}).Infof("Replaced IP for %s zone %s new ip %s", instance, zone, addr.IP) oldNode, err := utils.GetNodeByIP(addr.IP) if err == nil { utils.TagNode(oldNode, &types.IPAddress{IP: "0.0.0.0", Labels: map[string]string{}}, cfg) @@ -269,14 +254,12 @@ func GetAddressUsedByInstance(projectID, instance, zone string, cfg *config.Conf func IsInstanceUsesReservedIP(projectID, instance, zone string, cfg *config.Config) bool { computeService, err := compute.NewService(context.Background()) if err != nil { - logrus.WithError(err).Error("failed to get create compute service") return false } region := zone[:len(zone)-2] filter := "(labels." + cfg.LabelKey + "=" + cfg.LabelValue + ")" addresses, err := computeService.Addresses.List(projectID, region).Filter("(status=IN_USE) AND " + filter).Do() if err != nil { - logrus.Error(err) return false } @@ -292,7 +275,6 @@ func IsInstanceUsesReservedIP(projectID, instance, zone string, cfg *config.Conf func Kubeip(instance <-chan types.Instance, cfg *config.Config) { for { inst := <-instance - logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "Kubeip"}).Infof("Working on %s in zone %s", inst.Name, inst.Zone) _ = replaceIP(inst.ProjectID, inst.Zone, inst.Name, inst.Pool, cfg) } } @@ -311,7 +293,6 @@ func getAddressDetails(ip, region, projectID string) (*types.IPAddress, error) { if len(addresses.Items) != 1 { address := addresses.Items[0] - logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "getAddressDetails"}).Infof("Node ip is reserved %s %s", ip, fmt.Sprint(address.Labels)) return &types.IPAddress{IP: address.Address, Labels: address.Labels}, nil } @@ -321,18 +302,15 @@ func getAddressDetails(ip, region, projectID string) (*types.IPAddress, error) { func isAddressReserved(ip, region, projectID string) bool { computeService, err := compute.NewService(context.Background()) if err != nil { - logrus.WithError(err).Error("failed to get create compute service") return false } filter := "address=" + "\"" + ip + "\"" addresses, err := computeService.Addresses.List(projectID, region).Filter(filter).Do() if err != nil { - logrus.Error(err) return false } if len(addresses.Items) != 0 { - logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "isAddressReserved"}).Infof("Node ip is reserved %s", ip) return true } return false @@ -342,7 +320,6 @@ func isAddressReserved(ip, region, projectID string) bool { func AddTagIfMissing(projectID, instance, zone string, cfg *config.Config) { computeService, err := compute.NewService(context.Background()) if err != nil { - logrus.WithError(err).Error("failed to get create compute service") return } inst, err := computeService.Instances.Get(projectID, zone, instance).Do() @@ -360,7 +337,6 @@ func AddTagIfMissing(projectID, instance, zone string, cfg *config.Config) { if err != nil { return } - logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "AddTagIfMissing"}).Infof("Tagging %s", instance) utils.TagNode(instance, addressDetails, cfg) } } From c0fcbfb78b035cb57f9407ecc6ab4eccde132c7b Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Thu, 21 Sep 2023 18:26:17 +0300 Subject: [PATCH 07/66] v2 draft --- .github/ISSUE_TEMPLATE/bug_report.md | 35 -- .github/ISSUE_TEMPLATE/feature_request.md | 17 - README.md | 341 +------------ cmd/main.go | 96 +++- deploy/kubeip-configmap.yaml | 20 - deploy/kubeip-deployment.yaml | 137 ----- go.mod | 68 +-- go.sum | 585 +--------------------- internal/config/config.go | 82 +-- internal/controller/controller.go | 492 ------------------ internal/kipcompute/compute.go | 342 ------------- internal/types/types.go | 16 - internal/utils/k8sutil.go | 173 ------- 13 files changed, 104 insertions(+), 2300 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/bug_report.md delete mode 100644 .github/ISSUE_TEMPLATE/feature_request.md delete mode 100644 deploy/kubeip-configmap.yaml delete mode 100644 deploy/kubeip-deployment.yaml delete mode 100644 internal/controller/controller.go delete mode 100644 internal/kipcompute/compute.go delete mode 100644 internal/types/types.go delete mode 100644 internal/utils/k8sutil.go diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index b735373..0000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve - ---- - -**Describe the bug** -A clear and concise description of what the bug is. - -**To Reproduce** -Steps to reproduce the behavior: -1. Go to '...' -2. Click on '....' -3. Scroll down to '....' -4. See error - -**Expected behavior** -A clear and concise description of what you expected to happen. - -**Screenshots** -If applicable, add screenshots to help explain your problem. - -**Desktop (please complete the following information):** - - OS: [e.g. iOS] - - Browser [e.g. chrome, safari] - - Version [e.g. 22] - -**Smartphone (please complete the following information):** - - Device: [e.g. iPhone6] - - OS: [e.g. iOS8.1] - - Browser [e.g. stock browser, safari] - - Version [e.g. 22] - -**Additional context** -Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index 066b2d9..0000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -name: Feature request -about: Suggest an idea for this project - ---- - -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] - -**Describe the solution you'd like** -A clear and concise description of what you want to happen. - -**Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. - -**Additional context** -Add any other context or screenshots about the feature request here. diff --git a/README.md b/README.md index 70a1f56..7bcec55 100644 --- a/README.md +++ b/README.md @@ -2,342 +2,7 @@ # What is kubeIP? -Many applications need to be whitelisted by users based on a Source IP Address. As of today, Google Kubernetes Engine doesn't support assigning a static pool of IP addresses to the GKE cluster. Using kubeIP, this problem is solved by assigning GKE nodes external IP addresses from a predefined list. kubeIP monitors the Kubernetes API for new/removed nodes and applies the changes accordingly. +Many applications need to be whitelisted by users based on a Source IP Address. As of today, cloud-manages Kubernetes Engines do not support +assigning a static pool of IP addresses to the Kubernetes cluster. Using kubeIP, this problem is solved by assigning Kubernetes nodes +external IP addresses from a predefined list. -# Deploy kubeIP (without building from source) - -If you just want to use kubeIP (instead of building it yourself from source), please follow the instructions in this section. You’ll need Kubernetes version 1.10 or newer. You'll also need the Google Cloud SDK. You can install the [Google Cloud SDK](https://cloud.google.com/sdk) (which also installs kubectl). - -To configure your Google Cloud SDK, set default project as: - -``` -gcloud config set project {your project_id} -``` - -Set the environment variables and make sure to configure before continuing: - -``` -export GCP_REGION= -export GCP_ZONE= -export GKE_CLUSTER_NAME= -export PROJECT_ID=$(gcloud config list --format 'value(core.project)') -export KUBEIP_NODEPOOL= -export KUBEIP_SELF_NODEPOOL= -``` - -**Get credentials for yourself** - -If you haven't alrady, set up your GKE cluster credentials with -(replace `$GKE_CLUSTER_NAME` with your real GKE cluster name): - -``` -gcloud container clusters get-credentials $GKE_CLUSTER_NAME \ - --region $GCP_ZONE \ - --project $PROJECT_ID -``` - -You will need admin access to the cluster to deploy to the `kube-system` namespace. -Either set the relevant IAM rolebinding for your user, or get RBAC permissions with: - -``` -kubectl create clusterrolebinding cluster-admin-binding \ - --clusterrole cluster-admin --user `gcloud config list --format 'value(core.account)'` -``` - -**Creating an IAM Service Account** - -Create a Service Account with this command: - -``` -gcloud iam service-accounts create kubeip-service-account --display-name "kubeIP" -``` - -Create and attach a custom kubeIP role to the service account by running the following commands: - -``` -gcloud iam roles create kubeip --project $PROJECT_ID --file roles.yaml - -gcloud projects add-iam-policy-binding $PROJECT_ID \ - --member=serviceAccount:kubeip-service-account@$PROJECT_ID.iam.gserviceaccount.com \ - --role=projects/$PROJECT_ID/roles/kubeip \ - --condition=None -``` - -**Getting credentials to the deployment** - -Note: If you use Workload Identity in your cluster, you do not need to upload a credential file. -You can just remove the GOOGLE_APPLICATION_CREDENTIALS environment variable from the manifest -and the Google Cloud SDK will pick up the credentials from the metadata server as per normal -operation. - -Generate the Key using the following command: -``` -gcloud iam service-accounts keys create key.json \ - --iam-account kubeip-service-account@$PROJECT_ID.iam.gserviceaccount.com -``` - -Create a Kubernetes secret object by running: -``` -kubectl create secret generic kubeip-key --from-file=key.json -n kube-system -``` - -**Create Static, Reserved IP Addresses:** - -Create as many static IP addresses for the number of nodes in your GKE cluster (this example creates 10 addresses) so you will have enough addresses when your cluster scales up (manually or automatically): - -``` -for i in {1..10}; do gcloud compute addresses create kubeip-ip$i --project=$PROJECT_ID --region=$GCP_REGION; done -``` - -Add labels to reserved IP addresses. A common practice is to assign a unique value per cluster (for example cluster name): - -``` -for i in {1..10}; do gcloud beta compute addresses update kubeip-ip$i --update-labels kubeip=$GKE_CLUSTER_NAME --region $GCP_REGION; done -``` - -``` -sed -i -e "s/reserved/$GKE_CLUSTER_NAME/g" -e "s/default-pool/$KUBEIP_NODEPOOL/g" deploy/kubeip-configmap.yaml -``` - -Make sure the `deploy/kubeip-configmap.yaml` file contains the correct values: - - - The `KUBEIP_LABELVALUE` should be your GKE's cluster name - - The `KUBEIP_NODEPOOL` should match the name of your GKE node-pool on which kubeIP will operate - - The `KUBEIP_FORCEASSIGNMENT` - controls whether kubeIP should assign static IPs to existing nodes in the node-pool and defaults to true - -We recommend that KUBEIP_NODEPOOL should *NOT* be the same as KUBEIP_SELF_NODEPOOL - - -If you would like to assign addresses to other node pools, then `KUBEIP_NODEPOOL` can be added to this nodepool `KUBEIP_ADDITIONALNODEPOOLS` as a comma separated list. -You should tag the addresses for this pool with the `KUBEIP_LABELKEY` value + `-node-pool` and assign the value of the node pool a name i.e., `kubeip-node-pool=my-node-pool` - -``` -sed -i -e "s/pool-kubip/$KUBEIP_SELF_NODEPOOL/g" deploy/kubeip-deployment.yaml -``` - -Deploy kubeIP by running: - -``` -kubectl apply -f deploy/. -``` - -Once you’ve assigned an IP address to a node kubeIP, a label will be created for that node `kubip_assigned` with the value of the IP address (`.` are replaced with `-`): - - `172.31.255.255 ==> 172-31-255-255` - -**Ordering IPs** - -KubeIP can order IPs based on the numeric value identified by `KUBEIP_ORDERBYLABELKEY`. - -IPs are ordered in descending order if `KUBEIP_ORDERBYDESC` is set to true, ascending order otherwise. - -Missing `KUBEIP_ORDERBYLABELKEY` or invalid values present on `KUBEIP_ORDERBYLABELKEY` will be assigned the lowest priority. - -When nodes are added, deleted or on tick, kubeIP will check whether the nodes have the most optimal IP assignment. What does this mean? - -E.g. Let's assume Node1 has IP_A, Node2 has IP_B and IP_A > IP_B, when we scale the cluster down the cluster two things might happen -1. Node 1 is deleted which results in a sub-optimal IP assignment since Node2 has IP_B and IP_A > IP_B -2. Node 2 is deleted maintaining optimal order. - -In the first case Node 2 is re-assigned IP_A. - -To order the IPs reserved above in asc order use - -``` -for i in {1..10}; do gcloud beta compute addresses update kubeip-ip$i --update-labels priority=$i --region=$GCP_REGION; done -``` - -and set - -``` -KUBEIP_ORDERBYLABELKEY: "priority" -KUBEIP_ORDERBYDESC: "false" -``` - -**Copy Labels** - -KubeIP will also copy all labels from the IP being assigned over to the node if `KUBEIP_COPYLABELS` is set to true. - -This is typically helpful when we want to have node selection not based on IP but more semantic label keys and values. - -As an example let's label `kubeip-ip1` with `platform_whitelisted=true`, to do this we execute the following command - -``` -gcloud beta compute addresses update kubeip-ip1 --update-labels "platform_whitelisted=true" --region=$GCP_REGION; -``` - -Now, when a node is assigned the IP address of `kubeip-ip1` it will also be labelled with `platform_whitelisted=true` as well as the default `kubip_assigned`. - -An IP can have multiple labels, all will be copied over. - -**Clear Labels** - -When IPs get assigned or re-assigned to achieve optimal IP assignment we can configure the system to clear any previous labels. Set `KUBEIP_CLEARLABELS` flag to `true` if you want this behaviour. - -This feature is required when labels are not overlapping. E.g. let's assume we have the following tagged IPs; IP_A and IP_B, order by priority - -``` -IP_A test_a=value_a,test_b=value_b,priority=1 -IP_B test_c=value_c,priority=2 -``` -Let's assume that the assignment was as follows - -``` -IP_A => NodeA -IP_B => NodeB -``` - -At this point `NodeA` has labels `test_a=value_a,test_b=value_b` and `NodeB` has labels `test_c=value_c`. Note priority is not copied over. - -If `NodeA` is deleted a re-assignment needs to happen (due to the fact that IP_A > IP_B) and `NodeB` would have -- `test_a=value_a,test_b=value_b,test_c=value_c` if `KUBEIP_CLEARLABELS="false"` and -- `test_a=value_a,test_b=value_b` if `KUBEIP_CLEARLABELS="true"` - -Note that `test_c` is not an overlapping label and hence might cause problems if `KUBEIP_CLEARLABELS` is not set to `true`. - -**Dry Run Mode** - -Dry run mode allows debugging the operations performed by KubeIP without actually performing the operations. - -ONLY use this mode during development of new features on KubeIP. - - -# Deploy & Build From Source - -You need Kubernetes version 1.10 or newer. You also need Docker version and kubectl 1.10.x or newer installed on your machine, as well as the Google Cloud SDK. You can install the [Google Cloud SDK](https://cloud.google.com/sdk) (which also installs kubectl). - - -**Clone Git Repository** - -Make sure your $GOPATH is [configured](https://github.com/golang/go/wiki/SettingGOPATH). You'll need to clone this repository to your `$GOPATH/src` folder. - -``` -mkdir -p $GOPATH/src/doitintl/kubeip -git clone https://github.com/doitintl/kubeip.git $GOPATH/src/doitintl/kubeip -cd $GOPATH/src/doitintl/kubeip -``` - -**Set Environment Variables** - -Replace **us-central1** with the region where your GKE cluster resides and **kubeip-cluster** with your real GKE cluster name - -``` -export GCP_REGION=us-central1 -export GCP_ZONE=us-central1-b -export GKE_CLUSTER_NAME=kubeip-cluster -export PROJECT_ID=$(gcloud config list --format 'value(core.project)') -``` - -**Develop kubeIP Locally** - -Compile the kubeIP binary and run tests - -``` -make -``` - -**Build kubeIP's Container Image** - - -Compile the kubeIP binary and build the Docker image as following: - -``` -make image -``` - -Tag the image using: - -``` -docker tag kubeip gcr.io/$PROJECT_ID/kubeip -``` - -Finally, push the image to Google Container Registry with: - -``` -docker push gcr.io/$PROJECT_ID/kubeip -``` - -Alternatively, you can export `REGISTRY` to `gcr.io/$PROJECT_ID` and run the script `build-all-and-push.sh` which builds and publishes the docker image. - -**Create IAM Service Account and obtain the Key in JSON format** - -Create a Service Account with this command: - -``` -gcloud iam service-accounts create kubeip-service-account --display-name "kubeIP" -``` - -Create and attach the custom kubeIP role to the service account by running the following commands: - -``` -gcloud iam roles create kubeip --project $PROJECT_ID --file roles.yaml - -gcloud projects add-iam-policy-binding $PROJECT_ID --member serviceAccount:kubeip-service-account@$PROJECT_ID.iam.gserviceaccount.com --role projects/$PROJECT_ID/roles/kubeip -``` - -Generate the Key using the following command: - -``` -gcloud iam service-accounts keys create key.json \ - --iam-account kubeip-service-account@$PROJECT_ID.iam.gserviceaccount.com -``` - -**Create Kubernetes Secret** - -Get your GKE cluster credentaials with (replace *cluster_name* with your real GKE cluster name): - -``` -gcloud container clusters get-credentials $GKE_CLUSTER_NAME \ - --region $GCP_ZONE \ - --project $PROJECT_ID -``` - -Create a Kubernetes secret by running: - -``` -kubectl create secret generic kubeip-key --from-file=key.json -n kube-system -``` - -**We need to get RBAC permissions first with** -``` -kubectl create clusterrolebinding cluster-admin-binding \ - --clusterrole cluster-admin --user `gcloud config list --format 'value(core.account)'` -``` - -**Create static reserved IP addresses:** - -Create as many static IP addresses for the number of nodes in your GKE cluster (this example creates 10 addresses) so you will have enough addresses when your cluster scales up (automatically or manually): - -``` -for i in {1..10}; do gcloud compute addresses create kubeip-ip$i --project=$PROJECT_ID --region=$GCP_REGION; done -``` - -Add labels to reserved IP addresses. A common practice is to assign a unique value per cluster. You can use your cluster name for example: - -``` -for i in {1..10}; do gcloud beta compute addresses update kubeip-ip$i --update-labels kubeip=$GKE_CLUSTER_NAME --region $GCP_REGION; done -``` - -Adjust the deploy/kubeip-configmap.yaml with your GKE cluster name (replace the GKE-cluster-name with your real GKE cluster name): - -``` -sed -i -e "s/reserved/$GKE_CLUSTER_NAME/g" deploy/kubeip-configmap.yaml -``` - -Adjust the `deploy/kubeip-deployment.yaml` to reflect your real container image path: - - - Edit the `image` to match your container image path, i.e. `gcr.io/$PROJECT_ID/kubeip` - -By default, kubeIP will only manage the nodes in default-pool nodepool. If you'd like kubeIP to manage another node-pool, please update the `KUBEIP_NODEPOOL` setting in `deploy/kubeip-configmap.yaml` file before deploying. You can also update the `KUBEIP_LABELKEY` and `KUBEIP_LABELVALUE` to control which static external IP addresses the kubeIP will look for to assign to your nodes. - -The `KUBEIP_FORCEASSIGNMENT` (which defaults to true) will check on startup and every five minutes if there are nodes in the node-pool that are not assigned to a reserved address. If such nodes are found, then kubeIP will assign a reserved address (if one is available to them): - -Deploy kubeIP by running - -``` -kubectl apply -f deploy/. -``` - -References: - - - Event listening code was take from [kubewatch](https://github.com/bitnami-labs/kubewatch/) diff --git a/cmd/main.go b/cmd/main.go index d1f56ff..1db7a46 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,10 +1,14 @@ package main import ( + "context" + "fmt" + "os" + "runtime" + "github.com/doitintl/kubeip/internal/config" - "github.com/doitintl/kubeip/internal/controller" - "github.com/doitintl/kubeip/internal/kipcompute" "github.com/sirupsen/logrus" + "github.com/urfave/cli/v2" "sigs.k8s.io/controller-runtime/pkg/manager/signals" ) @@ -50,32 +54,78 @@ func prepareLogger(level string, json bool) *logrus.Entry { return log } -func main() { +func run(_ context.Context, _ *logrus.Entry, _ config.Config) error { + return nil +} + +func runCmd(c *cli.Context) error { ctx := signals.SetupSignalHandler() - cfg := config.NewConfig() - logger := prepareLogger(cfg.LogLevel, cfg.LogJSON) - logger.WithField("config", cfg).Info("using kubeIP configuration") + log := prepareLogger(c.String("log-level"), c.Bool("json")) + cfg := config.LoadConfig(c) - cluster, err := kipcompute.ClusterName() - if err != nil { - logger.WithError(err).Fatal("failed to get cluster name") + if err := run(ctx, log, cfg); err != nil { + log.Fatalf("eks-lens agent failed: %v", err) } - project, err := kipcompute.ProjectName() - if err != nil { - logger.WithError(err).Fatal("failed to get project name") - } + return nil +} - logger = logger.WithFields(logrus.Fields{ - "cluster": cluster, - "project": project, - "branch": gitBranch, - "commit": gitCommit, - "buildDate": buildDate, - }) - logger.Info("starting kubeIP controller") +func main() { + app := &cli.App{ + Commands: []*cli.Command{ + { + Name: "run", + Usage: "run agent", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cluster-name", + Usage: "Kubernetes cluster name (not needed if running in cluster)", + EnvVars: []string{"CLUSTER_NAME"}, + Category: "Configuration", + }, + &cli.PathFlag{ + Name: "kubeconfig", + Usage: "Path to Kubernetes configuration file (not needed if running in cluster)", + EnvVars: []string{"KUBECONFIG"}, + Category: "Configuration", + }, + &cli.StringFlag{ + Name: "log-level", + Usage: "set log level (debug, info(*), warning, error, fatal, panic)", + Value: "info", + EnvVars: []string{"LOG_LEVEL"}, + Category: "Logging", + }, + &cli.BoolFlag{ + Name: "json", + Usage: "produce log in JSON format: Logstash and Splunk friendly", + EnvVars: []string{"LOG_JSON"}, + Category: "Logging", + }, + &cli.BoolFlag{ + Name: "develop-mode", + Usage: "enable develop mode", + EnvVars: []string{"DEV_MODE"}, + Category: "Development", + }, + }, + Action: runCmd, + }, + }, + Name: "kubeip-agent", + Usage: "replaces the node's public IP address with a static public IP address", + Version: version, + } + cli.VersionPrinter = func(c *cli.Context) { + fmt.Printf("kubeip-agent %s\n", version) + fmt.Printf(" Build date: %s\n", buildDate) + fmt.Printf(" Git commit: %s\n", gitCommit) + fmt.Printf(" Git branch: %s\n", gitBranch) + fmt.Printf(" Built with: %s\n", runtime.Version()) + } - if err = controller.Start(ctx, logger, project, cluster, cfg); err != nil { - logrus.WithError(err).Fatal("failed to start kubeIP controller") + err := app.Run(os.Args) + if err != nil { + logrus.Fatal(err) } } diff --git a/deploy/kubeip-configmap.yaml b/deploy/kubeip-configmap.yaml deleted file mode 100644 index fba262f..0000000 --- a/deploy/kubeip-configmap.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: v1 -data: - KUBEIP_LABELKEY: "kubeip" - KUBEIP_LABELVALUE: "reserved" - KUBEIP_NODEPOOL: "default-pool" - KUBEIP_FORCEASSIGNMENT: "true" - KUBEIP_ADDITIONALNODEPOOLS: "" - KUBEIP_TICKER: "5" - KUBEIP_ALLNODEPOOLS: "false" - KUBEIP_ORDERBYLABELKEY: "priority" - KUBEIP_ORDERBYDESC: "true" - KUBEIP_COPYLABELS: "false" - KUBEIP_CLEARLABELS: "false" - KUBEIP_DRYRUN: "false" -kind: ConfigMap -metadata: - labels: - app: kubeip - name: kubeip-config - namespace: kube-system diff --git a/deploy/kubeip-deployment.yaml b/deploy/kubeip-deployment.yaml deleted file mode 100644 index 8985c55..0000000 --- a/deploy/kubeip-deployment.yaml +++ /dev/null @@ -1,137 +0,0 @@ -# We need to get RBAC permissions first with -# kubectl create clusterrolebinding cluster-admin-binding \ -# --clusterrole cluster-admin --user `gcloud config list --format 'value(core.account)'` - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: kubeip - namespace: kube-system -spec: - replicas: 1 - selector: - matchLabels: - app: kubeip - template: - metadata: - labels: - app: kubeip - spec: - priorityClassName: system-cluster-critical - nodeSelector: - cloud.google.com/gke-nodepool: pool-kubip - containers: - - name: "kubeip" - image: doitintl/kubeip:latest - imagePullPolicy: Always - volumeMounts: - - name: google-cloud-key - mountPath: /var/secrets/google - env: - - name: "KUBEIP_LABELKEY" - valueFrom: - configMapKeyRef: - key: "KUBEIP_LABELKEY" - name: "kubeip-config" - - name: "KUBEIP_LABELVALUE" - valueFrom: - configMapKeyRef: - key: "KUBEIP_LABELVALUE" - name: "kubeip-config" - - name: "KUBEIP_NODEPOOL" - valueFrom: - configMapKeyRef: - key: "KUBEIP_NODEPOOL" - name: "kubeip-config" - - name: "KUBEIP_FORCEASSIGNMENT" - valueFrom: - configMapKeyRef: - key: "KUBEIP_FORCEASSIGNMENT" - name: "kubeip-config" - - name: "KUBEIP_ORDERBYLABELKEY" - valueFrom: - configMapKeyRef: - key: "KUBEIP_ORDERBYLABELKEY" - name: "kubeip-config" - - name: "KUBEIP_ORDERBYDESC" - valueFrom: - configMapKeyRef: - key: "KUBEIP_ORDERBYDESC" - name: "kubeip-config" - - name: "KUBEIP_ADDITIONALNODEPOOLS" - valueFrom: - configMapKeyRef: - key: "KUBEIP_ADDITIONALNODEPOOLS" - name: "kubeip-config" - - name: "KUBEIP_COPYLABELS" - valueFrom: - configMapKeyRef: - key: "KUBEIP_COPYLABELS" - name: "kubeip-config" - - name: "KUBEIP_CLEARLABELS" - valueFrom: - configMapKeyRef: - key: "KUBEIP_CLEARLABELS" - name: "kubeip-config" - - name: "KUBEIP_DRYRUN" - valueFrom: - configMapKeyRef: - key: "KUBEIP_DRYRUN" - name: "kubeip-config" - - name: "KUBEIP_TICKER" - valueFrom: - configMapKeyRef: - key: "KUBEIP_TICKER" - name: "kubeip-config" - - name: "KUBEIP_ALLNODEPOOLS" - valueFrom: - configMapKeyRef: - key: "KUBEIP_ALLNODEPOOLS" - name: "kubeip-config" - - - - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /var/secrets/google/key.json - restartPolicy: Always - serviceAccountName: kubeip-sa - volumes: - - name: google-cloud-key - secret: - secretName: kubeip-key - ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kubeip-sa - namespace: kube-system - ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: kubeip-sa - namespace: kube-system -rules: -- apiGroups: [""] - resources: ["nodes"] - verbs: ["get","list","watch","patch"] -- apiGroups: [""] - resources: ["pods"] - verbs: ["get","list","watch"] - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: kubeip-sa -subjects: - - kind: ServiceAccount - name: kubeip-sa - namespace: kube-system -roleRef: - kind: ClusterRole - name: kubeip-sa - apiGroup: rbac.authorization.k8s.io diff --git a/go.mod b/go.mod index da86141..fec6dac 100644 --- a/go.mod +++ b/go.mod @@ -3,72 +3,16 @@ module github.com/doitintl/kubeip go 1.21 require ( - cloud.google.com/go/compute/metadata v0.2.3 - github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.3 - github.com/spf13/viper v1.16.0 - golang.org/x/net v0.15.0 - golang.org/x/time v0.3.0 - google.golang.org/api v0.142.0 - k8s.io/api v0.28.2 - k8s.io/apimachinery v0.28.2 - k8s.io/client-go v0.28.2 + github.com/urfave/cli/v2 v2.25.7 sigs.k8s.io/controller-runtime v0.16.2 ) require ( - cloud.google.com/go/compute v1.23.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.9.0 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-logr/logr v1.2.4 // indirect - github.com/go-openapi/jsonpointer v0.19.6 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.22.3 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.5.9 // indirect - github.com/google/gofuzz v1.2.0 // indirect - github.com/google/s2a-go v0.1.7 // indirect - github.com/google/uuid v1.3.1 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect - github.com/googleapis/gax-go/v2 v2.12.0 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect - github.com/imdario/mergo v0.3.6 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/magiconair/properties v1.8.7 // indirect - github.com/mailru/easyjson v0.7.7 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pelletier/go-toml/v2 v2.0.8 // indirect - github.com/spf13/afero v1.9.5 // indirect - github.com/spf13/cast v1.5.1 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/subosito/gotenv v1.4.2 // indirect - go.opencensus.io v0.24.0 // indirect - golang.org/x/crypto v0.13.0 // indirect - golang.org/x/oauth2 v0.12.0 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/stretchr/testify v1.8.3 // indirect + github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect + golang.org/x/net v0.15.0 // indirect golang.org/x/sys v0.12.0 // indirect - golang.org/x/term v0.12.0 // indirect - golang.org/x/text v0.13.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230913181813-007df8e322eb // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect - gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/klog/v2 v2.100.1 // indirect - k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect - k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect - sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index 01ed5a9..45ea39f 100644 --- a/go.sum +++ b/go.sum @@ -1,615 +1,46 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= -cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= -github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= -github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= -github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= -github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= -github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= -github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= -github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= -github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= -github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= -github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= -github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= -github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= -github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc= -github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= -github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= -go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= +github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= -golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.142.0 h1:mf+7EJ94fi5ZcnpPy+m0Yv2dkz8bKm+UL0snTCuwXlY= -google.golang.org/api v0.142.0/go.mod h1:zJAN5o6HRqR7O+9qJUFOWrZkYE66RH+efPBdTLA4xBA= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 h1:L6iMMGrtzgHsWofoFcihmDEMYeDR9KN/ThbPWGrh++g= -google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8= -google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 h1:nIgk/EEq3/YlnmVVXVnm14rC2oxgs1o0ong4sD/rd44= -google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5/go.mod h1:5DZzOUPCLYL3mNkQ0ms0F3EuUNZ7py1Bqeq6sxzI7/Q= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230913181813-007df8e322eb h1:Isk1sSH7bovx8Rti2wZK0UZF6oraBDK74uoyLEEVFN0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230913181813-007df8e322eb/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.28.2 h1:9mpl5mOb6vXZvqbQmankOfPIGiudghwCoLl1EYfUZbw= -k8s.io/api v0.28.2/go.mod h1:RVnJBsjU8tcMq7C3iaRSGMeaKt2TWEUXcpIt/90fjEg= -k8s.io/apimachinery v0.28.2 h1:KCOJLrc6gu+wV1BYgwik4AF4vXOlVJPdiqn0yAWWwXQ= -k8s.io/apimachinery v0.28.2/go.mod h1:RdzF87y/ngqk9H4z3EL2Rppv5jj95vGS/HaFXrLDApU= -k8s.io/client-go v0.28.2 h1:DNoYI1vGq0slMBN/SWKMZMw0Rq+0EQW6/AK4v9+3VeY= -k8s.io/client-go v0.28.2/go.mod h1:sMkApowspLuc7omj1FOSUxSoqjr+d5Q0Yc0LOFnYFJY= -k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= -k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= -k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= -k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= -k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/controller-runtime v0.16.2 h1:mwXAVuEk3EQf478PQwQ48zGOXvW27UJc8NHktQVuIPU= sigs.k8s.io/controller-runtime v0.16.2/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/internal/config/config.go b/internal/config/config.go index 5e0d759..d9310a5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,78 +1,24 @@ package config import ( - "strings" - "time" - - "github.com/spf13/viper" -) - -const ( - defaultTicker = 5 * time.Minute + "github.com/urfave/cli/v2" ) -// Config kubeip configuration type Config struct { - KubeConfigPath string - LabelKey string - LabelValue string - NodePool string - ForceAssignment bool - AdditionalNodePools []string - Ticker time.Duration - AllNodePools bool - OrderByLabelKey string - OrderByDesc bool - CopyLabels bool - ClearLabels bool - LogLevel string - LogJSON bool - DryRun bool -} + // KubeConfigPath is the path to the kubeconfig file + KubeConfigPath string `json:"kubeconfig"` + // ClusterName is the name of the EKS cluster + ClusterName string `json:"cluster-name"` + // DevelopMode mode + DevelopMode bool `json:"develop-mode"` + // Weight Model -func setConfigDefaults() { - viper.SetDefault("KubeConfigPath", "") - viper.SetDefault("LabelKey", "kubeip") - viper.SetDefault("LabelValue", "reserved") - viper.SetDefault("NodePool", "default-pool") - viper.SetDefault("ForceAssignment", true) - viper.SetDefault("AdditionalNodePools", "") - viper.SetDefault("Ticker", defaultTicker) - viper.SetDefault("AllNodePools", false) - viper.SetDefault("OrderByLabelKey", "priority") - viper.SetDefault("OrderByDesc", true) - viper.SetDefault("CopyLabels", true) - viper.SetDefault("ClearLabels", true) - viper.SetDefault("LogLevel", "info") - viper.SetDefault("LogJSON", false) - viper.SetDefault("DryRun", false) } -// NewConfig initialize kubeip configuration -func NewConfig() *Config { - viper.SetEnvPrefix("kubeip") - viper.AutomaticEnv() - setConfigDefaults() - - var AdditionalNodePools []string - AdditionalNodePoolsStr := viper.GetString("additionalnodepools") - if len(AdditionalNodePoolsStr) > 0 { - AdditionalNodePools = strings.Split(AdditionalNodePoolsStr, ",") - } - - c := Config{ - LabelKey: viper.GetString("labelkey"), - LabelValue: viper.GetString("labelvalue"), - NodePool: viper.GetString("nodepool"), - ForceAssignment: viper.GetBool("forceassignment"), - AdditionalNodePools: AdditionalNodePools, - Ticker: viper.GetDuration("ticker"), - AllNodePools: viper.GetBool("allnodepools"), - OrderByLabelKey: viper.GetString("orderbylabelkey"), - OrderByDesc: viper.GetBool("orderbydesc"), - CopyLabels: viper.GetBool("copylabels"), - ClearLabels: viper.GetBool("clearlabels"), - DryRun: viper.GetBool("dryrun"), - } - return &c +func LoadConfig(c *cli.Context) Config { + var cfg Config + cfg.KubeConfigPath = c.String("kubeconfig") + cfg.ClusterName = c.String("cluster-name") + cfg.DevelopMode = c.Bool("develop-mode") + return cfg } diff --git a/internal/controller/controller.go b/internal/controller/controller.go deleted file mode 100644 index d1d2496..0000000 --- a/internal/controller/controller.go +++ /dev/null @@ -1,492 +0,0 @@ -package controller - -import ( - "fmt" - "os" - "strings" - "time" - - "github.com/doitintl/kubeip/internal/config" - "github.com/doitintl/kubeip/internal/kipcompute" - "github.com/doitintl/kubeip/internal/types" - "github.com/doitintl/kubeip/internal/utils" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "golang.org/x/net/context" - "golang.org/x/time/rate" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/apimachinery/pkg/watch" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/cache" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/util/workqueue" -) - -const ( - nodeResource = "node" - createEvent = "create" - deleteEvent = "delete" - maxInstances = 100 - rateLimit = 10 - burstTokens = 100 - baseDaley = time.Second - maxDelay = 100 * time.Second - nodeCacheSyncPeriod = 5 * time.Minute -) - -var ( - errTimeout = errors.New("timeout error") -) - -// AddressInstanceTuple object -type AddressInstanceTuple struct { - address string - instance types.Instance -} - -// Controller object -type Controller struct { - logger logrus.FieldLogger - clientset kubernetes.Interface - queue workqueue.RateLimitingInterface - informer cache.SharedInformer - instance chan<- types.Instance - projectID string - clusterName string - config *config.Config - ticker *time.Ticker - processing bool -} - -// Event indicate the informerEvent -type Event struct { - key string - eventType string - resourceType string -} - -var ( - serverStartTime time.Time - errEmptyPath = errors.New("empty path") -) - -const ( - maxRetries = 5 - prefix = "kube-system/kube-proxy-" -) - -func kubeConfigFromPath(kubepath string) (*rest.Config, error) { - if kubepath == "" { - return nil, errEmptyPath - } - - data, err := os.ReadFile(kubepath) - if err != nil { - return nil, errors.Wrapf(err, "reading kubeconfig at %s", kubepath) - } - - cfg, err := clientcmd.RESTConfigFromKubeConfig(data) - if err != nil { - return nil, errors.Wrapf(err, "building rest config from kubeconfig at %s", kubepath) - } - - return cfg, nil -} - -func retrieveKubeConfig(log logrus.FieldLogger, cfg *config.Config) (*rest.Config, error) { - kubeconfig, err := kubeConfigFromPath(cfg.KubeConfigPath) - if err != nil && !errors.Is(err, errEmptyPath) { - return nil, errors.Wrap(err, "retrieving kube config from path") - } - - if kubeconfig != nil { - log.Debug("using kube config from env variables") - return kubeconfig, nil - } - - inClusterConfig, err := rest.InClusterConfig() - if err != nil { - return nil, errors.Wrap(err, "retrieving in cluster kube config") - } - log.Debug("using in cluster kube config") - return inClusterConfig, nil -} - -// Start kubeip controller -func Start(ctx context.Context, log logrus.FieldLogger, project, cluster string, cfg *config.Config) error { - kubeConfig, err := retrieveKubeConfig(log, cfg) - if err != nil { - return errors.Wrap(err, "retrieving kube config") - } - kubeClient, err := kubernetes.NewForConfig(kubeConfig) - if err != nil { - return errors.Wrap(err, "initializing kubernetes client") - } - // Create a new Node informer - nodeInformer := cache.NewSharedInformer( - &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (object runtime.Object, err error) { - return kubeClient.CoreV1().Nodes().List(context.Background(), options) //nolint:wrapcheck - }, - WatchFunc: func(options metav1.ListOptions) (retWc watch.Interface, err error) { - return kubeClient.CoreV1().Nodes().Watch(context.Background(), options) //nolint:wrapcheck - }, - }, - &corev1.Node{}, - nodeCacheSyncPeriod, - ) - - ctrl, err := newResourceController(log, project, cluster, kubeClient, nodeInformer) - if err != nil { - return errors.Wrap(err, "creating resource controller") - } - ctrl.config = cfg - ctrl.ticker = time.NewTicker(ctrl.config.Ticker) - stopCh := make(chan struct{}) - defer close(stopCh) - - instance := make(chan types.Instance, maxInstances) - ctrl.instance = instance - go ctrl.Run(stopCh) - go ctrl.forceAssignment() - - kipcompute.Kubeip(instance, cfg) - - // wait till context is canceled - <-ctx.Done() - return nil -} - -func newResourceController(log logrus.FieldLogger, project, cluster string, client kubernetes.Interface, informer cache.SharedInformer) (*Controller, error) { - queue := workqueue.NewRateLimitingQueue(workqueue.NewMaxOfRateLimiter( - workqueue.NewItemExponentialFailureRateLimiter(baseDaley, maxDelay), - // 10 qps, 100 bucket size. This is only for retry speed, and it's only the overall factor (not per item) - &workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(rateLimit), burstTokens)}, - )) - - _, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - key, err := cache.MetaNamespaceKeyFunc(obj) - if err == nil { - queue.Add(&Event{key, createEvent, nodeResource}) - } - }, - DeleteFunc: func(obj interface{}) { - key, err := cache.MetaNamespaceKeyFunc(obj) - if err == nil { - queue.Add(&Event{key, deleteEvent, nodeResource}) - } - }, - }) - if err != nil { - return nil, errors.Wrap(err, "adding event handler") - } - - return &Controller{ - logger: log.WithField("internal", "kubeip-node"), - projectID: project, - clusterName: cluster, - clientset: client, - informer: informer, - queue: queue, - processing: false, - }, nil -} - -// Run starts the kubeip controller -func (c *Controller) Run(stopCh <-chan struct{}) { - defer utilruntime.HandleCrash() - defer c.queue.ShutDown() - - serverStartTime = time.Now().Local() - go c.informer.Run(stopCh) - if !cache.WaitForCacheSync(stopCh, c.HasSynced) { - utilruntime.HandleError(errTimeout) - return - } - - c.logger.Info("kubeip controller synced and ready") - wait.Until(c.runWorker, time.Second, stopCh) -} - -// HasSynced is required for the cache.Controller interface. -func (c *Controller) HasSynced() bool { - return c.informer.HasSynced() -} - -// LastSyncResourceVersion is required for the cache.Controller interface. -func (c *Controller) LastSyncResourceVersion() string { - return c.informer.LastSyncResourceVersion() -} - -func (c *Controller) runWorker() { - for c.processNextItem() { - } -} - -func (c *Controller) processNextItem() bool { - newEvent, quit := c.queue.Get() - - if quit { - return false - } - defer c.queue.Done(newEvent) - err := c.processItem(newEvent.(Event)) - if err == nil { - // No error, reset the ratelimit counters - c.queue.Forget(newEvent) - } else if c.queue.NumRequeues(newEvent) < maxRetries { - c.logger.WithError(err).Errorf("error processing %s (will retry)", newEvent.(Event).key) - c.queue.AddRateLimited(newEvent) - } else { - // err != nil and too many retries - c.logger.WithError(err).Errorf("error processing %s (giving up)", newEvent.(Event).key) - c.queue.Forget(newEvent) - utilruntime.HandleError(err) - } - - return true -} - -func (c *Controller) isNodePoolMonitored(pool string) bool { - if c.config.AllNodePools { - return true - } - if strings.EqualFold(pool, c.config.NodePool) { - return true - } - for _, ns := range c.config.AdditionalNodePools { - if strings.EqualFold(pool, ns) { - return true - } - } - return false -} - -func (c *Controller) processItem(newEvent Event) error { - obj, _, err := c.informer.GetStore().GetByKey(newEvent.key) - if err != nil { - return errors.Wrapf(err, "getting object from informer by key %s", newEvent.key) - } - // get object's metadata - objectMeta := utils.GetObjectMetaData(obj) - - // process events based on its type - switch newEvent.eventType { - case "delete": - if strings.HasPrefix(newEvent.key, prefix) { - // A node has been deleted... we need to check whether the assignment is still optimal - c.forceAssignmentOnce(true) - return nil - } - case "create": - // compare CreationTimestamp and serverStartTime and alert only on latest events - // Could be Replaced by using Delta or DeltaFIFO - if objectMeta.CreationTimestamp.Sub(serverStartTime).Seconds() > 0 { - if strings.HasPrefix(newEvent.key, prefix) { - kubeClient := utils.GetClient() - node := newEvent.key[len(prefix):] - var options metav1.GetOptions - options.Kind = "Node" - options.APIVersion = "1" - var nodeMeta *corev1.Node - nodeMeta, err = kubeClient.CoreV1().Nodes().Get(context.Background(), node, options) - if err != nil { - return errors.Wrap(err, "failed to get node") - } - - labels := nodeMeta.Labels - var pool string - var ok bool - if pool, ok = labels["cloud.google.com/gke-nodepool"]; ok { - if !c.isNodePoolMonitored(pool) { - return nil - } - } else { - return errors.New("failed to find node pool") - } - var inst types.Instance - if nodeZone, ok := labels["failure-domain.beta.kubernetes.io/zone"]; ok { - inst.Zone = nodeZone - } else { - return errors.New("failed to find zone") - } - inst.Name = node - inst.ProjectID = c.projectID - inst.Pool = pool - c.instance <- inst - return nil - } - } - } - return nil -} - -func isNodeReady(node *corev1.Node) bool { - for _, condition := range node.Status.Conditions { - if condition.Type == corev1.NodeReady { - // If the node is unknown we assume that it is ready, we do not want to do IP changes so rapidly. - return condition.Status == corev1.ConditionTrue || condition.Status == corev1.ConditionUnknown - } - } - return false -} - -func (c *Controller) processAllNodes(shouldCheckOptimalIPAssignment bool) error { //nolint:funlen,gocognit,gocyclo - kubeClient := utils.GetClient() - nodelist, err := kubeClient.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) - if err != nil { - return errors.Wrap(err, "failed to list nodes") - } - nodesOfInterest := make([]types.Instance, 0, len(nodelist.Items)) - - for node := range nodelist.Items { - var inst types.Instance - labels := nodelist.Items[node].GetLabels() - if pool, ok := labels["cloud.google.com/gke-nodepool"]; ok { - if !c.isNodePoolMonitored(pool) { - continue - } - inst.Pool = pool - } else { - c.logger.Warn("failed to find node pool") - continue - } - - if nodeZone, ok := labels["failure-domain.beta.kubernetes.io/zone"]; ok { - inst.Zone = nodeZone - } else { - c.logger.Warn("failed to find zone") - continue - } - inst.ProjectID = c.projectID - inst.Name = nodelist.Items[node].GetName() - - // If node is not ready we will basically remove the node IP just in case - if !isNodeReady(&nodelist.Items[node]) { - c.logger.Debugf("node %s in zone %s is not ready, remove IP for reuse", inst.Name, inst.Zone) - // Delete the IP we will re-assign this - err = kipcompute.DeleteIP(c.projectID, inst.Zone, inst.Name, c.config) - if err != nil { - return errors.Wrap(err, "failed to delete IP") - } - continue - } - nodesOfInterest = append(nodesOfInterest, inst) - } - - // Should we check that the IPs assigned to the current nodes are in fact the best possible IPs to assign? - if shouldCheckOptimalIPAssignment { - // Determining the required IP per region - regionsCount := make(map[string][]types.Instance) - c.logger.Debugf("collected %d Nodes of interest...calculating number of IPs required", len(nodesOfInterest)) - for _, inst := range nodesOfInterest { - zone := inst.Zone - region := zone[:len(zone)-2] - regionsCount[region] = append(regionsCount[region], inst) - } - - // Determining the most optimal nodes per region. - for region, instances := range regionsCount { - addresses, err := kipcompute.GetAllAddresses(c.projectID, region, false, c.config) - if err != nil { - return errors.Wrap(err, "failed to retrieve all addresses") - } - - var topMostAddresses []string - for _, address := range addresses.Items[:utils.Min(len(instances), len(addresses.Items))] { - topMostAddresses = append(topMostAddresses, address.Address) - } - - // Retrieve all addresses in the region. - var usedAddresses []AddressInstanceTuple - for _, instance := range instances { - address, errGet := kipcompute.GetAddressUsedByInstance(c.projectID, instance.Name, instance.Zone, c.config) - if errGet != nil { - return errors.Wrap(errGet, "failed to retrieve address") - } - usedAddresses = append(usedAddresses, AddressInstanceTuple{ - address, - instance, - }) - } - - // Remove all addresses that are not in the top most addresses. - var toRemove []AddressInstanceTuple - for _, usedAddress := range usedAddresses { - if usedAddress.address != "0.0.0.0" && !utils.Contains(topMostAddresses, usedAddress.address) { - toRemove = append(toRemove, usedAddress) - } - } - if len(toRemove) > 0 { - for _, remove := range toRemove { - // Delete the IP we will re-assign this - err = kipcompute.DeleteIP(c.projectID, remove.instance.Zone, remove.instance.Name, c.config) - if err != nil { - return errors.Wrap(err, "failed to delete IP") - } - } - } - } - } - - // Find all nodes that do not have an IP assigned and send them to the channel for assignment. - for _, inst := range nodesOfInterest { - if !kipcompute.IsInstanceUsesReservedIP(c.projectID, inst.Name, inst.Zone, c.config) { - c.logger.WithFields(logrus.Fields{ - "instance": inst.Name, - "pool": inst.Pool, - }).Debugf("found unassigned node in pool") - c.instance <- inst - } - } - return nil -} - -func (c *Controller) forceAssignmentOnce(shouldCheckOptimalIPAssignment bool) { - if !c.processing { - c.processing = true - if c.config.ForceAssignment { - err := c.processAllNodes(shouldCheckOptimalIPAssignment) - if err != nil { - c.logger.WithError(err).Error("failed to process all nodes") - } - } - c.assignMissingTags() - c.processing = false - } -} - -func (c *Controller) forceAssignment() { - c.forceAssignmentOnce(true) - for range c.ticker.C { - c.forceAssignmentOnce(false) - } -} - -func (c *Controller) assignMissingTags() { - nodePools := c.config.AdditionalNodePools - nodePools = append(nodePools, c.config.NodePool) - - kubeClient := utils.GetClient() - - for _, pool := range nodePools { - label := fmt.Sprintf("!kubip_assigned,cloud.google.com/gke-nodepool=%s", pool) - nodelist, err := kubeClient.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{LabelSelector: label}) - if err != nil { - c.logger.Warn("failed to list nodes") - continue - } - for _, node := range nodelist.Items { - labels := node.GetLabels() - if nodeZone, ok := labels["failure-domain.beta.kubernetes.io/zone"]; ok { - kipcompute.AddTagIfMissing(c.projectID, node.GetName(), nodeZone, c.config) - } - } - } -} diff --git a/internal/kipcompute/compute.go b/internal/kipcompute/compute.go deleted file mode 100644 index 7787c58..0000000 --- a/internal/kipcompute/compute.go +++ /dev/null @@ -1,342 +0,0 @@ -// Copyright © 2021 DoiT International -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package kipcompute - -import ( - "math" - "sort" - "strconv" - "strings" - "time" - - "cloud.google.com/go/compute/metadata" - "github.com/doitintl/kubeip/internal/config" - "github.com/doitintl/kubeip/internal/types" - "github.com/doitintl/kubeip/internal/utils" - "github.com/pkg/errors" - "golang.org/x/net/context" - "google.golang.org/api/compute/v0.beta" //nolint:goimports -) - -const ( - waitTime = 2 * time.Second -) - -// ClusterName get GKE cluster name from metadata -func ClusterName() (string, error) { - return metadata.InstanceAttributeValue("cluster-name") //nolint:wrapcheck -} - -// ProjectName get GCP project name from metadata -func ProjectName() (string, error) { - return metadata.ProjectID() //nolint:wrapcheck -} - -func getPriorityOrder(address *compute.Address, cfg *config.Config) int { - var defaultValue int - if cfg.OrderByDesc { - defaultValue = math.MinInt - } else { - defaultValue = math.MaxInt - } - - strVal, ok := address.Labels[cfg.OrderByLabelKey] - if ok { - intVal, err := strconv.Atoi(strVal) - if err != nil { - return defaultValue - } - return intVal - } - - return defaultValue -} - -// GetAllAddresses retrieves all addresses matching the query. -func GetAllAddresses(projectID, region string, filterJustReserved bool, cfg *config.Config) (*compute.AddressList, error) { - return getAllAddresses(projectID, region, cfg.NodePool, filterJustReserved, cfg) -} - -func getAllAddresses(projectID, region, pool string, filterJustReserved bool, cfg *config.Config) (*compute.AddressList, error) { - computeService, err := compute.NewService(context.Background()) - if err != nil { - return nil, errors.Wrap(err, "failed to get create compute service") - } - var filter string - if cfg.AllNodePools || strings.EqualFold(pool, cfg.NodePool) { - filter = "(labels." + cfg.LabelKey + "=" + cfg.LabelValue + ")" + " AND (-labels." + cfg.LabelKey + "-node-pool:*)" - } else { - filter = "(labels." + cfg.LabelKey + "=" + cfg.LabelValue + ")" + " AND " + "(labels." + cfg.LabelKey + "-node-pool=" + pool + ")" - } - - var computedFilter string - if filterJustReserved { - computedFilter = "(status=RESERVED) AND " + filter - } else { - computedFilter = filter - } - - var addresses *compute.AddressList - addresses, err = computeService.Addresses.List(projectID, region).Filter(computedFilter).Do() - - if err != nil { - return nil, errors.Wrap(err, "failed to list addresses") - } - - // Right now the SDK does not support filter and order together, so we do it programmatically. - sort.SliceStable(addresses.Items, func(i, j int) bool { - address1 := addresses.Items[i] - address2 := addresses.Items[j] - val1 := getPriorityOrder(address1, cfg) - val2 := getPriorityOrder(address2, cfg) - if cfg.OrderByDesc { - return val1 > val2 - } - return val1 < val2 - }) - - return addresses, nil -} - -func findFreeAddress(projectID, region, pool string, cfg *config.Config) (*types.IPAddress, error) { - addresses, err := getAllAddresses(projectID, region, pool, true, cfg) - if err != nil { - return nil, err - } - - if len(addresses.Items) != 0 { - address := addresses.Items[0] - return &types.IPAddress{IP: address.Address, Labels: address.Labels}, nil - } - return nil, errors.New("no free address found") -} - -// DeleteIP delete current IP on GKE node -func DeleteIP(projectID, zone, instance string, cfg *config.Config) error { - computeService, err := compute.NewService(context.Background()) - if err != nil { - return errors.Wrap(err, "failed to get create compute service") - } - inst, err := computeService.Instances.Get(projectID, zone, instance).Do() - if err != nil { - return errors.Wrapf(err, "failed to get instance %s zone %s", instance, zone) - } - if len(inst.NetworkInterfaces) > 0 && len(inst.NetworkInterfaces[0].AccessConfigs) > 0 { - accessConfigName := inst.NetworkInterfaces[0].AccessConfigs[0].Name - if !cfg.DryRun { - op, err := computeService.Instances.DeleteAccessConfig(projectID, zone, instance, accessConfigName, "nic0").Do() - if err != nil { - return errors.Wrap(err, "failed to delete access config") - } - err = waitForCompilation(projectID, zone, op) - if err != nil { - return errors.Wrap(err, "failed to wait for compilation") - } - } - } - // Delete an prior tags. - utils.TagNode(instance, &types.IPAddress{IP: "0.0.0.0", Labels: map[string]string{}}, cfg) - return nil -} - -func addIP(projectID, zone, instance string, addr *types.IPAddress, cfg *config.Config) error { - computeService, err := compute.NewService(context.Background()) - if err != nil { - return errors.Wrap(err, "failed to get create compute service") - } - if !cfg.DryRun { - accessConfig := &compute.AccessConfig{ - Name: "External NAT", - Type: "ONE_TO_ONE_NAT", - NatIP: addr.IP, - Kind: "compute#accessConfig", - } - var op *compute.Operation - op, err = computeService.Instances.AddAccessConfig(projectID, zone, instance, "nic0", accessConfig).Do() - if err != nil { - return errors.Wrap(err, "failed to add access config") - } - err = waitForCompilation(projectID, zone, op) - if err != nil { - return errors.Wrap(err, "failed to wait for compilation") - } - } - return nil -} - -func replaceIP(projectID, zone, instance, pool string, cfg *config.Config) error { - region := zone[:len(zone)-2] - addr, err := findFreeAddress(projectID, region, pool, cfg) - // Check if we found address. - if err != nil { - return errors.Wrap(err, "failed to find free address") - } - - err = DeleteIP(projectID, zone, instance, cfg) - if err != nil { - return errors.Wrap(err, "failed to delete IP") - } - - err = addIP(projectID, zone, instance, addr, cfg) - if err != nil { - return errors.Wrap(err, "failed to add IP") - } - - oldNode, err := utils.GetNodeByIP(addr.IP) - if err == nil { - utils.TagNode(oldNode, &types.IPAddress{IP: "0.0.0.0", Labels: map[string]string{}}, cfg) - } - utils.TagNode(instance, addr, cfg) - return nil -} - -func waitForCompilation(projectID, zone string, operation *compute.Operation) (err error) { - computeService, err := compute.NewService(context.Background()) - if err != nil { - return errors.Wrap(err, "failed to get create compute service") - } - for { - var op *compute.Operation - op, err = computeService.ZoneOperations.Get(projectID, zone, operation.Name).Do() - if err != nil { - return errors.Wrapf(err, "failed to get zone operations resource: %s", operation.Name) - } - if !strings.EqualFold(op.Status, "done") { - time.Sleep(waitTime) - } else { - return nil - } - } -} - -// GetAddressUsedByInstance returns the IP used by this instance or the broadcast address otherwise. -func GetAddressUsedByInstance(projectID, instance, zone string, cfg *config.Config) (string, error) { - computeService, err := compute.NewService(context.Background()) - if err != nil { - return "", errors.Wrap(err, "failed to get create compute service") - } - - region := zone[:len(zone)-2] - filter := "(labels." + cfg.LabelKey + "=" + cfg.LabelValue + ")" - addresses, err := computeService.Addresses.List(projectID, region).Filter(filter).Do() - if err != nil { - return "", errors.Wrap(err, "failed to list addresses") - } - - for _, addr := range addresses.Items { - if len(addr.Users) > 0 && strings.Contains(addr.Users[0], instance) { - return addr.Address, nil - } - } - - return "0.0.0.0", nil -} - -// IsInstanceUsesReservedIP test if GKE node is using reserved IP -func IsInstanceUsesReservedIP(projectID, instance, zone string, cfg *config.Config) bool { - computeService, err := compute.NewService(context.Background()) - if err != nil { - return false - } - region := zone[:len(zone)-2] - filter := "(labels." + cfg.LabelKey + "=" + cfg.LabelValue + ")" - addresses, err := computeService.Addresses.List(projectID, region).Filter("(status=IN_USE) AND " + filter).Do() - if err != nil { - return false - } - - for _, addr := range addresses.Items { - if strings.Contains(addr.Users[0], instance) { - return true - } - } - return false -} - -// Kubeip replace GKE node IP -func Kubeip(instance <-chan types.Instance, cfg *config.Config) { - for { - inst := <-instance - _ = replaceIP(inst.ProjectID, inst.Zone, inst.Name, inst.Pool, cfg) - } -} - -func getAddressDetails(ip, region, projectID string) (*types.IPAddress, error) { - computeService, err := compute.NewService(context.Background()) - if err != nil { - return nil, errors.Wrap(err, "failed to get create compute service") - } - filter := "address=" + "\"" + ip + "\"" - - addresses, err := computeService.Addresses.List(projectID, region).Filter(filter).Do() - if err != nil { - return nil, errors.Wrap(err, "failed to list addresses") - } - - if len(addresses.Items) != 1 { - address := addresses.Items[0] - return &types.IPAddress{IP: address.Address, Labels: address.Labels}, nil - } - - return nil, errors.New("more than one address found") -} - -func isAddressReserved(ip, region, projectID string) bool { - computeService, err := compute.NewService(context.Background()) - if err != nil { - return false - } - filter := "address=" + "\"" + ip + "\"" - addresses, err := computeService.Addresses.List(projectID, region).Filter(filter).Do() - if err != nil { - return false - } - - if len(addresses.Items) != 0 { - return true - } - return false -} - -// AddTagIfMissing add GKE node tag if missing -func AddTagIfMissing(projectID, instance, zone string, cfg *config.Config) { - computeService, err := compute.NewService(context.Background()) - if err != nil { - return - } - inst, err := computeService.Instances.Get(projectID, zone, instance).Do() - if err != nil { - return - } - var ip string - for _, c := range inst.NetworkInterfaces[0].AccessConfigs { - if c.NatIP != "" { - ip = c.NatIP - } - } - if isAddressReserved(ip, zone[:len(zone)-2], projectID) { - addressDetails, err := getAddressDetails(ip, zone, projectID) - if err != nil { - return - } - utils.TagNode(instance, addressDetails, cfg) - } -} diff --git a/internal/types/types.go b/internal/types/types.go deleted file mode 100644 index b506a76..0000000 --- a/internal/types/types.go +++ /dev/null @@ -1,16 +0,0 @@ -package types - -// Instance GKE Instance VM -type Instance struct { - ProjectID string - Name string - Zone string - Pool string -} - -// IPAddress GKE IP -type IPAddress struct { - IP string - Name string - Labels map[string]string -} diff --git a/internal/utils/k8sutil.go b/internal/utils/k8sutil.go deleted file mode 100644 index 6f73a57..0000000 --- a/internal/utils/k8sutil.go +++ /dev/null @@ -1,173 +0,0 @@ -package utils - -import ( - "bytes" - "fmt" - "strings" - - "github.com/doitintl/kubeip/internal/config" - "github.com/doitintl/kubeip/internal/types" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "golang.org/x/net/context" - appsv1 "k8s.io/api/apps/v1" - batchv1 "k8s.io/api/batch/v1" - apiv1 "k8s.io/api/core/v1" - extv1beta1 "k8s.io/api/extensions/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - typesv1 "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" -) - -// Min helper method to determine the minimum between two numbers -func Min(a, b int) int { - if a < b { - return a - } - return b -} - -// Contains helper method to determine if a string is contained in an array -func Contains(s []string, e string) bool { - for _, a := range s { - if strings.EqualFold(a, e) { - return true - } - } - return false -} - -// GetClient returns a k8s clientset to the request from inside of cluster -func GetClient() kubernetes.Interface { - cfg, err := rest.InClusterConfig() - if err != nil { - logrus.Fatalf("Can not get kubernetes config: %v", err) - } - - clientset, err := kubernetes.NewForConfig(cfg) - if err != nil { - logrus.Fatalf("Can not create kubernetes client: %v", err) - } - - return clientset -} - -// GetObjectMetaData returns metadata of a given k8s object -func GetObjectMetaData(obj interface{}) metav1.ObjectMeta { - var objectMeta metav1.ObjectMeta - - switch object := obj.(type) { - case *appsv1.Deployment: - objectMeta = object.ObjectMeta - case *apiv1.ReplicationController: - objectMeta = object.ObjectMeta - case *appsv1.ReplicaSet: - objectMeta = object.ObjectMeta - case *appsv1.DaemonSet: - objectMeta = object.ObjectMeta - case *apiv1.Service: - objectMeta = object.ObjectMeta - case *apiv1.Pod: - objectMeta = object.ObjectMeta - case *batchv1.Job: - objectMeta = object.ObjectMeta - case *apiv1.PersistentVolume: - objectMeta = object.ObjectMeta - case *apiv1.Namespace: - objectMeta = object.ObjectMeta - case *apiv1.Secret: - objectMeta = object.ObjectMeta - case *extv1beta1.Ingress: - objectMeta = object.ObjectMeta - } - return objectMeta -} - -func clearLabels(m map[string]string, cfg *config.Config) string { - stringBuffer := new(bytes.Buffer) - for key := range m { - if !strings.EqualFold(key, cfg.OrderByLabelKey) && - !strings.EqualFold(key, cfg.LabelKey) && - !strings.Contains(key, "kubip_assigned") && - !strings.Contains(key, "kubernetes") && - !strings.Contains(key, "google") && - !strings.Contains(key, "gke") { - fmt.Fprintf(stringBuffer, " ,\"%s\":null", key) - } - } - return stringBuffer.String() -} - -func createLabelKeyValuePairs(m map[string]string, cfg *config.Config) string { - stringBuffer := new(bytes.Buffer) - for key, value := range m { - if !strings.EqualFold(key, cfg.OrderByLabelKey) && - !strings.EqualFold(key, cfg.LabelKey) && - !strings.Contains(key, "kubip_assigned") && - !strings.Contains(key, "kubernetes") && - !strings.Contains(key, "google") && - !strings.Contains(key, "gke") { - fmt.Fprintf(stringBuffer, " ,\"%s\":\"%s\"", key, value) - } - } - return stringBuffer.String() -} - -// TagNode tag GKE node with "kubip_assigned" label (with typo) and also copy the labels present on the address if the copyLabels flag is set to true -func TagNode(node string, ip *types.IPAddress, cfg *config.Config) { - kubeClient := GetClient() - logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "tagNode"}).Infof("Tagging node %s as %s", node, ip.IP) - // replace . with - in IP address - dashIP := strings.Replace(ip.IP, ".", "-", 4) //nolint:gomnd - var labelString string - - if cfg.CopyLabels { - var labelsToClear string - if cfg.ClearLabels { - result, err := kubeClient.CoreV1().Nodes().Get(context.Background(), node, metav1.GetOptions{}) - if err != nil { - logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "tagNode"}).Error(err) - } else { - logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "tagNode"}).Infof("Clear label tag for node %s with ip %s and clear tags %s", node, ip.IP, result.Labels) - createLabelKeyValuePairs(result.Labels, cfg) - labelsToClear = clearLabels(result.Labels, cfg) - } - } else { - labelsToClear = "" - } - - labelString = "{" + "\"" + "kubip_assigned" + "\":\"" + dashIP + "\"" + labelsToClear + createLabelKeyValuePairs(ip.Labels, cfg) + "}" - } else { - labelString = "{" + "\"" + "kubip_assigned" + "\":\"" + dashIP + "\"" + "}" - } - patch := fmt.Sprintf(`{"metadata":{"labels":%v}}`, labelString) - - if cfg.DryRun { - logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "tagNode"}).Infof("Tagging node %s as %s with tags %s ", node, ip.IP, labelString) - } else { - logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "tagNode"}).Infof("Tagging node %s as %s with tags %s ", node, ip.IP, labelString) - _, err := kubeClient.CoreV1().Nodes().Patch(context.Background(), node, typesv1.MergePatchType, []byte(patch), metav1.PatchOptions{}) - if err != nil { - logrus.WithFields(logrus.Fields{"internal": "kubeip", "function": "tagNode"}).Infof("Error occurred while tagging node %s as %s with tags %s ", node, ip.IP, labelString) - logrus.Error(err) - } - } -} - -// GetNodeByIP get GKE node by IP -func GetNodeByIP(ip string) (string, error) { - kubeClient := GetClient() - dashIP := strings.Replace(ip, ".", "-", 4) //nolint:gomnd - label := fmt.Sprintf("kubip_assigned=%v", dashIP) - l, err := kubeClient.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{ - LabelSelector: label, - }) - if err != nil { - return "", errors.Wrap(err, "failed to list nodes") - } - if len(l.Items) == 0 { - return "", errors.New("did not found matching node with IP") - } - return l.Items[0].GetName(), nil -} From ced5718e4cca5025b859d73346d316bcbf636c9e Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Thu, 21 Sep 2023 19:03:05 +0300 Subject: [PATCH 08/66] rename docker image to kubeip-agent --- .github/workflows/build.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9344f61..9451032 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -73,7 +73,7 @@ jobs: id: meta uses: docker/metadata-action@v4 with: - images: ${{ github.repository }} + images: ${{ github.repository }}-agent tags: | type=ref,event=branch type=ref,event=pr @@ -83,8 +83,8 @@ jobs: type=sha labels: | github.run.id=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - org.opencontainers.image.title=kubeIP - org.opencontainers.image.description=kubeIP controller + org.opencontainers.image.title=kubeip-agent + org.opencontainers.image.description=kubeip agent org.opencontainers.image.vendor=DoiT International - name: build and push From cae358917e763a466f21c4c1cc25c761febd208a Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Thu, 21 Sep 2023 19:10:10 +0300 Subject: [PATCH 09/66] new flags: retry interval and retry attempts --- cmd/main.go | 23 ++++++++++++++++++++++- internal/config/config.go | 9 +++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 1db7a46..b607fc8 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "runtime" + "time" "github.com/doitintl/kubeip/internal/config" "github.com/sirupsen/logrus" @@ -19,6 +20,12 @@ var ( gitBranch string ) +const ( + // DefaultRetryInterval is the default retry interval + defaultRetryInterval = 5 * time.Minute + defaultRetryAttempts = 10 +) + func prepareLogger(level string, json bool) *logrus.Entry { logger := logrus.New() @@ -85,10 +92,24 @@ func main() { }, &cli.PathFlag{ Name: "kubeconfig", - Usage: "Path to Kubernetes configuration file (not needed if running in cluster)", + Usage: "path to Kubernetes configuration file (not needed if running in cluster)", EnvVars: []string{"KUBECONFIG"}, Category: "Configuration", }, + &cli.DurationFlag{ + Name: "retry-interval", + Usage: "when the agent fails to assign the static public IP address, it will retry after this interval", + Value: defaultRetryInterval, + EnvVars: []string{"RETRY_INTERVAL"}, + Category: "Configuration", + }, + &cli.IntFlag{ + Name: "retry-attempts", + Usage: "number of attempts to assign the static public IP address", + Value: defaultRetryAttempts, + EnvVars: []string{"RETRY_ATTEMPTS"}, + Category: "Configuration", + }, &cli.StringFlag{ Name: "log-level", Usage: "set log level (debug, info(*), warning, error, fatal, panic)", diff --git a/internal/config/config.go b/internal/config/config.go index d9310a5..88a2613 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,6 +1,8 @@ package config import ( + "time" + "github.com/urfave/cli/v2" ) @@ -11,8 +13,10 @@ type Config struct { ClusterName string `json:"cluster-name"` // DevelopMode mode DevelopMode bool `json:"develop-mode"` - // Weight Model - + // Retry interval + RetryInterval time.Duration `json:"retry-interval"` + // Retry attempts + RetryAttempts int `json:"retry-attempts"` } func LoadConfig(c *cli.Context) Config { @@ -20,5 +24,6 @@ func LoadConfig(c *cli.Context) Config { cfg.KubeConfigPath = c.String("kubeconfig") cfg.ClusterName = c.String("cluster-name") cfg.DevelopMode = c.Bool("develop-mode") + cfg.RetryInterval = c.Duration("retry-interval") return cfg } From 97563137bd9c8f99024b145fb62454dc6755476a Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Thu, 21 Sep 2023 19:34:32 +0300 Subject: [PATCH 10/66] node type added --- cmd/main.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/main.go b/cmd/main.go index b607fc8..2dafc16 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -61,7 +61,11 @@ func prepareLogger(level string, json bool) *logrus.Entry { return log } -func run(_ context.Context, _ *logrus.Entry, _ config.Config) error { +func run(ctx context.Context, log *logrus.Entry, _ config.Config) error { + log.Infof("assigning static public IP address to the node") + + <-ctx.Done() + log.Info("release static public IP address from the node") return nil } From 2f608cdda4c2f5952ad973d2c247917dd67dc453 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Thu, 21 Sep 2023 19:34:42 +0300 Subject: [PATCH 11/66] node type added --- internal/types/node.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 internal/types/node.go diff --git a/internal/types/node.go b/internal/types/node.go new file mode 100644 index 0000000..91f283c --- /dev/null +++ b/internal/types/node.go @@ -0,0 +1,16 @@ +package types + +type Address struct { + IP string + Type string +} + +type Node struct { + Name string + Cluster string + Pool string + Spot bool + Region string + Zone string + Addresses []string +} From 1ac1b82b8f3e9affcc238480cd59e324c1c75053 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Tue, 26 Sep 2023 09:08:49 +0300 Subject: [PATCH 12/66] explore k8s node --- cmd/main.go | 97 ++++++++++++++++++++++++++--- go.mod | 39 ++++++++++++ go.sum | 126 ++++++++++++++++++++++++++++++++++++++ internal/config/config.go | 6 +- internal/types/node.go | 27 ++++---- 5 files changed, 271 insertions(+), 24 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 2dafc16..3fdc1d0 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -8,16 +8,28 @@ import ( "time" "github.com/doitintl/kubeip/internal/config" + "github.com/doitintl/kubeip/internal/node" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/controller-runtime/pkg/manager/signals" ) +type contextKey string + +const ( + developModeKey contextKey = "develop-mode" +) + var ( - version string - buildDate string - gitCommit string - gitBranch string + version string + buildDate string + gitCommit string + gitBranch string + errEmptyPath = errors.New("empty path") ) const ( @@ -54,6 +66,9 @@ func prepareLogger(level string, json bool) *logrus.Entry { logger.SetFormatter(&logrus.JSONFormatter{}) } + // record the file name and line number of the log + logger.SetReportCaller(true) + log := logger.WithFields(logrus.Fields{ "version": version, }) @@ -61,11 +76,36 @@ func prepareLogger(level string, json bool) *logrus.Entry { return log } -func run(ctx context.Context, log *logrus.Entry, _ config.Config) error { - log.Infof("assigning static public IP address to the node") +func run(c context.Context, log *logrus.Entry, cfg config.Config) error { + ctx, cancel := context.WithCancel(c) + defer cancel() + // add debug mode to context + if cfg.DevelopMode { + ctx = context.WithValue(ctx, developModeKey, true) + } + + log.WithField("develop-mode", cfg.DevelopMode).Infof("kubeip agent started") + + restconfig, err := retrieveKubeConfig(log, cfg) + if err != nil { + return errors.Wrap(err, "retrieving kube config") + } + + clientset, err := kubernetes.NewForConfig(restconfig) + if err != nil { + return errors.Wrap(err, "initializing kubernetes client") + } + + explorer := node.NewExplorer(clientset) + n, err := explorer.GetNode(ctx) + if err != nil { + return errors.Wrap(err, "getting node") + } + + log.Debug("node name: ", n.Name) <-ctx.Done() - log.Info("release static public IP address from the node") + log.Infof("kubeip agent stopped") return nil } @@ -89,14 +129,14 @@ func main() { Usage: "run agent", Flags: []cli.Flag{ &cli.StringFlag{ - Name: "cluster-name", - Usage: "Kubernetes cluster name (not needed if running in cluster)", + Name: "node-name", + Usage: "Kubernetes node name (not needed if running in node)", EnvVars: []string{"CLUSTER_NAME"}, Category: "Configuration", }, &cli.PathFlag{ Name: "kubeconfig", - Usage: "path to Kubernetes configuration file (not needed if running in cluster)", + Usage: "path to Kubernetes configuration file (not needed if running in node)", EnvVars: []string{"KUBECONFIG"}, Category: "Configuration", }, @@ -154,3 +194,40 @@ func main() { logrus.Fatal(err) } } + +func kubeConfigFromPath(kubepath string) (*rest.Config, error) { + if kubepath == "" { + return nil, errEmptyPath + } + + data, err := os.ReadFile(kubepath) + if err != nil { + return nil, errors.Wrapf(err, "reading kubeconfig at %s", kubepath) + } + + cfg, err := clientcmd.RESTConfigFromKubeConfig(data) + if err != nil { + return nil, errors.Wrapf(err, "building rest config from kubeconfig at %s", kubepath) + } + + return cfg, nil +} + +func retrieveKubeConfig(log logrus.FieldLogger, cfg config.Config) (*rest.Config, error) { + kubeconfig, err := kubeConfigFromPath(cfg.KubeConfigPath) + if err != nil && !errors.Is(err, errEmptyPath) { + return nil, errors.Wrap(err, "retrieving kube config from path") + } + + if kubeconfig != nil { + log.Debug("using kube config from env variables") + return kubeconfig, nil + } + + inClusterConfig, err := rest.InClusterConfig() + if err != nil { + return nil, errors.Wrap(err, "retrieving in node kube config") + } + log.Debug("using in node kube config") + return inClusterConfig, nil +} diff --git a/go.mod b/go.mod index fec6dac..597e053 100644 --- a/go.mod +++ b/go.mod @@ -3,16 +3,55 @@ module github.com/doitintl/kubeip go 1.21 require ( + github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.3 github.com/urfave/cli/v2 v2.25.7 + k8s.io/apimachinery v0.28.1 + k8s.io/client-go v0.28.1 sigs.k8s.io/controller-runtime v0.16.2 ) require ( github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.9.0 // indirect + github.com/go-logr/logr v1.2.4 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.3 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.5.9 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/imdario/mergo v0.3.6 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/testify v1.8.3 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect golang.org/x/net v0.15.0 // indirect + golang.org/x/oauth2 v0.8.0 // indirect golang.org/x/sys v0.12.0 // indirect + golang.org/x/term v0.12.0 // indirect + golang.org/x/text v0.13.0 // indirect + golang.org/x/time v0.3.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.30.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/api v0.28.1 // indirect + k8s.io/klog/v2 v2.100.1 // indirect + k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect + k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index 45ea39f..0aec49c 100644 --- a/go.sum +++ b/go.sum @@ -1,46 +1,172 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= +github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.28.1 h1:i+0O8k2NPBCPYaMB+uCkseEbawEt/eFaiRqUx8aB108= +k8s.io/api v0.28.1/go.mod h1:uBYwID+66wiL28Kn2tBjBYQdEU0Xk0z5qF8bIBqk/Dg= +k8s.io/apimachinery v0.28.1 h1:EJD40og3GizBSV3mkIoXQBsws32okPOy+MkRyzh6nPY= +k8s.io/apimachinery v0.28.1/go.mod h1:X0xh/chESs2hP9koe+SdIAcXWcQ+RM5hy0ZynB+yEvw= +k8s.io/client-go v0.28.1 h1:pRhMzB8HyLfVwpngWKE8hDcXRqifh1ga2Z/PU9SXVK8= +k8s.io/client-go v0.28.1/go.mod h1:pEZA3FqOsVkCc07pFVzK076R+P/eXqsgx5zuuRWukNE= +k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= +k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= +k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/controller-runtime v0.16.2 h1:mwXAVuEk3EQf478PQwQ48zGOXvW27UJc8NHktQVuIPU= sigs.k8s.io/controller-runtime v0.16.2/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/internal/config/config.go b/internal/config/config.go index 88a2613..9b055b0 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -9,8 +9,8 @@ import ( type Config struct { // KubeConfigPath is the path to the kubeconfig file KubeConfigPath string `json:"kubeconfig"` - // ClusterName is the name of the EKS cluster - ClusterName string `json:"cluster-name"` + // ClusterName is the name of the EKS node + ClusterName string `json:"node-name"` // DevelopMode mode DevelopMode bool `json:"develop-mode"` // Retry interval @@ -22,7 +22,7 @@ type Config struct { func LoadConfig(c *cli.Context) Config { var cfg Config cfg.KubeConfigPath = c.String("kubeconfig") - cfg.ClusterName = c.String("cluster-name") + cfg.ClusterName = c.String("node-name") cfg.DevelopMode = c.Bool("develop-mode") cfg.RetryInterval = c.Duration("retry-interval") return cfg diff --git a/internal/types/node.go b/internal/types/node.go index 91f283c..654adaf 100644 --- a/internal/types/node.go +++ b/internal/types/node.go @@ -1,16 +1,21 @@ package types -type Address struct { - IP string - Type string -} +import "net" + +type CloudProvider string + +const ( + CloudProviderGCP CloudProvider = "gcp" + CloudProviderAWS CloudProvider = "aws" + CloudProviderAzure CloudProvider = "azure" +) type Node struct { - Name string - Cluster string - Pool string - Spot bool - Region string - Zone string - Addresses []string + Name string + Cloud CloudProvider + Pool string + Region string + Zone string + ExternalIPs []net.IP + InternalIPs []net.IP } From d6b101d33d9d2afba630e9851f7b72f1029aead0 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Tue, 26 Sep 2023 09:08:56 +0300 Subject: [PATCH 13/66] explore k8s node --- internal/node/explorer.go | 144 +++++++++++++++++ internal/node/explorer_test.go | 286 +++++++++++++++++++++++++++++++++ 2 files changed, 430 insertions(+) create mode 100644 internal/node/explorer.go create mode 100644 internal/node/explorer_test.go diff --git a/internal/node/explorer.go b/internal/node/explorer.go new file mode 100644 index 0000000..92c4dbf --- /dev/null +++ b/internal/node/explorer.go @@ -0,0 +1,144 @@ +package node + +import ( + "context" + "net" + "os" + "strings" + + "github.com/doitintl/kubeip/internal/types" + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +const ( + podInfoDir = "/etc/podinfo/" +) + +type Explorer interface { + GetNode(ctx context.Context) (*types.Node, error) +} + +type explorer struct { + client kubernetes.Interface +} + +func getNodeName(file string) (string, error) { + // get node name from file + nodeName, err := os.ReadFile(file) + if err != nil { + return "", errors.Wrapf(err, "failed to read %s", file) + } + return string(nodeName), nil +} + +func NewExplorer(client kubernetes.Interface) Explorer { + return &explorer{ + client: client, + } +} + +func getCloudProvider(providerID string) (types.CloudProvider, error) { + if strings.HasPrefix(providerID, "aws://") { + return types.CloudProviderAWS, nil + } + if strings.HasPrefix(providerID, "azure://") { + return types.CloudProviderAzure, nil + } + if strings.HasPrefix(providerID, "gce://") { + return types.CloudProviderGCP, nil + } + return "", errors.Errorf("unsupported cloud provider: %s", providerID) +} + +func getNodePool(providerID types.CloudProvider, labels map[string]string) (string, error) { + var ok bool + var pool string + if providerID == types.CloudProviderAWS { + pool, ok = labels["eks.amazonaws.com/nodegroup"] + } else if providerID == types.CloudProviderAzure { + pool, ok = labels["node.kubernetes.io/instancegroup"] + } else if providerID == types.CloudProviderGCP { + pool, ok = labels["cloud.google.com/gke-nodepool"] + } else { + return "", errors.Errorf("unsupported cloud provider: %s", providerID) + } + if !ok { + return "", errors.Errorf("failed to get node pool") + } + return pool, nil +} + +func getAddresses(addresses []v1.NodeAddress) ([]net.IP, []net.IP, error) { + var externalIPs []net.IP + var internalIPs []net.IP + for _, address := range addresses { + ip := net.ParseIP(address.Address) + if ip == nil { + return nil, nil, errors.Errorf("failed to parse IP address: %s", address.Address) + } + if address.Type == v1.NodeExternalIP { + externalIPs = append(externalIPs, ip) + } else if address.Type == v1.NodeInternalIP { + internalIPs = append(internalIPs, ip) + } + } + return externalIPs, internalIPs, nil +} + +// GetNode returns the node object +func (d *explorer) GetNode(ctx context.Context) (*types.Node, error) { + // get node name from downward API + nodeName, err := getNodeName(podInfoDir + "nodeName") + if err != nil { + return nil, errors.Wrap(err, "failed to get node name from downward API") + } + + // get node object from API server + n, err := d.client.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil { + return nil, errors.Wrap(err, "failed to get kubernetes node") + } + + // get cloud provider from node spec + cloudProvider, err := getCloudProvider(n.Spec.ProviderID) + if err != nil { + return nil, errors.Wrap(err, "failed to get cloud provider") + } + + // get node region from node labels + region, ok := n.Labels["topology.kubernetes.io/region"] + if !ok { + return nil, errors.Errorf("failed to get node region") + } + + // get node zone from node labels + zone, ok := n.Labels["topology.kubernetes.io/zone"] + if !ok { + return nil, errors.Errorf("failed to get node zone") + } + + // get node pool from node labels + pool, err := getNodePool(cloudProvider, n.Labels) + if err != nil { + return nil, errors.Wrap(err, "failed to get node pool") + } + + // get node addresses + externalIPs, internalIPs, err := getAddresses(n.Status.Addresses) + if err != nil { + return nil, errors.Wrap(err, "failed to get node addresses") + } + + return &types.Node{ + Name: nodeName, + Cloud: cloudProvider, + Region: region, + Zone: zone, + Pool: pool, + ExternalIPs: externalIPs, + InternalIPs: internalIPs, + }, nil +} diff --git a/internal/node/explorer_test.go b/internal/node/explorer_test.go new file mode 100644 index 0000000..67bb5c8 --- /dev/null +++ b/internal/node/explorer_test.go @@ -0,0 +1,286 @@ +package node + +import ( + "net" + "os" + "reflect" + "testing" + + "github.com/doitintl/kubeip/internal/types" + v1 "k8s.io/api/core/v1" +) + +func Test_getNodeName(t *testing.T) { + tests := []struct { + name string + nodeName string + tearUp func(name string) (*os.File, error) + tearDown func(file string) error + want string + wantErr bool + }{ + { + name: "get node name from .run/podinfo/nodeName file", + nodeName: "test-node", + tearUp: func(name string) (*os.File, error) { + // Setup: create a temporary file and write some data to it + tmpfile, err := os.CreateTemp("", "nodeName") + if err != nil { + return nil, err + } + if _, err = tmpfile.Write([]byte(name)); err != nil { + return nil, err + } + if err = tmpfile.Close(); err != nil { + return nil, err + } + return tmpfile, nil + }, + tearDown: func(file string) error { + return os.Remove(file) + }, + want: "test-node", + }, + { + name: "no such file or directory", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fileName := "" + if tt.tearUp != nil { + f, err := tt.tearUp(tt.nodeName) + if err != nil { + t.Fatal(err) + } + fileName = f.Name() + defer func() { + if tt.tearDown != nil { + err = tt.tearDown(fileName) + if err != nil { + t.Fatal(err) + } + } + }() + } + got, err := getNodeName(fileName) + if (err != nil) != tt.wantErr { + t.Errorf("getNodeName() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("getNodeName() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getCloudProvider(t *testing.T) { + type args struct { + providerID string + } + tests := []struct { + name string + args args + want types.CloudProvider + wantErr bool + }{ + { + name: "aws", + args: args{ + providerID: "aws:///us-west-2b/i-06d71a5ffc05cc325", + }, + want: types.CloudProviderAWS, + }, + { + name: "azure", + args: args{ + providerID: "azure:///subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/aks-agentpool-12345678-vmss_0", + }, + want: types.CloudProviderAzure, + }, + { + name: "gcp", + args: args{ + providerID: "gce:///projects/123456789012/zones/us-west1-b/instances/gke-cluster-1-default-pool-12345678-0v0v", + }, + want: types.CloudProviderGCP, + }, + { + name: "unsupported", + args: args{ + providerID: "unsupported", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getCloudProvider(tt.args.providerID) + if (err != nil) != tt.wantErr { + t.Errorf("getCloudProvider() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("getCloudProvider() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getNodePool(t *testing.T) { + type args struct { + providerID types.CloudProvider + labels map[string]string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "aws", + args: args{ + providerID: types.CloudProviderAWS, + labels: map[string]string{ + "eks.amazonaws.com/nodegroup": "test-node-pool", + "beta.kubernetes.io/os": "linux", + }, + }, + want: "test-node-pool", + }, + { + name: "azure", + args: args{ + providerID: types.CloudProviderAzure, + labels: map[string]string{ + "node.kubernetes.io/instancegroup": "test-node-pool", + "beta.kubernetes.io/os": "linux", + }, + }, + want: "test-node-pool", + }, + { + name: "gcp", + args: args{ + providerID: types.CloudProviderGCP, + labels: map[string]string{ + "cloud.google.com/gke-nodepool": "test-node-pool", + "beta.kubernetes.io/os": "linux", + }, + }, + want: "test-node-pool", + }, + { + name: "unsupported", + args: args{ + providerID: "unsupported", + labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + }, + }, + wantErr: true, + }, + { + name: "no node pool", + args: args{ + providerID: types.CloudProviderAWS, + labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + }, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getNodePool(tt.args.providerID, tt.args.labels) + if (err != nil) != tt.wantErr { + t.Errorf("getNodePool() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("getNodePool() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getAddresses(t *testing.T) { + type args struct { + addresses []v1.NodeAddress + } + tests := []struct { + name string + args args + want []net.IP + want1 []net.IP + wantErr bool + }{ + { + name: "external and internal IPs", + args: args{ + addresses: []v1.NodeAddress{ + { + Type: v1.NodeExternalIP, + Address: "132.64.12.125", + }, + { + Type: v1.NodeInternalIP, + Address: "10.10.0.1", + }, + }, + }, + want: []net.IP{ + net.ParseIP("132.64.12.125"), + }, + want1: []net.IP{ + net.ParseIP("10.10.0.1"), + }, + }, + { + name: "no external IPs", + args: args{ + addresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.0.0.1"}, + }, + }, + want1: []net.IP{net.ParseIP("10.0.0.1")}, + }, + { + name: "no internal IPs", + args: args{ + addresses: []v1.NodeAddress{ + {Type: v1.NodeExternalIP, Address: "132.10.10.1"}, + }, + }, + want: []net.IP{net.ParseIP("132.10.10.1")}, + }, + { + name: "invalid IP address", + args: args{ + addresses: []v1.NodeAddress{ + {Type: v1.NodeExternalIP, Address: "invalid"}, + }, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1, err := getAddresses(tt.args.addresses) + if (err != nil) != tt.wantErr { + t.Errorf("getAddresses() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("getAddresses() got = %v, want %v", got, tt.want) + } + if !reflect.DeepEqual(got1, tt.want1) { + t.Errorf("getAddresses() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} From f1066c1d8a3af813ca760d452a338adbd2e1064b Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Tue, 26 Sep 2023 10:37:20 +0300 Subject: [PATCH 14/66] pass or discover node name --- cmd/main.go | 4 ++-- internal/config/config.go | 6 +++--- internal/node/explorer.go | 15 +++++++++------ 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 3fdc1d0..5161875 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -97,7 +97,7 @@ func run(c context.Context, log *logrus.Entry, cfg config.Config) error { } explorer := node.NewExplorer(clientset) - n, err := explorer.GetNode(ctx) + n, err := explorer.GetNode(ctx, cfg.NodeName) if err != nil { return errors.Wrap(err, "getting node") } @@ -131,7 +131,7 @@ func main() { &cli.StringFlag{ Name: "node-name", Usage: "Kubernetes node name (not needed if running in node)", - EnvVars: []string{"CLUSTER_NAME"}, + EnvVars: []string{"NODE_NAME"}, Category: "Configuration", }, &cli.PathFlag{ diff --git a/internal/config/config.go b/internal/config/config.go index 9b055b0..322551a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -9,8 +9,8 @@ import ( type Config struct { // KubeConfigPath is the path to the kubeconfig file KubeConfigPath string `json:"kubeconfig"` - // ClusterName is the name of the EKS node - ClusterName string `json:"node-name"` + // NodeName is the name of the EKS node + NodeName string `json:"node-name"` // DevelopMode mode DevelopMode bool `json:"develop-mode"` // Retry interval @@ -22,7 +22,7 @@ type Config struct { func LoadConfig(c *cli.Context) Config { var cfg Config cfg.KubeConfigPath = c.String("kubeconfig") - cfg.ClusterName = c.String("node-name") + cfg.NodeName = c.String("node-name") cfg.DevelopMode = c.Bool("develop-mode") cfg.RetryInterval = c.Duration("retry-interval") return cfg diff --git a/internal/node/explorer.go b/internal/node/explorer.go index 92c4dbf..b55b08f 100644 --- a/internal/node/explorer.go +++ b/internal/node/explorer.go @@ -18,7 +18,7 @@ const ( ) type Explorer interface { - GetNode(ctx context.Context) (*types.Node, error) + GetNode(ctx context.Context, nodeName string) (*types.Node, error) } type explorer struct { @@ -89,11 +89,14 @@ func getAddresses(addresses []v1.NodeAddress) ([]net.IP, []net.IP, error) { } // GetNode returns the node object -func (d *explorer) GetNode(ctx context.Context) (*types.Node, error) { - // get node name from downward API - nodeName, err := getNodeName(podInfoDir + "nodeName") - if err != nil { - return nil, errors.Wrap(err, "failed to get node name from downward API") +func (d *explorer) GetNode(ctx context.Context, nodeName string) (*types.Node, error) { + // get node name from downward API if nodeName is empty + if nodeName == "" { + var err error + nodeName, err = getNodeName(podInfoDir + "nodeName") + if err != nil { + return nil, errors.Wrap(err, "failed to get node name from downward API") + } } // get node object from API server From d7bbc17147c6ca7cb207878cacb0289c8a04eb9a Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Tue, 26 Sep 2023 12:21:17 +0300 Subject: [PATCH 15/66] test with fake k8s client --- go.mod | 3 +- go.sum | 2 + internal/node/explorer_test.go | 119 +++++++++++++++++++++++++++++++++ 3 files changed, 123 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 597e053..d4487a0 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.3 github.com/urfave/cli/v2 v2.25.7 + k8s.io/api v0.28.1 k8s.io/apimachinery v0.28.1 k8s.io/client-go v0.28.1 sigs.k8s.io/controller-runtime v0.16.2 @@ -15,6 +16,7 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect + github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect @@ -47,7 +49,6 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/api v0.28.1 // indirect k8s.io/klog/v2 v2.100.1 // indirect k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect diff --git a/go.sum b/go.sum index 0aec49c..945a425 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= +github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= diff --git a/internal/node/explorer_test.go b/internal/node/explorer_test.go index 67bb5c8..d7a0067 100644 --- a/internal/node/explorer_test.go +++ b/internal/node/explorer_test.go @@ -1,6 +1,7 @@ package node import ( + "context" "net" "os" "reflect" @@ -8,6 +9,9 @@ import ( "github.com/doitintl/kubeip/internal/types" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" ) func Test_getNodeName(t *testing.T) { @@ -284,3 +288,118 @@ func Test_getAddresses(t *testing.T) { }) } } + +func Test_explorer_GetNode(t *testing.T) { + type fields struct { + client kubernetes.Interface + } + type args struct { + nodeName string + } + tests := []struct { + name string + fields fields + args args + want *types.Node + wantErr bool + }{ + { + name: "get node", + fields: fields{ + client: fake.NewSimpleClientset(&v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + Labels: map[string]string{ + "eks.amazonaws.com/nodegroup": "test-node-pool", + "beta.kubernetes.io/os": "linux", + "topology.kubernetes.io/region": "us-west-2", + "topology.kubernetes.io/zone": "us-west-2b", + }, + }, + Spec: v1.NodeSpec{ + ProviderID: "aws:///us-west-2b/i-06d71a5ffc05cc325", + }, + Status: v1.NodeStatus{ + Addresses: []v1.NodeAddress{ + {Type: v1.NodeExternalIP, Address: "132.10.10.1"}, + {Type: v1.NodeInternalIP, Address: "10.10.0.1"}, + }, + }, + }), + }, + args: args{ + nodeName: "test-node", + }, + want: &types.Node{ + Name: "test-node", + Cloud: types.CloudProviderAWS, + Pool: "test-node-pool", + Region: "us-west-2", + Zone: "us-west-2b", + ExternalIPs: []net.IP{ + net.ParseIP("132.10.10.1"), + }, + InternalIPs: []net.IP{ + net.ParseIP("10.10.0.1"), + }, + }, + }, + { + name: "failed to get region", + fields: fields{ + client: fake.NewSimpleClientset(&v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + Labels: map[string]string{ + "eks.amazonaws.com/nodegroup": "test-node-pool", + "beta.kubernetes.io/os": "linux", + }, + }, + Spec: v1.NodeSpec{ + ProviderID: "aws:///us-west-2b/i-06d71a5ffc05cc325", + }, + }), + }, + args: args{ + nodeName: "test-node", + }, + wantErr: true, + }, + { + name: "failed to get zone", + fields: fields{ + client: fake.NewSimpleClientset(&v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + Labels: map[string]string{ + "eks.amazonaws.com/nodegroup": "test-node-pool", + "beta.kubernetes.io/os": "linux", + }, + }, + Spec: v1.NodeSpec{ + ProviderID: "aws:///us-west-2b/i-06d71a5ffc05cc325", + }, + }), + }, + args: args{ + nodeName: "test-node", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := &explorer{ + client: tt.fields.client, + } + got, err := d.GetNode(context.Background(), tt.args.nodeName) + if (err != nil) != tt.wantErr { + t.Errorf("GetNode() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetNode() got = %v, want %v", got, tt.want) + } + }) + } +} From 36cf72dbb3498929a3fe5779cc6eb54b1994cb2a Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Tue, 26 Sep 2023 12:26:22 +0300 Subject: [PATCH 16/66] multi-arch build speedup --- Dockerfile | 6 ++++-- makefile | 8 ++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index f59fd02..46f24d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21-alpine AS builder +FROM --platform=$BUILDPLATFORM golang:1.21-alpine AS builder # add CA certificates and TZ for local time RUN apk --update add ca-certificates tzdata make git @@ -19,9 +19,11 @@ COPY . ./ ARG VERSION ARG COMMIT ARG BRANCH +ARG TARGETOS +ARG TARGETARCH # Build the binary with make (using the version, commit and branch) -RUN make build VERSION=${VERSION} COMMIT=${COMMIT} BRANCH=${BRANCH} +RUN make build VERSION=${VERSION} COMMIT=${COMMIT} BRANCH=${BRANCH} TARGETOS=${TARGETOS} TARGETARCH=${TARGETARCH} # final image FROM scratch diff --git a/makefile b/makefile index 7b6673a..799adc6 100644 --- a/makefile +++ b/makefile @@ -10,7 +10,9 @@ GOLINT=golangci-lint LINT_CONFIG = $(CURDIR)/.golangci.yaml BIN=$(CURDIR)/.bin -BINARY_NAME=kubeip +BINARY_NAME=kubeip-agent +TARGETOS := $(or $(TARGETOS), linux) +TARGETARCH := $(or $(TARGETARCH), amd64) DATE ?= $(shell date +%FT%T%z) @@ -27,6 +29,8 @@ Q = $(if $(filter 1,$V),,@) M = $(shell printf "\033[34;1m▶\033[0m") export CGO_ENABLED=0 +export GOOS=$(TARGETOS) +export GOARCH=$(TARGETARCH) # main task all: lint test build ; $(info $(M) build, test and deploy ...) @ ## release cycle @@ -37,7 +41,7 @@ setup-lint: # Tasks -build: ; $(info $(M) building binary ...) @ ## build with local Go SDK +build: ; $(info $(M) building $(GOOS)/$(GOARCH) binary...) @ ## build with local Go SDK $(GOBUILD) -v \ -ldflags '-X main.version=$(VERSION) -X main.buildDate=$(DATE) -X main.gitCommit=$(COMMIT) -X main.gitBranch=$(BRANCH)' \ -o $(BIN)/$(BINARY_NAME) ./cmd/. From 85df4ee6a5f529952ce72fe3c231349e0facd508 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Tue, 26 Sep 2023 14:33:35 +0300 Subject: [PATCH 17/66] skip non ip addresses --- cmd/main.go | 4 ++-- internal/node/explorer.go | 3 +++ internal/node/explorer_test.go | 11 +++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 5161875..daca400 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -25,7 +25,7 @@ const ( ) var ( - version string + version = "dev" buildDate string gitCommit string gitBranch string @@ -102,7 +102,7 @@ func run(c context.Context, log *logrus.Entry, cfg config.Config) error { return errors.Wrap(err, "getting node") } - log.Debug("node name: ", n.Name) + log.WithField("node", n).Debug("node details") <-ctx.Done() log.Infof("kubeip agent stopped") diff --git a/internal/node/explorer.go b/internal/node/explorer.go index b55b08f..38765af 100644 --- a/internal/node/explorer.go +++ b/internal/node/explorer.go @@ -75,6 +75,9 @@ func getAddresses(addresses []v1.NodeAddress) ([]net.IP, []net.IP, error) { var externalIPs []net.IP var internalIPs []net.IP for _, address := range addresses { + if address.Type != v1.NodeExternalIP && address.Type != v1.NodeInternalIP { + continue + } ip := net.ParseIP(address.Address) if ip == nil { return nil, nil, errors.Errorf("failed to parse IP address: %s", address.Address) diff --git a/internal/node/explorer_test.go b/internal/node/explorer_test.go index d7a0067..535e060 100644 --- a/internal/node/explorer_test.go +++ b/internal/node/explorer_test.go @@ -271,6 +271,17 @@ func Test_getAddresses(t *testing.T) { }, wantErr: true, }, + { + name: "skip unsupported IP type", + args: args{ + addresses: []v1.NodeAddress{ + {Type: v1.NodeHostName, Address: "test-node"}, + {Type: v1.NodeInternalDNS, Address: "test-node-internal"}, + {Type: v1.NodeExternalIP, Address: "132.10.10.1"}, + }, + }, + want: []net.IP{net.ParseIP("132.10.10.1")}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From fbc62a0eae8ad35a2b05dcacc3b30176bbf186ff Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Tue, 26 Sep 2023 15:55:40 +0300 Subject: [PATCH 18/66] get instance id from providerID --- internal/node/explorer.go | 15 ++++++++ internal/node/explorer_test.go | 64 +++++++++++++++++++++++++++++++--- internal/types/node.go | 1 + 3 files changed, 75 insertions(+), 5 deletions(-) diff --git a/internal/node/explorer.go b/internal/node/explorer.go index 38765af..723c8d2 100644 --- a/internal/node/explorer.go +++ b/internal/node/explorer.go @@ -53,6 +53,14 @@ func getCloudProvider(providerID string) (types.CloudProvider, error) { return "", errors.Errorf("unsupported cloud provider: %s", providerID) } +func getInstance(providerID string) (string, error) { + s := strings.Split(providerID, "/") + if len(s) < 2 { + return "", errors.Errorf("failed to get instance ID") + } + return s[len(s)-1], nil +} + func getNodePool(providerID types.CloudProvider, labels map[string]string) (string, error) { var ok bool var pool string @@ -114,6 +122,12 @@ func (d *explorer) GetNode(ctx context.Context, nodeName string) (*types.Node, e return nil, errors.Wrap(err, "failed to get cloud provider") } + // get instance ID from provider ID + instance, err := getInstance(n.Spec.ProviderID) + if err != nil { + return nil, errors.Wrap(err, "failed to get instance ID") + } + // get node region from node labels region, ok := n.Labels["topology.kubernetes.io/region"] if !ok { @@ -140,6 +154,7 @@ func (d *explorer) GetNode(ctx context.Context, nodeName string) (*types.Node, e return &types.Node{ Name: nodeName, + Instance: instance, Cloud: cloudProvider, Region: region, Zone: zone, diff --git a/internal/node/explorer_test.go b/internal/node/explorer_test.go index 535e060..89a1a9b 100644 --- a/internal/node/explorer_test.go +++ b/internal/node/explorer_test.go @@ -342,11 +342,12 @@ func Test_explorer_GetNode(t *testing.T) { nodeName: "test-node", }, want: &types.Node{ - Name: "test-node", - Cloud: types.CloudProviderAWS, - Pool: "test-node-pool", - Region: "us-west-2", - Zone: "us-west-2b", + Name: "test-node", + Instance: "i-06d71a5ffc05cc325", + Cloud: types.CloudProviderAWS, + Pool: "test-node-pool", + Region: "us-west-2", + Zone: "us-west-2b", ExternalIPs: []net.IP{ net.ParseIP("132.10.10.1"), }, @@ -414,3 +415,56 @@ func Test_explorer_GetNode(t *testing.T) { }) } } + +func Test_getInstance(t *testing.T) { + type args struct { + providerID string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "aws", + args: args{ + providerID: "aws:///us-west-2b/i-06d71a5ffc05cc325", + }, + want: "i-06d71a5ffc05cc325", + }, + { + name: "azure", + args: args{ + providerID: "azure:///subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/aks-agentpool-12345678-vmss_0", + }, + want: "aks-agentpool-12345678-vmss_0", + }, + { + name: "gcp", + args: args{ + providerID: "gce:///projects/123456789012/zones/us-west1-b/instances/gke-cluster-1-default-pool-12345678-0v0v", + }, + want: "gke-cluster-1-default-pool-12345678-0v0v", + }, + { + name: "unsupported", + args: args{ + providerID: "unsupported", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getInstance(tt.args.providerID) + if (err != nil) != tt.wantErr { + t.Errorf("getInstance() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("getInstance() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/types/node.go b/internal/types/node.go index 654adaf..0632d06 100644 --- a/internal/types/node.go +++ b/internal/types/node.go @@ -12,6 +12,7 @@ const ( type Node struct { Name string + Instance string Cloud CloudProvider Pool string Region string From e1b1f210e4f1aefb1bb0136467ca26135e897848 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Sun, 8 Oct 2023 20:40:19 +0300 Subject: [PATCH 19/66] draft gcp implementation completed --- cmd/main.go | 61 ++++++++- go.mod | 17 ++- go.sum | 107 +++++++++++++++- internal/address/assigner.go | 31 +++++ internal/address/aws.go | 8 ++ internal/address/azure.go | 8 ++ internal/address/gcp.go | 231 +++++++++++++++++++++++++++++++++++ internal/config/config.go | 19 ++- 8 files changed, 467 insertions(+), 15 deletions(-) create mode 100644 internal/address/assigner.go create mode 100644 internal/address/aws.go create mode 100644 internal/address/azure.go create mode 100644 internal/address/gcp.go diff --git a/cmd/main.go b/cmd/main.go index daca400..8761a54 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -7,6 +7,7 @@ import ( "runtime" "time" + "github.com/doitintl/kubeip/internal/address" "github.com/doitintl/kubeip/internal/config" "github.com/doitintl/kubeip/internal/node" "github.com/pkg/errors" @@ -76,7 +77,7 @@ func prepareLogger(level string, json bool) *logrus.Entry { return log } -func run(c context.Context, log *logrus.Entry, cfg config.Config) error { +func run(c context.Context, log *logrus.Entry, cfg *config.Config) error { ctx, cancel := context.WithCancel(c) defer cancel() // add debug mode to context @@ -102,7 +103,37 @@ func run(c context.Context, log *logrus.Entry, cfg config.Config) error { return errors.Wrap(err, "getting node") } - log.WithField("node", n).Debug("node details") + // assign static public IP address with retry (interval and attempts) + assigner, err := address.NewAssigner(ctx, log, n.Cloud, cfg) + if err != nil { + return errors.Wrap(err, "initializing assigner") + } + // retry counter + retryCounter := 0 + // ticker for retry interval + ticker := time.NewTicker(cfg.RetryInterval) + defer ticker.Stop() + for { + err = assigner.Assign(n.Instance, n.Zone, cfg.Filter, cfg.OrderBy) + if err != nil { + log.WithError(err).Errorf("failed to assign static public IP address to node %s", n.Name) + if retryCounter < cfg.RetryAttempts { + retryCounter++ + log.Infof("retrying after %v", cfg.RetryInterval) + } else { + log.Infof("reached maximum number of retries (%d)", cfg.RetryAttempts) + return errors.Wrap(err, "reached maximum number of retries") + } + select { + case <-ticker.C: + continue + case <-ctx.Done(): + log.Infof("kubeip agent stopped") + return nil + } + } + break + } <-ctx.Done() log.Infof("kubeip agent stopped") @@ -134,6 +165,18 @@ func main() { EnvVars: []string{"NODE_NAME"}, Category: "Configuration", }, + &cli.StringFlag{ + Name: "project", + Usage: "name of the GCP project or the AWS account ID (not needed if running in node)", + EnvVars: []string{"PROJECT"}, + Category: "Configuration", + }, + &cli.StringFlag{ + Name: "region", + Usage: "name of the GCP region or the AWS region (not needed if running in node)", + EnvVars: []string{"REGION"}, + Category: "Configuration", + }, &cli.PathFlag{ Name: "kubeconfig", Usage: "path to Kubernetes configuration file (not needed if running in node)", @@ -147,6 +190,18 @@ func main() { EnvVars: []string{"RETRY_INTERVAL"}, Category: "Configuration", }, + &cli.StringSliceFlag{ + Name: "filter", + Usage: "filter for the IP addresses", + EnvVars: []string{"FILTER"}, + Category: "Configuration", + }, + &cli.StringFlag{ + Name: "order-by", + Usage: "order by for the IP addresses", + EnvVars: []string{"ORDER_BY"}, + Category: "Configuration", + }, &cli.IntFlag{ Name: "retry-attempts", Usage: "number of attempts to assign the static public IP address", @@ -213,7 +268,7 @@ func kubeConfigFromPath(kubepath string) (*rest.Config, error) { return cfg, nil } -func retrieveKubeConfig(log logrus.FieldLogger, cfg config.Config) (*rest.Config, error) { +func retrieveKubeConfig(log logrus.FieldLogger, cfg *config.Config) (*rest.Config, error) { kubeconfig, err := kubeConfigFromPath(cfg.KubeConfigPath) if err != nil && !errors.Is(err, errEmptyPath) { return nil, errors.Wrap(err, "retrieving kube config from path") diff --git a/go.mod b/go.mod index d4487a0..707ed36 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,8 @@ require ( github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.3 github.com/urfave/cli/v2 v2.25.7 + golang.org/x/oauth2 v0.12.0 + google.golang.org/api v0.143.0 k8s.io/api v0.28.1 k8s.io/apimachinery v0.28.1 k8s.io/client-go v0.28.1 @@ -13,6 +15,8 @@ require ( ) require ( + cloud.google.com/go/compute v1.23.0 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect @@ -22,11 +26,15 @@ require ( github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/google/uuid v1.3.1 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.1 // indirect + github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -38,14 +46,17 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/testify v1.8.3 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect + go.opencensus.io v0.24.0 // indirect + golang.org/x/crypto v0.13.0 // indirect golang.org/x/net v0.15.0 // indirect - golang.org/x/oauth2 v0.8.0 // indirect golang.org/x/sys v0.12.0 // indirect golang.org/x/term v0.12.0 // indirect golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 // indirect + google.golang.org/grpc v1.57.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 945a425..7cfb5f1 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,12 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= +cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -6,6 +15,10 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -21,12 +34,32 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEe github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -35,8 +68,15 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.1 h1:SBWmZhjUDRorQxrN0nwzf+AHBxnbFjViHQS4P0yVpmQ= +github.com/googleapis/enterprise-certificate-proxy v0.3.1/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= +github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -69,6 +109,7 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -93,23 +134,42 @@ github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRT github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= -golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= +golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -126,6 +186,10 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -135,12 +199,41 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.143.0 h1:o8cekTkqhywkbZT6p1UHJPZ9+9uuCAJs/KYomxZB8fA= +google.golang.org/api v0.143.0/go.mod h1:FoX9DO9hT7DLNn97OuoZAGSDuNAXdJRuGK98rSUgurk= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb h1:XFBgcDwm7irdHTbz4Zk2h7Mh+eis4nfJEFQFYzJzuIA= +google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= +google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb h1:lK0oleSc7IQsUxO3U5TjL9DWlsxpEBemh+zpB7IqhWI= +google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 h1:N3bU/SQDCDyD6R528GJ/PwW9KjYcJA3dgyH+MovAkIM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:KSqppvjFjtoCI+KGd4PELB0qLNxdJHRGqRI09mB6pQA= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -152,6 +245,8 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.28.1 h1:i+0O8k2NPBCPYaMB+uCkseEbawEt/eFaiRqUx8aB108= k8s.io/api v0.28.1/go.mod h1:uBYwID+66wiL28Kn2tBjBYQdEU0Xk0z5qF8bIBqk/Dg= k8s.io/apimachinery v0.28.1 h1:EJD40og3GizBSV3mkIoXQBsws32okPOy+MkRyzh6nPY= diff --git a/internal/address/assigner.go b/internal/address/assigner.go new file mode 100644 index 0000000..1bce70e --- /dev/null +++ b/internal/address/assigner.go @@ -0,0 +1,31 @@ +package address + +import ( + "context" + + "github.com/doitintl/kubeip/internal/config" + "github.com/doitintl/kubeip/internal/types" + "github.com/sirupsen/logrus" +) + +type Assigner interface { + Assign(instanceID, zone string, filter []string, orderBy string) error +} + +type assigner struct { +} + +func NewAssigner(ctx context.Context, logger *logrus.Entry, provider types.CloudProvider, cfg *config.Config) (Assigner, error) { + if provider == types.CloudProviderAWS { + return &awsAssigner{}, nil + } else if provider == types.CloudProviderAzure { + return &azureAssigner{}, nil + } else if provider == types.CloudProviderGCP { + return NewGCPAssigner(ctx, logger, cfg.Project, cfg.Region) + } + return &assigner{}, nil +} + +func (a *assigner) Assign(_, _ string, _ []string, _ string) error { + return nil +} diff --git a/internal/address/aws.go b/internal/address/aws.go new file mode 100644 index 0000000..a859f05 --- /dev/null +++ b/internal/address/aws.go @@ -0,0 +1,8 @@ +package address + +type awsAssigner struct { +} + +func (a *awsAssigner) Assign(_, _ string, _ []string, _ string) error { + return nil +} diff --git a/internal/address/azure.go b/internal/address/azure.go new file mode 100644 index 0000000..46ed863 --- /dev/null +++ b/internal/address/azure.go @@ -0,0 +1,8 @@ +package address + +type azureAssigner struct { +} + +func (a *azureAssigner) Assign(_, _ string, _ []string, _ string) error { + return nil +} diff --git a/internal/address/gcp.go b/internal/address/gcp.go new file mode 100644 index 0000000..1dc7636 --- /dev/null +++ b/internal/address/gcp.go @@ -0,0 +1,231 @@ +package address + +import ( + "context" + "fmt" + "strings" + "time" + + "cloud.google.com/go/compute/metadata" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "google.golang.org/api/compute/v1" +) + +const ( + operationDone = "DONE" // operation status DONE + inUseStatus = "IN_USE" + reservedStatus = "RESERVED" // static IP addresses that are reserved but not currently in use + defaultTimeout = 10 * time.Minute + defaultNetworkInterface = "nic0" + accessConfigType = "ONE_TO_ONE_NAT" + accessConfigKind = "compute#accessConfig" +) + +type gcpAssigner struct { + client *compute.Service + project string + region string + logger *logrus.Entry +} + +func NewGCPAssigner(ctx context.Context, logger *logrus.Entry, project, region string) (Assigner, error) { + // initialize Google Cloud client + client, err := compute.NewService(ctx) + if err != nil { + return nil, errors.Wrap(err, "failed to initialize Google Cloud client") + } + + // get project ID from metadata server + if project == "" { + project, err = metadata.ProjectID() + if err != nil { + return nil, errors.Wrap(err, "failed to get project ID from metadata server") + } + } + + // get region from metadata server + if region == "" { + region, err = metadata.InstanceAttributeValue("cluster-location") + if err != nil { + return nil, errors.Wrap(err, "failed to get region from metadata server") + } + // if cluster-location is zone, extract region from zone + if len(region) > 3 && region[len(region)-3] == '-' { + region = region[:len(region)-3] + } + } + + return &gcpAssigner{ + client: client, + project: project, + region: region, + logger: logger, + }, nil +} + +func (a *gcpAssigner) waitForOperation(op *compute.Operation, zone string, timeout time.Duration) error { + if op == nil { + a.logger.Warn("operation is nil") + return nil + } + // Create a context that can be cancelled + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Create a timer that cancels the context after timeout + timer := time.AfterFunc(timeout, func() { + cancel() + }) + defer timer.Stop() + var err error + name := op.Name + for op.Status != operationDone { + // Pass the cancellable context to the Wait method + op, err = a.client.ZoneOperations.Wait(a.project, zone, name).Context(ctx).Do() + if err != nil { + // If the context was cancelled, return a timeout error + if errors.Is(err, context.Canceled) { + return errors.New("operation timed out") + } + return errors.Wrapf(err, "failed to get operation %s", name) + } + // If the operation has an error, return it + if op != nil && op.Error != nil { + return errors.Errorf("operation %s failed with error %v", op.Name, op.Error.Errors) + } + } + return nil +} + +func (a *gcpAssigner) deleteInstanceAddress(instance *compute.Instance, zone string) error { + // Check if the instance has at least one network interface + if len(instance.NetworkInterfaces) == 0 { + a.logger.WithField("instance", instance.Name).Info("instance has no network interfaces") + return nil + } + // get instance network interface + networkInterface := instance.NetworkInterfaces[0] + // get instance network interface access config + if len(networkInterface.AccessConfigs) == 0 { + a.logger.WithField("instance", instance.Name).Info("instance network interface has no access configs") + return nil + } + accessConfig := networkInterface.AccessConfigs[0] + // get instance network interface access config name + accessConfigName := accessConfig.Name + // delete instance network interface access config + a.logger.WithFields(logrus.Fields{ + "instance": instance.Name, + "address": accessConfig.NatIP, + }).Infof("deleting ephemeral public IP address from instance") + op, err := a.client.Instances.DeleteAccessConfig(a.project, zone, instance.Name, accessConfigName, networkInterface.Name).Do() + if err != nil { + return errors.Wrapf(err, "failed to delete access config %s from instance %s", accessConfigName, instance.Name) + } + // wait for operation to complete + if err = a.waitForOperation(op, zone, defaultTimeout); err != nil { + return errors.Wrapf(err, "failed to wait for operation %s", op.Name) + } + return nil +} + +func (a *gcpAssigner) addInstanceAddress(instance *compute.Instance, zone string, address *compute.Address) error { + // add instance network interface access config + a.logger.WithFields(logrus.Fields{ + "instance": instance.Name, + "address": address.Address, + }).Infof("adding reserved public IP address to instance") + op, err := a.client.Instances.AddAccessConfig(a.project, zone, instance.Name, defaultNetworkInterface, &compute.AccessConfig{ + Name: address.Name, + Type: accessConfigType, + Kind: accessConfigKind, + NatIP: address.Address, + }).Do() + if err != nil { + return errors.Wrapf(err, "failed to add access config %s to instance %s", address.Name, instance.Name) + } + // wait for operation to complete + if err = a.waitForOperation(op, zone, defaultTimeout); err != nil { + return errors.Wrapf(err, "failed to wait for operation %s", op.Name) + } + return nil +} + +func (a *gcpAssigner) Assign(instanceID, zone string, filter []string, orderBy string) error { + // check if instance already has a public static IP address assigned + instance, err := a.client.Instances.Get(a.project, zone, instanceID).Do() + if err != nil { + return errors.Wrapf(err, "failed to get instance %s", instanceID) + } + assigned, err := a.listAddresses(nil, "", inUseStatus) + if err != nil { + return errors.Wrap(err, "failed to list assigned addresses") + } + if len(assigned) > 0 { + for _, address := range assigned { + for _, user := range address.Users { + if user == instance.SelfLink { + a.logger.WithFields(logrus.Fields{ + "instance": instanceID, + "address": address.Address, + }).Infof("instance already has a static public IP address assigned") + return nil + } + } + } + } + + // get available reserved public IP addresses + addresses, err := a.listAddresses(filter, orderBy, reservedStatus) + if err != nil { + return errors.Wrap(err, "failed to list available addresses") + } + if len(addresses) == 0 { + return errors.Errorf("no available addresses") + } + + // delete current ephemeral public IP address + if err = a.deleteInstanceAddress(instance, zone); err != nil { + return errors.Wrap(err, "failed to delete current public IP address") + } + + // assign first available static public IP address to the instance + address := addresses[0] + if err = a.addInstanceAddress(instance, zone, address); err != nil { + return errors.Wrap(err, "failed to assign static public IP address") + } + + return nil +} + +func (a *gcpAssigner) listAddresses(filter []string, orderBy, status string) ([]*compute.Address, error) { + call := a.client.Addresses.List(a.project, a.region) + var filters []string + // filter public addresses (with status) + filters = append(filters, fmt.Sprintf("(status=%s)", status)) + filters = append(filters, "(addressType=EXTERNAL)") + // filter addresses by provided filter: labels.key=value + for _, f := range filter { + filters = append(filters, fmt.Sprintf("(%s)", f)) + } + // set the filter + call = call.Filter(strings.Join(filters, " ")) + // sort addresses by + if orderBy != "" { + call = call.OrderBy(orderBy) + } + // get all addresses + var addresses []*compute.Address + for { + list, err := call.Do() + if err != nil { + return nil, errors.Wrap(err, "failed to list available addresses") + } + addresses = append(addresses, list.Items...) + if list.NextPageToken == "" { + return addresses, nil + } + call = call.PageToken(list.NextPageToken) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 322551a..7ffdd74 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -9,21 +9,34 @@ import ( type Config struct { // KubeConfigPath is the path to the kubeconfig file KubeConfigPath string `json:"kubeconfig"` - // NodeName is the name of the EKS node + // NodeName is the name of the Kubernetes node NodeName string `json:"node-name"` + // Project is the name of the GCP project or the AWS account ID + Project string `json:"project"` + // Region is the name of the GCP region or the AWS region + Region string `json:"region"` // DevelopMode mode DevelopMode bool `json:"develop-mode"` + // Filter is the filter for the IP addresses + Filter []string `json:"filter"` + // OrderBy is the order by for the IP addresses + OrderBy string `json:"order-by"` // Retry interval RetryInterval time.Duration `json:"retry-interval"` // Retry attempts RetryAttempts int `json:"retry-attempts"` } -func LoadConfig(c *cli.Context) Config { +func LoadConfig(c *cli.Context) *Config { var cfg Config cfg.KubeConfigPath = c.String("kubeconfig") cfg.NodeName = c.String("node-name") cfg.DevelopMode = c.Bool("develop-mode") cfg.RetryInterval = c.Duration("retry-interval") - return cfg + cfg.RetryAttempts = c.Int("retry-attempts") + cfg.Filter = c.StringSlice("filter") + cfg.OrderBy = c.String("order-by") + cfg.Project = c.String("project") + cfg.Region = c.String("region") + return &cfg } From 875947da88c45b64d1c5a341d82969d985259936 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Mon, 9 Oct 2023 13:46:57 +0300 Subject: [PATCH 20/66] more tests with mocks --- cmd/main.go | 80 ++++++++++++++-------- cmd/main_test.go | 136 ++++++++++++++++++++++++++++++++++++++ go.mod | 8 ++- go.sum | 1 + internal/config/config.go | 2 +- makefile | 6 ++ mocks/address/Assigner.go | 77 +++++++++++++++++++++ mocks/node/Explorer.go | 93 ++++++++++++++++++++++++++ 8 files changed, 370 insertions(+), 33 deletions(-) create mode 100644 cmd/main_test.go create mode 100644 mocks/address/Assigner.go create mode 100644 mocks/node/Explorer.go diff --git a/cmd/main.go b/cmd/main.go index 8761a54..4620822 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -9,7 +9,8 @@ import ( "github.com/doitintl/kubeip/internal/address" "github.com/doitintl/kubeip/internal/config" - "github.com/doitintl/kubeip/internal/node" + nd "github.com/doitintl/kubeip/internal/node" + "github.com/doitintl/kubeip/internal/types" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" @@ -77,6 +78,39 @@ func prepareLogger(level string, json bool) *logrus.Entry { return log } +func assignAddress(c context.Context, log *logrus.Entry, assigner address.Assigner, node *types.Node, cfg *config.Config) error { + ctx, cancel := context.WithCancel(c) + defer cancel() + // retry counter + retryCounter := 0 + // ticker for retry interval + ticker := time.NewTicker(cfg.RetryInterval) + defer ticker.Stop() + + for { + err := assigner.Assign(node.Instance, node.Zone, cfg.Filter, cfg.OrderBy) + if err != nil { + log.WithError(err).Errorf("failed to assign static public IP address to node %s", node.Name) + if retryCounter < cfg.RetryAttempts { + retryCounter++ + log.Infof("retrying after %v", cfg.RetryInterval) + } else { + log.Infof("reached maximum number of retries (%d)", cfg.RetryAttempts) + return errors.Wrap(err, "reached maximum number of retries") + } + select { + case <-ticker.C: + continue + case <-ctx.Done(): + log.Infof("kubeip agent stopped") + return errors.Wrap(err, "context is done") + } + } + break + } + return nil +} + func run(c context.Context, log *logrus.Entry, cfg *config.Config) error { ctx, cancel := context.WithCancel(c) defer cancel() @@ -84,7 +118,6 @@ func run(c context.Context, log *logrus.Entry, cfg *config.Config) error { if cfg.DevelopMode { ctx = context.WithValue(ctx, developModeKey, true) } - log.WithField("develop-mode", cfg.DevelopMode).Infof("kubeip agent started") restconfig, err := retrieveKubeConfig(log, cfg) @@ -97,7 +130,7 @@ func run(c context.Context, log *logrus.Entry, cfg *config.Config) error { return errors.Wrap(err, "initializing kubernetes client") } - explorer := node.NewExplorer(clientset) + explorer := nd.NewExplorer(clientset) n, err := explorer.GetNode(ctx, cfg.NodeName) if err != nil { return errors.Wrap(err, "getting node") @@ -108,42 +141,31 @@ func run(c context.Context, log *logrus.Entry, cfg *config.Config) error { if err != nil { return errors.Wrap(err, "initializing assigner") } - // retry counter - retryCounter := 0 - // ticker for retry interval - ticker := time.NewTicker(cfg.RetryInterval) - defer ticker.Stop() - for { - err = assigner.Assign(n.Instance, n.Zone, cfg.Filter, cfg.OrderBy) + // assign static public IP address + errorCh := make(chan error) + go func() { + e := assignAddress(ctx, log, assigner, n, cfg) + if e != nil { + errorCh <- e + } + }() + + select { + case err = <-errorCh: if err != nil { - log.WithError(err).Errorf("failed to assign static public IP address to node %s", n.Name) - if retryCounter < cfg.RetryAttempts { - retryCounter++ - log.Infof("retrying after %v", cfg.RetryInterval) - } else { - log.Infof("reached maximum number of retries (%d)", cfg.RetryAttempts) - return errors.Wrap(err, "reached maximum number of retries") - } - select { - case <-ticker.C: - continue - case <-ctx.Done(): - log.Infof("kubeip agent stopped") - return nil - } + return errors.Wrap(err, "assigning static public IP address") } - break + case <-ctx.Done(): + log.Infof("kubeip agent stopped") } - <-ctx.Done() - log.Infof("kubeip agent stopped") return nil } func runCmd(c *cli.Context) error { ctx := signals.SetupSignalHandler() log := prepareLogger(c.String("log-level"), c.Bool("json")) - cfg := config.LoadConfig(c) + cfg := config.NewConfig(c) if err := run(ctx, log, cfg); err != nil { log.Fatalf("eks-lens agent failed: %v", err) diff --git a/cmd/main_test.go b/cmd/main_test.go new file mode 100644 index 0000000..113f6eb --- /dev/null +++ b/cmd/main_test.go @@ -0,0 +1,136 @@ +package main + +import ( + "context" + "testing" + "time" + + "github.com/doitintl/kubeip/internal/address" + "github.com/doitintl/kubeip/internal/config" + "github.com/doitintl/kubeip/internal/types" + mocks "github.com/doitintl/kubeip/mocks/address" + "github.com/pkg/errors" +) + +func Test_assignAddress(t *testing.T) { + type args struct { + c context.Context + assignerFn func(t *testing.T) address.Assigner + node *types.Node + cfg *config.Config + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "assign address successfully", + args: args{ + c: context.Background(), + assignerFn: func(t *testing.T) address.Assigner { + mock := mocks.NewAssigner(t) + mock.EXPECT().Assign("test-instance", "test-zone", []string{"test-filter"}, "test-order-by").Return(nil) + return mock + }, + node: &types.Node{ + Name: "test-node", + Instance: "test-instance", + Region: "test-region", + Zone: "test-zone", + }, + cfg: &config.Config{ + Filter: []string{"test-filter"}, + OrderBy: "test-order-by", + RetryAttempts: 3, + RetryInterval: time.Millisecond, + }, + }, + }, + { + name: "assign address after a few retries", + args: args{ + c: context.Background(), + assignerFn: func(t *testing.T) address.Assigner { + mock := mocks.NewAssigner(t) + mock.EXPECT().Assign("test-instance", "test-zone", []string{"test-filter"}, "test-order-by").Return(errors.New("first error")).Once() + mock.EXPECT().Assign("test-instance", "test-zone", []string{"test-filter"}, "test-order-by").Return(errors.New("second error")).Once() + mock.EXPECT().Assign("test-instance", "test-zone", []string{"test-filter"}, "test-order-by").Return(nil).Once() + return mock + }, + node: &types.Node{ + Name: "test-node", + Instance: "test-instance", + Region: "test-region", + Zone: "test-zone", + }, + cfg: &config.Config{ + Filter: []string{"test-filter"}, + OrderBy: "test-order-by", + RetryAttempts: 3, + RetryInterval: time.Millisecond, + }, + }, + }, + { + name: "error after a few retries and reached maximum number of retries", + args: args{ + c: context.Background(), + assignerFn: func(t *testing.T) address.Assigner { + mock := mocks.NewAssigner(t) + mock.EXPECT().Assign("test-instance", "test-zone", []string{"test-filter"}, "test-order-by").Return(errors.New("error")).Times(4) + return mock + }, + node: &types.Node{ + Name: "test-node", + Instance: "test-instance", + Region: "test-region", + Zone: "test-zone", + }, + cfg: &config.Config{ + Filter: []string{"test-filter"}, + OrderBy: "test-order-by", + RetryAttempts: 3, + RetryInterval: time.Millisecond, + }, + }, + wantErr: true, + }, + { + name: "error after a few retries and context is done", + args: args{ + c: func() context.Context { + ctx, _ := context.WithTimeout(context.Background(), 10*time.Millisecond) + return ctx + }(), + assignerFn: func(t *testing.T) address.Assigner { + mock := mocks.NewAssigner(t) + mock.EXPECT().Assign("test-instance", "test-zone", []string{"test-filter"}, "test-order-by").Return(errors.New("error")).Maybe() + return mock + }, + node: &types.Node{ + Name: "test-node", + Instance: "test-instance", + Region: "test-region", + Zone: "test-zone", + }, + cfg: &config.Config{ + Filter: []string{"test-filter"}, + OrderBy: "test-order-by", + RetryAttempts: 3, + RetryInterval: 15 * time.Millisecond, + }, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + log := prepareLogger("debug", false) + assigner := tt.args.assignerFn(t) + if err := assignAddress(tt.args.c, log, assigner, tt.args.node, tt.args.cfg); (err != nil) != tt.wantErr { + t.Errorf("assignAddress() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/go.mod b/go.mod index 707ed36..885f9e0 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,11 @@ module github.com/doitintl/kubeip go 1.21 require ( + cloud.google.com/go/compute/metadata v0.2.3 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.3 + github.com/stretchr/testify v1.8.3 github.com/urfave/cli/v2 v2.25.7 - golang.org/x/oauth2 v0.12.0 google.golang.org/api v0.143.0 k8s.io/api v0.28.1 k8s.io/apimachinery v0.28.1 @@ -16,7 +17,6 @@ require ( require ( cloud.google.com/go/compute v1.23.0 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect @@ -42,13 +42,15 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/testify v1.8.3 // indirect + github.com/stretchr/objx v0.5.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.opencensus.io v0.24.0 // indirect golang.org/x/crypto v0.13.0 // indirect golang.org/x/net v0.15.0 // indirect + golang.org/x/oauth2 v0.12.0 // indirect golang.org/x/sys v0.12.0 // indirect golang.org/x/term v0.12.0 // indirect golang.org/x/text v0.13.0 // indirect diff --git a/go.sum b/go.sum index 7cfb5f1..c7cc230 100644 --- a/go.sum +++ b/go.sum @@ -120,6 +120,7 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= diff --git a/internal/config/config.go b/internal/config/config.go index 7ffdd74..f765cc5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -27,7 +27,7 @@ type Config struct { RetryAttempts int `json:"retry-attempts"` } -func LoadConfig(c *cli.Context) *Config { +func NewConfig(c *cli.Context) *Config { var cfg Config cfg.KubeConfigPath = c.String("kubeconfig") cfg.NodeName = c.String("node-name") diff --git a/makefile b/makefile index 799adc6..8d8d17b 100644 --- a/makefile +++ b/makefile @@ -7,6 +7,7 @@ GOTEST=$(GOCMD) test GOGET=$(GOCMD) get GOTOOL=$(GOCMD) tool GOLINT=golangci-lint +GOMOCK=mockery LINT_CONFIG = $(CURDIR)/.golangci.yaml BIN=$(CURDIR)/.bin @@ -38,6 +39,8 @@ all: lint test build ; $(info $(M) build, test and deploy ...) @ ## release cycl # Tools setup-lint: $(GOCMD) install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.54.2 +setup-mockery: + $(GOCMD) install github.com/vektra/mockery/v2@v2.35.2 # Tasks @@ -51,6 +54,9 @@ lint: setup-lint; $(info $(M) running golangci-lint ...) @ ## run golangci-lint # conflict when multiple go versions are installed $Q $(GOLINT) run -v -c $(LINT_CONFIG) --out-format checkstyle ./... > golangci-lint.out +mock: setup-mockery ; $(info $(M) running mockery ...) @ ## run mockery to generate mocks + $Q $(GOMOCK) --dir internal --all --keeptree --with-expecter + test: ; $(info $(M) running test ...) @ ## run tests with coverage $Q $(GOCMD) fmt ./... $Q $(GOTEST) -v -cover ./... -coverprofile=coverage.out diff --git a/mocks/address/Assigner.go b/mocks/address/Assigner.go new file mode 100644 index 0000000..bddc658 --- /dev/null +++ b/mocks/address/Assigner.go @@ -0,0 +1,77 @@ +// Code generated by mockery v2.30.16. DO NOT EDIT. + +package mocks + +import mock "github.com/stretchr/testify/mock" + +// Assigner is an autogenerated mock type for the Assigner type +type Assigner struct { + mock.Mock +} + +type Assigner_Expecter struct { + mock *mock.Mock +} + +func (_m *Assigner) EXPECT() *Assigner_Expecter { + return &Assigner_Expecter{mock: &_m.Mock} +} + +// Assign provides a mock function with given fields: instanceID, zone, filter, orderBy +func (_m *Assigner) Assign(instanceID string, zone string, filter []string, orderBy string) error { + ret := _m.Called(instanceID, zone, filter, orderBy) + + var r0 error + if rf, ok := ret.Get(0).(func(string, string, []string, string) error); ok { + r0 = rf(instanceID, zone, filter, orderBy) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Assigner_Assign_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Assign' +type Assigner_Assign_Call struct { + *mock.Call +} + +// Assign is a helper method to define mock.On call +// - instanceID string +// - zone string +// - filter []string +// - orderBy string +func (_e *Assigner_Expecter) Assign(instanceID interface{}, zone interface{}, filter interface{}, orderBy interface{}) *Assigner_Assign_Call { + return &Assigner_Assign_Call{Call: _e.mock.On("Assign", instanceID, zone, filter, orderBy)} +} + +func (_c *Assigner_Assign_Call) Run(run func(instanceID string, zone string, filter []string, orderBy string)) *Assigner_Assign_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string), args[1].(string), args[2].([]string), args[3].(string)) + }) + return _c +} + +func (_c *Assigner_Assign_Call) Return(_a0 error) *Assigner_Assign_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Assigner_Assign_Call) RunAndReturn(run func(string, string, []string, string) error) *Assigner_Assign_Call { + _c.Call.Return(run) + return _c +} + +// NewAssigner creates a new instance of Assigner. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewAssigner(t interface { + mock.TestingT + Cleanup(func()) +}) *Assigner { + mock := &Assigner{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/node/Explorer.go b/mocks/node/Explorer.go new file mode 100644 index 0000000..85c961f --- /dev/null +++ b/mocks/node/Explorer.go @@ -0,0 +1,93 @@ +// Code generated by mockery v2.30.16. DO NOT EDIT. + +package mocks + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" + + types "github.com/doitintl/kubeip/internal/types" +) + +// Explorer is an autogenerated mock type for the Explorer type +type Explorer struct { + mock.Mock +} + +type Explorer_Expecter struct { + mock *mock.Mock +} + +func (_m *Explorer) EXPECT() *Explorer_Expecter { + return &Explorer_Expecter{mock: &_m.Mock} +} + +// GetNode provides a mock function with given fields: ctx, nodeName +func (_m *Explorer) GetNode(ctx context.Context, nodeName string) (*types.Node, error) { + ret := _m.Called(ctx, nodeName) + + var r0 *types.Node + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) (*types.Node, error)); ok { + return rf(ctx, nodeName) + } + if rf, ok := ret.Get(0).(func(context.Context, string) *types.Node); ok { + r0 = rf(ctx, nodeName) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.Node) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, nodeName) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Explorer_GetNode_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetNode' +type Explorer_GetNode_Call struct { + *mock.Call +} + +// GetNode is a helper method to define mock.On call +// - ctx context.Context +// - nodeName string +func (_e *Explorer_Expecter) GetNode(ctx interface{}, nodeName interface{}) *Explorer_GetNode_Call { + return &Explorer_GetNode_Call{Call: _e.mock.On("GetNode", ctx, nodeName)} +} + +func (_c *Explorer_GetNode_Call) Run(run func(ctx context.Context, nodeName string)) *Explorer_GetNode_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *Explorer_GetNode_Call) Return(_a0 *types.Node, _a1 error) *Explorer_GetNode_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Explorer_GetNode_Call) RunAndReturn(run func(context.Context, string) (*types.Node, error)) *Explorer_GetNode_Call { + _c.Call.Return(run) + return _c +} + +// NewExplorer creates a new instance of Explorer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewExplorer(t interface { + mock.TestingT + Cleanup(func()) +}) *Explorer { + mock := &Explorer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} From ee062c1e505f25158ca60a18e81675d8e365af03 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Mon, 9 Oct 2023 19:18:35 +0300 Subject: [PATCH 21/66] mock GCP list and test --- internal/address/gcp.go | 15 ++- internal/address/gcp_test.go | 81 +++++++++++++ internal/cloud/gcp.go | 46 ++++++++ mocks/cloud/ListCall.go | 222 +++++++++++++++++++++++++++++++++++ mocks/cloud/Lister.go | 80 +++++++++++++ 5 files changed, 439 insertions(+), 5 deletions(-) create mode 100644 internal/address/gcp_test.go create mode 100644 internal/cloud/gcp.go create mode 100644 mocks/cloud/ListCall.go create mode 100644 mocks/cloud/Lister.go diff --git a/internal/address/gcp.go b/internal/address/gcp.go index 1dc7636..73c02d9 100644 --- a/internal/address/gcp.go +++ b/internal/address/gcp.go @@ -7,6 +7,7 @@ import ( "time" "cloud.google.com/go/compute/metadata" + "github.com/doitintl/kubeip/internal/cloud" "github.com/pkg/errors" "github.com/sirupsen/logrus" "google.golang.org/api/compute/v1" @@ -24,6 +25,7 @@ const ( type gcpAssigner struct { client *compute.Service + lister cloud.Lister project string region string logger *logrus.Entry @@ -58,6 +60,7 @@ func NewGCPAssigner(ctx context.Context, logger *logrus.Entry, project, region s return &gcpAssigner{ client: client, + lister: cloud.NewLister(client), project: project, region: region, logger: logger, @@ -200,11 +203,13 @@ func (a *gcpAssigner) Assign(instanceID, zone string, filter []string, orderBy s } func (a *gcpAssigner) listAddresses(filter []string, orderBy, status string) ([]*compute.Address, error) { - call := a.client.Addresses.List(a.project, a.region) - var filters []string - // filter public addresses (with status) - filters = append(filters, fmt.Sprintf("(status=%s)", status)) - filters = append(filters, "(addressType=EXTERNAL)") + call := a.lister.List(a.project, a.region) + // Initialize filters with known filters + filters := []string{ + fmt.Sprintf("(status=%s)", status), + "(addressType=EXTERNAL)", + } + // filter addresses by provided filter: labels.key=value for _, f := range filter { filters = append(filters, fmt.Sprintf("(%s)", f)) diff --git a/internal/address/gcp_test.go b/internal/address/gcp_test.go new file mode 100644 index 0000000..0379a14 --- /dev/null +++ b/internal/address/gcp_test.go @@ -0,0 +1,81 @@ +package address + +import ( + "reflect" + "testing" + + "github.com/doitintl/kubeip/internal/cloud" + mocks "github.com/doitintl/kubeip/mocks/cloud" + "github.com/sirupsen/logrus" + "google.golang.org/api/compute/v1" +) + +func Test_gcpAssigner_listAddresses(t *testing.T) { + type fields struct { + listerFn func(t *testing.T) cloud.Lister + project string + region string + } + type args struct { + filter []string + orderBy string + status string + } + tests := []struct { + name string + fields fields + args args + want []*compute.Address + wantErr bool + }{ + { + name: "list addresses successfully", + fields: fields{ + project: "test-project", + region: "test-region", + listerFn: func(t *testing.T) cloud.Lister { + mock := mocks.NewLister(t) + mockCall := mocks.NewListCall(t) + mock.EXPECT().List("test-project", "test-region").Return(mockCall) + mockCall.EXPECT().Filter("(status=RESERVED) (addressType=EXTERNAL) (test-filter-1) (test-filter-2)").Return(mockCall) + mockCall.EXPECT().OrderBy("test-order-by").Return(mockCall) + mockCall.EXPECT().Do().Return(&compute.AddressList{ + Items: []*compute.Address{ + {Name: "test-address-1", Status: "RESERVED", Address: "10.10.0.1", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, + {Name: "test-address-2", Status: "RESERVED", Address: "10.10.0.2", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, + }, + }, nil) + return mock + }, + }, + args: args{ + filter: []string{"test-filter-1", "test-filter-2"}, + orderBy: "test-order-by", + status: "RESERVED", + }, + want: []*compute.Address{ + {Name: "test-address-1", Status: "RESERVED", Address: "10.10.0.1", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, + {Name: "test-address-2", Status: "RESERVED", Address: "10.10.0.2", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, + }, + }, + } + for _, tt := range tests { + logger := logrus.NewEntry(logrus.New()) + t.Run(tt.name, func(t *testing.T) { + a := &gcpAssigner{ + lister: tt.fields.listerFn(t), + project: tt.fields.project, + region: tt.fields.region, + logger: logger, + } + got, err := a.listAddresses(tt.args.filter, tt.args.orderBy, tt.args.status) + if (err != nil) != tt.wantErr { + t.Errorf("listAddresses() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("listAddresses() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/cloud/gcp.go b/internal/cloud/gcp.go new file mode 100644 index 0000000..6adc75a --- /dev/null +++ b/internal/cloud/gcp.go @@ -0,0 +1,46 @@ +package cloud + +import "google.golang.org/api/compute/v1" + +type ListCall interface { + Filter(filter string) ListCall + OrderBy(orderBy string) ListCall + PageToken(pageToken string) ListCall + Do() (*compute.AddressList, error) +} + +type Lister interface { + List(projectID, region string) ListCall +} + +func NewLister(client *compute.Service) Lister { + return &gcpLister{client: client} +} + +type gcpLister struct { + client *compute.Service +} + +type gcpListCall struct { + call *compute.AddressesListCall +} + +func (l *gcpLister) List(projectID, region string) ListCall { + return &gcpListCall{l.client.Addresses.List(projectID, region)} +} + +func (c *gcpListCall) Filter(filter string) ListCall { + return &gcpListCall{c.call.Filter(filter)} +} + +func (c *gcpListCall) OrderBy(orderBy string) ListCall { + return &gcpListCall{c.call.OrderBy(orderBy)} +} + +func (c *gcpListCall) PageToken(pageToken string) ListCall { + return &gcpListCall{c.call.PageToken(pageToken)} +} + +func (c *gcpListCall) Do() (*compute.AddressList, error) { + return c.call.Do() //nolint:wrapcheck +} diff --git a/mocks/cloud/ListCall.go b/mocks/cloud/ListCall.go new file mode 100644 index 0000000..a3d483c --- /dev/null +++ b/mocks/cloud/ListCall.go @@ -0,0 +1,222 @@ +// Code generated by mockery v2.30.16. DO NOT EDIT. + +package mocks + +import ( + cloud "github.com/doitintl/kubeip/internal/cloud" + compute "google.golang.org/api/compute/v1" + + mock "github.com/stretchr/testify/mock" +) + +// ListCall is an autogenerated mock type for the ListCall type +type ListCall struct { + mock.Mock +} + +type ListCall_Expecter struct { + mock *mock.Mock +} + +func (_m *ListCall) EXPECT() *ListCall_Expecter { + return &ListCall_Expecter{mock: &_m.Mock} +} + +// Do provides a mock function with given fields: +func (_m *ListCall) Do() (*compute.AddressList, error) { + ret := _m.Called() + + var r0 *compute.AddressList + var r1 error + if rf, ok := ret.Get(0).(func() (*compute.AddressList, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() *compute.AddressList); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*compute.AddressList) + } + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ListCall_Do_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Do' +type ListCall_Do_Call struct { + *mock.Call +} + +// Do is a helper method to define mock.On call +func (_e *ListCall_Expecter) Do() *ListCall_Do_Call { + return &ListCall_Do_Call{Call: _e.mock.On("Do")} +} + +func (_c *ListCall_Do_Call) Run(run func()) *ListCall_Do_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *ListCall_Do_Call) Return(_a0 *compute.AddressList, _a1 error) *ListCall_Do_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *ListCall_Do_Call) RunAndReturn(run func() (*compute.AddressList, error)) *ListCall_Do_Call { + _c.Call.Return(run) + return _c +} + +// Filter provides a mock function with given fields: filter +func (_m *ListCall) Filter(filter string) cloud.ListCall { + ret := _m.Called(filter) + + var r0 cloud.ListCall + if rf, ok := ret.Get(0).(func(string) cloud.ListCall); ok { + r0 = rf(filter) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(cloud.ListCall) + } + } + + return r0 +} + +// ListCall_Filter_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Filter' +type ListCall_Filter_Call struct { + *mock.Call +} + +// Filter is a helper method to define mock.On call +// - filter string +func (_e *ListCall_Expecter) Filter(filter interface{}) *ListCall_Filter_Call { + return &ListCall_Filter_Call{Call: _e.mock.On("Filter", filter)} +} + +func (_c *ListCall_Filter_Call) Run(run func(filter string)) *ListCall_Filter_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string)) + }) + return _c +} + +func (_c *ListCall_Filter_Call) Return(_a0 cloud.ListCall) *ListCall_Filter_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *ListCall_Filter_Call) RunAndReturn(run func(string) cloud.ListCall) *ListCall_Filter_Call { + _c.Call.Return(run) + return _c +} + +// OrderBy provides a mock function with given fields: orderBy +func (_m *ListCall) OrderBy(orderBy string) cloud.ListCall { + ret := _m.Called(orderBy) + + var r0 cloud.ListCall + if rf, ok := ret.Get(0).(func(string) cloud.ListCall); ok { + r0 = rf(orderBy) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(cloud.ListCall) + } + } + + return r0 +} + +// ListCall_OrderBy_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'OrderBy' +type ListCall_OrderBy_Call struct { + *mock.Call +} + +// OrderBy is a helper method to define mock.On call +// - orderBy string +func (_e *ListCall_Expecter) OrderBy(orderBy interface{}) *ListCall_OrderBy_Call { + return &ListCall_OrderBy_Call{Call: _e.mock.On("OrderBy", orderBy)} +} + +func (_c *ListCall_OrderBy_Call) Run(run func(orderBy string)) *ListCall_OrderBy_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string)) + }) + return _c +} + +func (_c *ListCall_OrderBy_Call) Return(_a0 cloud.ListCall) *ListCall_OrderBy_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *ListCall_OrderBy_Call) RunAndReturn(run func(string) cloud.ListCall) *ListCall_OrderBy_Call { + _c.Call.Return(run) + return _c +} + +// PageToken provides a mock function with given fields: pageToken +func (_m *ListCall) PageToken(pageToken string) cloud.ListCall { + ret := _m.Called(pageToken) + + var r0 cloud.ListCall + if rf, ok := ret.Get(0).(func(string) cloud.ListCall); ok { + r0 = rf(pageToken) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(cloud.ListCall) + } + } + + return r0 +} + +// ListCall_PageToken_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PageToken' +type ListCall_PageToken_Call struct { + *mock.Call +} + +// PageToken is a helper method to define mock.On call +// - pageToken string +func (_e *ListCall_Expecter) PageToken(pageToken interface{}) *ListCall_PageToken_Call { + return &ListCall_PageToken_Call{Call: _e.mock.On("PageToken", pageToken)} +} + +func (_c *ListCall_PageToken_Call) Run(run func(pageToken string)) *ListCall_PageToken_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string)) + }) + return _c +} + +func (_c *ListCall_PageToken_Call) Return(_a0 cloud.ListCall) *ListCall_PageToken_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *ListCall_PageToken_Call) RunAndReturn(run func(string) cloud.ListCall) *ListCall_PageToken_Call { + _c.Call.Return(run) + return _c +} + +// NewListCall creates a new instance of ListCall. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewListCall(t interface { + mock.TestingT + Cleanup(func()) +}) *ListCall { + mock := &ListCall{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/cloud/Lister.go b/mocks/cloud/Lister.go new file mode 100644 index 0000000..e2f5f14 --- /dev/null +++ b/mocks/cloud/Lister.go @@ -0,0 +1,80 @@ +// Code generated by mockery v2.30.16. DO NOT EDIT. + +package mocks + +import ( + cloud "github.com/doitintl/kubeip/internal/cloud" + mock "github.com/stretchr/testify/mock" +) + +// Lister is an autogenerated mock type for the Lister type +type Lister struct { + mock.Mock +} + +type Lister_Expecter struct { + mock *mock.Mock +} + +func (_m *Lister) EXPECT() *Lister_Expecter { + return &Lister_Expecter{mock: &_m.Mock} +} + +// List provides a mock function with given fields: projectID, region +func (_m *Lister) List(projectID string, region string) cloud.ListCall { + ret := _m.Called(projectID, region) + + var r0 cloud.ListCall + if rf, ok := ret.Get(0).(func(string, string) cloud.ListCall); ok { + r0 = rf(projectID, region) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(cloud.ListCall) + } + } + + return r0 +} + +// Lister_List_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'List' +type Lister_List_Call struct { + *mock.Call +} + +// List is a helper method to define mock.On call +// - projectID string +// - region string +func (_e *Lister_Expecter) List(projectID interface{}, region interface{}) *Lister_List_Call { + return &Lister_List_Call{Call: _e.mock.On("List", projectID, region)} +} + +func (_c *Lister_List_Call) Run(run func(projectID string, region string)) *Lister_List_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string), args[1].(string)) + }) + return _c +} + +func (_c *Lister_List_Call) Return(_a0 cloud.ListCall) *Lister_List_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Lister_List_Call) RunAndReturn(run func(string, string) cloud.ListCall) *Lister_List_Call { + _c.Call.Return(run) + return _c +} + +// NewLister creates a new instance of Lister. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewLister(t interface { + mock.TestingT + Cleanup(func()) +}) *Lister { + mock := &Lister{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} From 91a552fcce1fd4a45f5a8497466272a69bab00c3 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Mon, 9 Oct 2023 19:26:25 +0300 Subject: [PATCH 22/66] test list with multiple pages --- internal/address/gcp_test.go | 40 ++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/internal/address/gcp_test.go b/internal/address/gcp_test.go index 0379a14..7a8959f 100644 --- a/internal/address/gcp_test.go +++ b/internal/address/gcp_test.go @@ -58,6 +58,46 @@ func Test_gcpAssigner_listAddresses(t *testing.T) { {Name: "test-address-2", Status: "RESERVED", Address: "10.10.0.2", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, }, }, + { + name: "list addresses with multiple pages successfully", + fields: fields{ + project: "test-project", + region: "test-region", + listerFn: func(t *testing.T) cloud.Lister { + mock := mocks.NewLister(t) + mockCall := mocks.NewListCall(t) + mock.EXPECT().List("test-project", "test-region").Return(mockCall) + mockCall.EXPECT().Filter("(status=RESERVED) (addressType=EXTERNAL) (test-filter-1) (test-filter-2)").Return(mockCall) + mockCall.EXPECT().OrderBy("test-order-by").Return(mockCall) + mockCall.EXPECT().Do().Return(&compute.AddressList{ + Items: []*compute.Address{ + {Name: "test-address-1", Status: "RESERVED", Address: "10.10.0.1", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, + {Name: "test-address-2", Status: "RESERVED", Address: "10.10.0.2", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, + }, + NextPageToken: "test-next-page-token", + }, nil).Once() + mockCall.EXPECT().PageToken("test-next-page-token").Return(mockCall) + mockCall.EXPECT().Do().Return(&compute.AddressList{ + Items: []*compute.Address{ + {Name: "test-address-3", Status: "RESERVED", Address: "10.10.0.3", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, + {Name: "test-address-4", Status: "RESERVED", Address: "10.10.0.4", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, + }, + }, nil).Once() + return mock + }, + }, + args: args{ + filter: []string{"test-filter-1", "test-filter-2"}, + orderBy: "test-order-by", + status: "RESERVED", + }, + want: []*compute.Address{ + {Name: "test-address-1", Status: "RESERVED", Address: "10.10.0.1", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, + {Name: "test-address-2", Status: "RESERVED", Address: "10.10.0.2", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, + {Name: "test-address-3", Status: "RESERVED", Address: "10.10.0.3", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, + {Name: "test-address-4", Status: "RESERVED", Address: "10.10.0.4", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, + }, + }, } for _, tt := range tests { logger := logrus.NewEntry(logrus.New()) From 26bc9e588a8e18519eee04ebc4acc25a143c6930 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Mon, 9 Oct 2023 19:32:01 +0300 Subject: [PATCH 23/66] simplify wait for op --- internal/address/gcp.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/internal/address/gcp.go b/internal/address/gcp.go index 73c02d9..92297ee 100644 --- a/internal/address/gcp.go +++ b/internal/address/gcp.go @@ -72,15 +72,10 @@ func (a *gcpAssigner) waitForOperation(op *compute.Operation, zone string, timeo a.logger.Warn("operation is nil") return nil } - // Create a context that can be cancelled - ctx, cancel := context.WithCancel(context.Background()) + // Create a context that will be cancelled with timeout + ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - // Create a timer that cancels the context after timeout - timer := time.AfterFunc(timeout, func() { - cancel() - }) - defer timer.Stop() var err error name := op.Name for op.Status != operationDone { From e3bd2452b414b3682ac35b7bc7b01504d5c29c12 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Tue, 10 Oct 2023 12:41:39 +0300 Subject: [PATCH 24/66] abstract GCP API for easier mock gen --- internal/address/gcp.go | 34 +++++++++++--------- internal/cloud/gcp_address.go | 26 +++++++++++++++ internal/cloud/gcp_instance.go | 19 +++++++++++ internal/cloud/{gcp.go => gcp_lister.go} | 0 internal/cloud/gcp_waiter.go | 40 ++++++++++++++++++++++++ internal/node/explorer.go | 5 +-- 6 files changed, 107 insertions(+), 17 deletions(-) create mode 100644 internal/cloud/gcp_address.go create mode 100644 internal/cloud/gcp_instance.go rename internal/cloud/{gcp.go => gcp_lister.go} (100%) create mode 100644 internal/cloud/gcp_waiter.go diff --git a/internal/address/gcp.go b/internal/address/gcp.go index 92297ee..1f2cbb2 100644 --- a/internal/address/gcp.go +++ b/internal/address/gcp.go @@ -24,11 +24,13 @@ const ( ) type gcpAssigner struct { - client *compute.Service - lister cloud.Lister - project string - region string - logger *logrus.Entry + lister cloud.Lister + waiter cloud.ZoneWaiter + addressManager cloud.AddressManager + instanceGetter cloud.InstanceGetter + project string + region string + logger *logrus.Entry } func NewGCPAssigner(ctx context.Context, logger *logrus.Entry, project, region string) (Assigner, error) { @@ -59,11 +61,13 @@ func NewGCPAssigner(ctx context.Context, logger *logrus.Entry, project, region s } return &gcpAssigner{ - client: client, - lister: cloud.NewLister(client), - project: project, - region: region, - logger: logger, + lister: cloud.NewLister(client), + waiter: cloud.NewZoneWaiter(client), + addressManager: cloud.NewAddressManager(client), + instanceGetter: cloud.NewInstanceGetter(client), + project: project, + region: region, + logger: logger, }, nil } @@ -80,7 +84,7 @@ func (a *gcpAssigner) waitForOperation(op *compute.Operation, zone string, timeo name := op.Name for op.Status != operationDone { // Pass the cancellable context to the Wait method - op, err = a.client.ZoneOperations.Wait(a.project, zone, name).Context(ctx).Do() + op, err = a.waiter.Wait(a.project, zone, name).Context(ctx).Do() if err != nil { // If the context was cancelled, return a timeout error if errors.Is(err, context.Canceled) { @@ -117,7 +121,7 @@ func (a *gcpAssigner) deleteInstanceAddress(instance *compute.Instance, zone str "instance": instance.Name, "address": accessConfig.NatIP, }).Infof("deleting ephemeral public IP address from instance") - op, err := a.client.Instances.DeleteAccessConfig(a.project, zone, instance.Name, accessConfigName, networkInterface.Name).Do() + op, err := a.addressManager.DeleteAccessConfig(a.project, zone, instance.Name, accessConfigName, networkInterface.Name) if err != nil { return errors.Wrapf(err, "failed to delete access config %s from instance %s", accessConfigName, instance.Name) } @@ -134,12 +138,12 @@ func (a *gcpAssigner) addInstanceAddress(instance *compute.Instance, zone string "instance": instance.Name, "address": address.Address, }).Infof("adding reserved public IP address to instance") - op, err := a.client.Instances.AddAccessConfig(a.project, zone, instance.Name, defaultNetworkInterface, &compute.AccessConfig{ + op, err := a.addressManager.AddAccessConfig(a.project, zone, instance.Name, defaultNetworkInterface, &compute.AccessConfig{ Name: address.Name, Type: accessConfigType, Kind: accessConfigKind, NatIP: address.Address, - }).Do() + }) if err != nil { return errors.Wrapf(err, "failed to add access config %s to instance %s", address.Name, instance.Name) } @@ -152,7 +156,7 @@ func (a *gcpAssigner) addInstanceAddress(instance *compute.Instance, zone string func (a *gcpAssigner) Assign(instanceID, zone string, filter []string, orderBy string) error { // check if instance already has a public static IP address assigned - instance, err := a.client.Instances.Get(a.project, zone, instanceID).Do() + instance, err := a.instanceGetter.Get(a.project, zone, instanceID) if err != nil { return errors.Wrapf(err, "failed to get instance %s", instanceID) } diff --git a/internal/cloud/gcp_address.go b/internal/cloud/gcp_address.go new file mode 100644 index 0000000..4b8adb7 --- /dev/null +++ b/internal/cloud/gcp_address.go @@ -0,0 +1,26 @@ +package cloud + +import ( + "google.golang.org/api/compute/v1" +) + +type AddressManager interface { + AddAccessConfig(project string, zone string, instance string, networkInterface string, accessconfig *compute.AccessConfig) (*compute.Operation, error) + DeleteAccessConfig(project string, zone string, instance string, accessConfig string, networkInterface string) (*compute.Operation, error) +} + +type addressManager struct { + client *compute.Service +} + +func NewAddressManager(client *compute.Service) AddressManager { + return &addressManager{client: client} +} + +func (m *addressManager) AddAccessConfig(project, zone, instance, networkInterface string, accessconfig *compute.AccessConfig) (*compute.Operation, error) { + return m.client.Instances.AddAccessConfig(project, zone, instance, networkInterface, accessconfig).Do() //nolint:wrapcheck +} + +func (m *addressManager) DeleteAccessConfig(project, zone, instance, accessConfig, networkInterface string) (*compute.Operation, error) { + return m.client.Instances.DeleteAccessConfig(project, zone, instance, accessConfig, networkInterface).Do() //nolint:wrapcheck +} diff --git a/internal/cloud/gcp_instance.go b/internal/cloud/gcp_instance.go new file mode 100644 index 0000000..10e7d6b --- /dev/null +++ b/internal/cloud/gcp_instance.go @@ -0,0 +1,19 @@ +package cloud + +import "google.golang.org/api/compute/v1" + +type InstanceGetter interface { + Get(projectID, zone, instance string) (*compute.Instance, error) +} + +type instanceGetter struct { + client *compute.Service +} + +func NewInstanceGetter(client *compute.Service) InstanceGetter { + return &instanceGetter{client: client} +} + +func (g *instanceGetter) Get(projectID, zone, instance string) (*compute.Instance, error) { + return g.client.Instances.Get(projectID, zone, instance).Do() //nolint:wrapcheck +} diff --git a/internal/cloud/gcp.go b/internal/cloud/gcp_lister.go similarity index 100% rename from internal/cloud/gcp.go rename to internal/cloud/gcp_lister.go diff --git a/internal/cloud/gcp_waiter.go b/internal/cloud/gcp_waiter.go new file mode 100644 index 0000000..0875eee --- /dev/null +++ b/internal/cloud/gcp_waiter.go @@ -0,0 +1,40 @@ +package cloud + +import ( + "context" + + "google.golang.org/api/compute/v1" +) + +type WaitCall interface { + Context(ctx context.Context) WaitCall + Do() (*compute.Operation, error) +} + +type ZoneWaiter interface { + Wait(projectID, region, operationName string) WaitCall +} + +type zoneWaiter struct { + client *compute.Service +} + +type zoneWaitCall struct { + call *compute.ZoneOperationsWaitCall +} + +func NewZoneWaiter(client *compute.Service) ZoneWaiter { + return &zoneWaiter{client: client} +} + +func (w *zoneWaiter) Wait(projectID, region, operationName string) WaitCall { + return &zoneWaitCall{w.client.ZoneOperations.Wait(projectID, region, operationName)} +} + +func (c *zoneWaitCall) Context(ctx context.Context) WaitCall { + return &zoneWaitCall{c.call.Context(ctx)} +} + +func (c *zoneWaitCall) Do() (*compute.Operation, error) { + return c.call.Do() //nolint:wrapcheck +} diff --git a/internal/node/explorer.go b/internal/node/explorer.go index 723c8d2..5b2cdb0 100644 --- a/internal/node/explorer.go +++ b/internal/node/explorer.go @@ -14,7 +14,8 @@ import ( ) const ( - podInfoDir = "/etc/podinfo/" + minProviderIDTokens = 2 + podInfoDir = "/etc/podinfo/" ) type Explorer interface { @@ -55,7 +56,7 @@ func getCloudProvider(providerID string) (types.CloudProvider, error) { func getInstance(providerID string) (string, error) { s := strings.Split(providerID, "/") - if len(s) < 2 { + if len(s) < minProviderIDTokens { return "", errors.Errorf("failed to get instance ID") } return s[len(s)-1], nil From 0e8e769e53e4460859f61af6fd535de89f040f64 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Tue, 10 Oct 2023 13:27:24 +0300 Subject: [PATCH 25/66] test wait operation --- internal/address/gcp_test.go | 137 ++++++++++++++++++++++++++++++ internal/node/explorer.go | 15 ++-- mocks/cloud/AddressManager.go | 151 ++++++++++++++++++++++++++++++++++ mocks/cloud/InstanceGetter.go | 91 ++++++++++++++++++++ mocks/cloud/WaitCall.go | 136 ++++++++++++++++++++++++++++++ mocks/cloud/ZoneWaiter.go | 81 ++++++++++++++++++ 6 files changed, 606 insertions(+), 5 deletions(-) create mode 100644 mocks/cloud/AddressManager.go create mode 100644 mocks/cloud/InstanceGetter.go create mode 100644 mocks/cloud/WaitCall.go create mode 100644 mocks/cloud/ZoneWaiter.go diff --git a/internal/address/gcp_test.go b/internal/address/gcp_test.go index 7a8959f..80f39fe 100644 --- a/internal/address/gcp_test.go +++ b/internal/address/gcp_test.go @@ -1,12 +1,16 @@ package address import ( + "context" "reflect" "testing" + "time" "github.com/doitintl/kubeip/internal/cloud" mocks "github.com/doitintl/kubeip/mocks/cloud" + "github.com/pkg/errors" "github.com/sirupsen/logrus" + tmock "github.com/stretchr/testify/mock" "google.golang.org/api/compute/v1" ) @@ -119,3 +123,136 @@ func Test_gcpAssigner_listAddresses(t *testing.T) { }) } } + +func Test_gcpAssigner_waitForOperation(t *testing.T) { + type fields struct { + waiterFn func(t *testing.T) cloud.ZoneWaiter + project string + logger *logrus.Entry + } + type args struct { + op *compute.Operation + zone string + timeout time.Duration + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "wait for operation successfully", + fields: fields{ + project: "test-project", + waiterFn: func(t *testing.T) cloud.ZoneWaiter { + mock := mocks.NewZoneWaiter(t) + mockCall := mocks.NewWaitCall(t) + mock.EXPECT().Wait("test-project", "test-zone", "test-operation").Return(mockCall) + mockCall.EXPECT().Context(tmock.Anything).Return(mockCall) + mockCall.EXPECT().Do().Return(&compute.Operation{Status: "DONE"}, nil) + return mock + }, + }, + args: args{ + op: &compute.Operation{Name: "test-operation", Status: "RUNNING"}, + zone: "test-zone", + timeout: time.Millisecond, + }, + }, + { + name: "wait for operation with a few retries successfully", + fields: fields{ + project: "test-project", + waiterFn: func(t *testing.T) cloud.ZoneWaiter { + mock := mocks.NewZoneWaiter(t) + mockCall := mocks.NewWaitCall(t) + mock.EXPECT().Wait("test-project", "test-zone", "test-operation").Return(mockCall) + mockCall.EXPECT().Context(tmock.Anything).Return(mockCall) + mockCall.EXPECT().Do().Return(&compute.Operation{Status: "RUNNING"}, nil).Times(2) + mockCall.EXPECT().Do().Return(&compute.Operation{Status: "DONE"}, nil) + return mock + }, + }, + args: args{ + op: &compute.Operation{Name: "test-operation", Status: "RUNNING"}, + zone: "test-zone", + timeout: time.Millisecond * 2, + }, + }, + { + name: "wait for operation with timeout", + fields: fields{ + project: "test-project", + waiterFn: func(t *testing.T) cloud.ZoneWaiter { + mock := mocks.NewZoneWaiter(t) + mockCall := mocks.NewWaitCall(t) + mock.EXPECT().Wait("test-project", "test-zone", "test-operation").Return(mockCall) + mockCall.EXPECT().Context(tmock.Anything).Return(mockCall) + mockCall.EXPECT().Do().Return(nil, context.Canceled) + return mock + }, + }, + args: args{ + op: &compute.Operation{Name: "test-operation", Status: "RUNNING"}, + zone: "test-zone", + timeout: time.Millisecond, + }, + wantErr: true, + }, + { + name: "wait for operation with error", + fields: fields{ + project: "test-project", + waiterFn: func(t *testing.T) cloud.ZoneWaiter { + mock := mocks.NewZoneWaiter(t) + mockCall := mocks.NewWaitCall(t) + mock.EXPECT().Wait("test-project", "test-zone", "test-operation").Return(mockCall) + mockCall.EXPECT().Context(tmock.Anything).Return(mockCall) + mockCall.EXPECT().Do().Return(nil, errors.New("test-error")) + return mock + }, + }, + args: args{ + op: &compute.Operation{Name: "test-operation", Status: "RUNNING"}, + zone: "test-zone", + timeout: time.Millisecond, + }, + wantErr: true, + }, + { + name: "wait for operation with error in operation", + fields: fields{ + project: "test-project", + waiterFn: func(t *testing.T) cloud.ZoneWaiter { + mock := mocks.NewZoneWaiter(t) + mockCall := mocks.NewWaitCall(t) + mock.EXPECT().Wait("test-project", "test-zone", "test-operation").Return(mockCall) + mockCall.EXPECT().Context(tmock.Anything).Return(mockCall) + mockCall.EXPECT().Do().Return(&compute.Operation{Status: "DONE", Error: &compute.OperationError{Errors: []*compute.OperationErrorErrors{{Code: "123", Message: "test-error"}}}}, nil) + return mock + }, + }, + args: args{ + op: &compute.Operation{Name: "test-operation", Status: "RUNNING"}, + zone: "test-zone", + timeout: time.Millisecond, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + logger := logrus.NewEntry(logrus.New()) + waiter := tt.fields.waiterFn(t) + a := &gcpAssigner{ + waiter: waiter, + project: tt.fields.project, + logger: logger, + } + if err := a.waitForOperation(tt.args.op, tt.args.zone, tt.args.timeout); (err != nil) != tt.wantErr { + t.Errorf("waitForOperation() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/internal/node/explorer.go b/internal/node/explorer.go index 5b2cdb0..8b0f0aa 100644 --- a/internal/node/explorer.go +++ b/internal/node/explorer.go @@ -16,6 +16,11 @@ import ( const ( minProviderIDTokens = 2 podInfoDir = "/etc/podinfo/" + awsPoolLabel = "eks.amazonaws.com/nodegroup" + azurePoolLabel = "node.kubernetes.io/instancegroup" + gcpPoolLabel = "cloud.google.com/gke-nodepool" + regionLabel = "topology.kubernetes.io/region" + zoneLabel = "topology.kubernetes.io/zone" ) type Explorer interface { @@ -66,11 +71,11 @@ func getNodePool(providerID types.CloudProvider, labels map[string]string) (stri var ok bool var pool string if providerID == types.CloudProviderAWS { - pool, ok = labels["eks.amazonaws.com/nodegroup"] + pool, ok = labels[awsPoolLabel] } else if providerID == types.CloudProviderAzure { - pool, ok = labels["node.kubernetes.io/instancegroup"] + pool, ok = labels[azurePoolLabel] } else if providerID == types.CloudProviderGCP { - pool, ok = labels["cloud.google.com/gke-nodepool"] + pool, ok = labels[gcpPoolLabel] } else { return "", errors.Errorf("unsupported cloud provider: %s", providerID) } @@ -130,13 +135,13 @@ func (d *explorer) GetNode(ctx context.Context, nodeName string) (*types.Node, e } // get node region from node labels - region, ok := n.Labels["topology.kubernetes.io/region"] + region, ok := n.Labels[regionLabel] if !ok { return nil, errors.Errorf("failed to get node region") } // get node zone from node labels - zone, ok := n.Labels["topology.kubernetes.io/zone"] + zone, ok := n.Labels[zoneLabel] if !ok { return nil, errors.Errorf("failed to get node zone") } diff --git a/mocks/cloud/AddressManager.go b/mocks/cloud/AddressManager.go new file mode 100644 index 0000000..b925b2c --- /dev/null +++ b/mocks/cloud/AddressManager.go @@ -0,0 +1,151 @@ +// Code generated by mockery v2.30.16. DO NOT EDIT. + +package mocks + +import ( + mock "github.com/stretchr/testify/mock" + compute "google.golang.org/api/compute/v1" +) + +// AddressManager is an autogenerated mock type for the AddressManager type +type AddressManager struct { + mock.Mock +} + +type AddressManager_Expecter struct { + mock *mock.Mock +} + +func (_m *AddressManager) EXPECT() *AddressManager_Expecter { + return &AddressManager_Expecter{mock: &_m.Mock} +} + +// AddAccessConfig provides a mock function with given fields: project, zone, instance, networkInterface, accessconfig +func (_m *AddressManager) AddAccessConfig(project string, zone string, instance string, networkInterface string, accessconfig *compute.AccessConfig) (*compute.Operation, error) { + ret := _m.Called(project, zone, instance, networkInterface, accessconfig) + + var r0 *compute.Operation + var r1 error + if rf, ok := ret.Get(0).(func(string, string, string, string, *compute.AccessConfig) (*compute.Operation, error)); ok { + return rf(project, zone, instance, networkInterface, accessconfig) + } + if rf, ok := ret.Get(0).(func(string, string, string, string, *compute.AccessConfig) *compute.Operation); ok { + r0 = rf(project, zone, instance, networkInterface, accessconfig) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*compute.Operation) + } + } + + if rf, ok := ret.Get(1).(func(string, string, string, string, *compute.AccessConfig) error); ok { + r1 = rf(project, zone, instance, networkInterface, accessconfig) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// AddressManager_AddAccessConfig_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AddAccessConfig' +type AddressManager_AddAccessConfig_Call struct { + *mock.Call +} + +// AddAccessConfig is a helper method to define mock.On call +// - project string +// - zone string +// - instance string +// - networkInterface string +// - accessconfig *compute.AccessConfig +func (_e *AddressManager_Expecter) AddAccessConfig(project interface{}, zone interface{}, instance interface{}, networkInterface interface{}, accessconfig interface{}) *AddressManager_AddAccessConfig_Call { + return &AddressManager_AddAccessConfig_Call{Call: _e.mock.On("AddAccessConfig", project, zone, instance, networkInterface, accessconfig)} +} + +func (_c *AddressManager_AddAccessConfig_Call) Run(run func(project string, zone string, instance string, networkInterface string, accessconfig *compute.AccessConfig)) *AddressManager_AddAccessConfig_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string), args[1].(string), args[2].(string), args[3].(string), args[4].(*compute.AccessConfig)) + }) + return _c +} + +func (_c *AddressManager_AddAccessConfig_Call) Return(_a0 *compute.Operation, _a1 error) *AddressManager_AddAccessConfig_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *AddressManager_AddAccessConfig_Call) RunAndReturn(run func(string, string, string, string, *compute.AccessConfig) (*compute.Operation, error)) *AddressManager_AddAccessConfig_Call { + _c.Call.Return(run) + return _c +} + +// DeleteAccessConfig provides a mock function with given fields: project, zone, instance, accessConfig, networkInterface +func (_m *AddressManager) DeleteAccessConfig(project string, zone string, instance string, accessConfig string, networkInterface string) (*compute.Operation, error) { + ret := _m.Called(project, zone, instance, accessConfig, networkInterface) + + var r0 *compute.Operation + var r1 error + if rf, ok := ret.Get(0).(func(string, string, string, string, string) (*compute.Operation, error)); ok { + return rf(project, zone, instance, accessConfig, networkInterface) + } + if rf, ok := ret.Get(0).(func(string, string, string, string, string) *compute.Operation); ok { + r0 = rf(project, zone, instance, accessConfig, networkInterface) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*compute.Operation) + } + } + + if rf, ok := ret.Get(1).(func(string, string, string, string, string) error); ok { + r1 = rf(project, zone, instance, accessConfig, networkInterface) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// AddressManager_DeleteAccessConfig_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteAccessConfig' +type AddressManager_DeleteAccessConfig_Call struct { + *mock.Call +} + +// DeleteAccessConfig is a helper method to define mock.On call +// - project string +// - zone string +// - instance string +// - accessConfig string +// - networkInterface string +func (_e *AddressManager_Expecter) DeleteAccessConfig(project interface{}, zone interface{}, instance interface{}, accessConfig interface{}, networkInterface interface{}) *AddressManager_DeleteAccessConfig_Call { + return &AddressManager_DeleteAccessConfig_Call{Call: _e.mock.On("DeleteAccessConfig", project, zone, instance, accessConfig, networkInterface)} +} + +func (_c *AddressManager_DeleteAccessConfig_Call) Run(run func(project string, zone string, instance string, accessConfig string, networkInterface string)) *AddressManager_DeleteAccessConfig_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string), args[1].(string), args[2].(string), args[3].(string), args[4].(string)) + }) + return _c +} + +func (_c *AddressManager_DeleteAccessConfig_Call) Return(_a0 *compute.Operation, _a1 error) *AddressManager_DeleteAccessConfig_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *AddressManager_DeleteAccessConfig_Call) RunAndReturn(run func(string, string, string, string, string) (*compute.Operation, error)) *AddressManager_DeleteAccessConfig_Call { + _c.Call.Return(run) + return _c +} + +// NewAddressManager creates a new instance of AddressManager. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewAddressManager(t interface { + mock.TestingT + Cleanup(func()) +}) *AddressManager { + mock := &AddressManager{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/cloud/InstanceGetter.go b/mocks/cloud/InstanceGetter.go new file mode 100644 index 0000000..2a72783 --- /dev/null +++ b/mocks/cloud/InstanceGetter.go @@ -0,0 +1,91 @@ +// Code generated by mockery v2.30.16. DO NOT EDIT. + +package mocks + +import ( + mock "github.com/stretchr/testify/mock" + compute "google.golang.org/api/compute/v1" +) + +// InstanceGetter is an autogenerated mock type for the InstanceGetter type +type InstanceGetter struct { + mock.Mock +} + +type InstanceGetter_Expecter struct { + mock *mock.Mock +} + +func (_m *InstanceGetter) EXPECT() *InstanceGetter_Expecter { + return &InstanceGetter_Expecter{mock: &_m.Mock} +} + +// Get provides a mock function with given fields: projectID, zone, instance +func (_m *InstanceGetter) Get(projectID string, zone string, instance string) (*compute.Instance, error) { + ret := _m.Called(projectID, zone, instance) + + var r0 *compute.Instance + var r1 error + if rf, ok := ret.Get(0).(func(string, string, string) (*compute.Instance, error)); ok { + return rf(projectID, zone, instance) + } + if rf, ok := ret.Get(0).(func(string, string, string) *compute.Instance); ok { + r0 = rf(projectID, zone, instance) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*compute.Instance) + } + } + + if rf, ok := ret.Get(1).(func(string, string, string) error); ok { + r1 = rf(projectID, zone, instance) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// InstanceGetter_Get_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Get' +type InstanceGetter_Get_Call struct { + *mock.Call +} + +// Get is a helper method to define mock.On call +// - projectID string +// - zone string +// - instance string +func (_e *InstanceGetter_Expecter) Get(projectID interface{}, zone interface{}, instance interface{}) *InstanceGetter_Get_Call { + return &InstanceGetter_Get_Call{Call: _e.mock.On("Get", projectID, zone, instance)} +} + +func (_c *InstanceGetter_Get_Call) Run(run func(projectID string, zone string, instance string)) *InstanceGetter_Get_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string), args[1].(string), args[2].(string)) + }) + return _c +} + +func (_c *InstanceGetter_Get_Call) Return(_a0 *compute.Instance, _a1 error) *InstanceGetter_Get_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *InstanceGetter_Get_Call) RunAndReturn(run func(string, string, string) (*compute.Instance, error)) *InstanceGetter_Get_Call { + _c.Call.Return(run) + return _c +} + +// NewInstanceGetter creates a new instance of InstanceGetter. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewInstanceGetter(t interface { + mock.TestingT + Cleanup(func()) +}) *InstanceGetter { + mock := &InstanceGetter{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/cloud/WaitCall.go b/mocks/cloud/WaitCall.go new file mode 100644 index 0000000..8516a60 --- /dev/null +++ b/mocks/cloud/WaitCall.go @@ -0,0 +1,136 @@ +// Code generated by mockery v2.30.16. DO NOT EDIT. + +package mocks + +import ( + cloud "github.com/doitintl/kubeip/internal/cloud" + compute "google.golang.org/api/compute/v1" + + context "context" + + mock "github.com/stretchr/testify/mock" +) + +// WaitCall is an autogenerated mock type for the WaitCall type +type WaitCall struct { + mock.Mock +} + +type WaitCall_Expecter struct { + mock *mock.Mock +} + +func (_m *WaitCall) EXPECT() *WaitCall_Expecter { + return &WaitCall_Expecter{mock: &_m.Mock} +} + +// Context provides a mock function with given fields: ctx +func (_m *WaitCall) Context(ctx context.Context) cloud.WaitCall { + ret := _m.Called(ctx) + + var r0 cloud.WaitCall + if rf, ok := ret.Get(0).(func(context.Context) cloud.WaitCall); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(cloud.WaitCall) + } + } + + return r0 +} + +// WaitCall_Context_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Context' +type WaitCall_Context_Call struct { + *mock.Call +} + +// Context is a helper method to define mock.On call +// - ctx context.Context +func (_e *WaitCall_Expecter) Context(ctx interface{}) *WaitCall_Context_Call { + return &WaitCall_Context_Call{Call: _e.mock.On("Context", ctx)} +} + +func (_c *WaitCall_Context_Call) Run(run func(ctx context.Context)) *WaitCall_Context_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *WaitCall_Context_Call) Return(_a0 cloud.WaitCall) *WaitCall_Context_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *WaitCall_Context_Call) RunAndReturn(run func(context.Context) cloud.WaitCall) *WaitCall_Context_Call { + _c.Call.Return(run) + return _c +} + +// Do provides a mock function with given fields: +func (_m *WaitCall) Do() (*compute.Operation, error) { + ret := _m.Called() + + var r0 *compute.Operation + var r1 error + if rf, ok := ret.Get(0).(func() (*compute.Operation, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() *compute.Operation); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*compute.Operation) + } + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// WaitCall_Do_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Do' +type WaitCall_Do_Call struct { + *mock.Call +} + +// Do is a helper method to define mock.On call +func (_e *WaitCall_Expecter) Do() *WaitCall_Do_Call { + return &WaitCall_Do_Call{Call: _e.mock.On("Do")} +} + +func (_c *WaitCall_Do_Call) Run(run func()) *WaitCall_Do_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *WaitCall_Do_Call) Return(_a0 *compute.Operation, _a1 error) *WaitCall_Do_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *WaitCall_Do_Call) RunAndReturn(run func() (*compute.Operation, error)) *WaitCall_Do_Call { + _c.Call.Return(run) + return _c +} + +// NewWaitCall creates a new instance of WaitCall. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewWaitCall(t interface { + mock.TestingT + Cleanup(func()) +}) *WaitCall { + mock := &WaitCall{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/cloud/ZoneWaiter.go b/mocks/cloud/ZoneWaiter.go new file mode 100644 index 0000000..7e7612d --- /dev/null +++ b/mocks/cloud/ZoneWaiter.go @@ -0,0 +1,81 @@ +// Code generated by mockery v2.30.16. DO NOT EDIT. + +package mocks + +import ( + cloud "github.com/doitintl/kubeip/internal/cloud" + mock "github.com/stretchr/testify/mock" +) + +// ZoneWaiter is an autogenerated mock type for the ZoneWaiter type +type ZoneWaiter struct { + mock.Mock +} + +type ZoneWaiter_Expecter struct { + mock *mock.Mock +} + +func (_m *ZoneWaiter) EXPECT() *ZoneWaiter_Expecter { + return &ZoneWaiter_Expecter{mock: &_m.Mock} +} + +// Wait provides a mock function with given fields: projectID, region, operationName +func (_m *ZoneWaiter) Wait(projectID string, region string, operationName string) cloud.WaitCall { + ret := _m.Called(projectID, region, operationName) + + var r0 cloud.WaitCall + if rf, ok := ret.Get(0).(func(string, string, string) cloud.WaitCall); ok { + r0 = rf(projectID, region, operationName) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(cloud.WaitCall) + } + } + + return r0 +} + +// ZoneWaiter_Wait_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Wait' +type ZoneWaiter_Wait_Call struct { + *mock.Call +} + +// Wait is a helper method to define mock.On call +// - projectID string +// - region string +// - operationName string +func (_e *ZoneWaiter_Expecter) Wait(projectID interface{}, region interface{}, operationName interface{}) *ZoneWaiter_Wait_Call { + return &ZoneWaiter_Wait_Call{Call: _e.mock.On("Wait", projectID, region, operationName)} +} + +func (_c *ZoneWaiter_Wait_Call) Run(run func(projectID string, region string, operationName string)) *ZoneWaiter_Wait_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string), args[1].(string), args[2].(string)) + }) + return _c +} + +func (_c *ZoneWaiter_Wait_Call) Return(_a0 cloud.WaitCall) *ZoneWaiter_Wait_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *ZoneWaiter_Wait_Call) RunAndReturn(run func(string, string, string) cloud.WaitCall) *ZoneWaiter_Wait_Call { + _c.Call.Return(run) + return _c +} + +// NewZoneWaiter creates a new instance of ZoneWaiter. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewZoneWaiter(t interface { + mock.TestingT + Cleanup(func()) +}) *ZoneWaiter { + mock := &ZoneWaiter{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} From cb83ccf86695338035bd532f14b4521ad8cdb9dd Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Wed, 11 Oct 2023 19:10:08 +0300 Subject: [PATCH 26/66] implement aws eip assigner --- cmd/main.go | 5 +- cmd/main_test.go | 13 +- go.mod | 14 +++ go.sum | 31 +++++ internal/address/assigner.go | 6 +- internal/address/aws.go | 207 ++++++++++++++++++++++++++++++- internal/address/azure.go | 4 +- internal/address/gcp.go | 18 +-- internal/address/gcp_test.go | 2 +- internal/cloud/aws_address.go | 37 ++++++ internal/cloud/aws_instance.go | 40 ++++++ internal/cloud/aws_lister.go | 53 ++++++++ mocks/address/Assigner.go | 27 ++-- mocks/cloud/Ec2InstanceGetter.go | 93 ++++++++++++++ mocks/cloud/EipAssigner.go | 82 ++++++++++++ mocks/cloud/EipLister.go | 93 ++++++++++++++ 16 files changed, 692 insertions(+), 33 deletions(-) create mode 100644 internal/cloud/aws_address.go create mode 100644 internal/cloud/aws_instance.go create mode 100644 internal/cloud/aws_lister.go create mode 100644 mocks/cloud/Ec2InstanceGetter.go create mode 100644 mocks/cloud/EipAssigner.go create mode 100644 mocks/cloud/EipLister.go diff --git a/cmd/main.go b/cmd/main.go index 4620822..f5e3306 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -88,7 +88,7 @@ func assignAddress(c context.Context, log *logrus.Entry, assigner address.Assign defer ticker.Stop() for { - err := assigner.Assign(node.Instance, node.Zone, cfg.Filter, cfg.OrderBy) + err := assigner.Assign(ctx, node.Instance, node.Zone, cfg.Filter, cfg.OrderBy) if err != nil { log.WithError(err).Errorf("failed to assign static public IP address to node %s", node.Name) if retryCounter < cfg.RetryAttempts { @@ -176,6 +176,9 @@ func runCmd(c *cli.Context) error { func main() { app := &cli.App{ + // use ";" instead of "," for slice flag separator + // AWS filter values can contain "," and shorthand filter format uses "," to separate Names and Values + SliceFlagSeparator: ";", Commands: []*cli.Command{ { Name: "run", diff --git a/cmd/main_test.go b/cmd/main_test.go index 113f6eb..3bd62b6 100644 --- a/cmd/main_test.go +++ b/cmd/main_test.go @@ -10,6 +10,7 @@ import ( "github.com/doitintl/kubeip/internal/types" mocks "github.com/doitintl/kubeip/mocks/address" "github.com/pkg/errors" + tmock "github.com/stretchr/testify/mock" ) func Test_assignAddress(t *testing.T) { @@ -30,7 +31,7 @@ func Test_assignAddress(t *testing.T) { c: context.Background(), assignerFn: func(t *testing.T) address.Assigner { mock := mocks.NewAssigner(t) - mock.EXPECT().Assign("test-instance", "test-zone", []string{"test-filter"}, "test-order-by").Return(nil) + mock.EXPECT().Assign(tmock.Anything, "test-instance", "test-zone", []string{"test-filter"}, "test-order-by").Return(nil) return mock }, node: &types.Node{ @@ -53,9 +54,9 @@ func Test_assignAddress(t *testing.T) { c: context.Background(), assignerFn: func(t *testing.T) address.Assigner { mock := mocks.NewAssigner(t) - mock.EXPECT().Assign("test-instance", "test-zone", []string{"test-filter"}, "test-order-by").Return(errors.New("first error")).Once() - mock.EXPECT().Assign("test-instance", "test-zone", []string{"test-filter"}, "test-order-by").Return(errors.New("second error")).Once() - mock.EXPECT().Assign("test-instance", "test-zone", []string{"test-filter"}, "test-order-by").Return(nil).Once() + mock.EXPECT().Assign(tmock.Anything, "test-instance", "test-zone", []string{"test-filter"}, "test-order-by").Return(errors.New("first error")).Once() + mock.EXPECT().Assign(tmock.Anything, "test-instance", "test-zone", []string{"test-filter"}, "test-order-by").Return(errors.New("second error")).Once() + mock.EXPECT().Assign(tmock.Anything, "test-instance", "test-zone", []string{"test-filter"}, "test-order-by").Return(nil).Once() return mock }, node: &types.Node{ @@ -78,7 +79,7 @@ func Test_assignAddress(t *testing.T) { c: context.Background(), assignerFn: func(t *testing.T) address.Assigner { mock := mocks.NewAssigner(t) - mock.EXPECT().Assign("test-instance", "test-zone", []string{"test-filter"}, "test-order-by").Return(errors.New("error")).Times(4) + mock.EXPECT().Assign(tmock.Anything, "test-instance", "test-zone", []string{"test-filter"}, "test-order-by").Return(errors.New("error")).Times(4) return mock }, node: &types.Node{ @@ -105,7 +106,7 @@ func Test_assignAddress(t *testing.T) { }(), assignerFn: func(t *testing.T) address.Assigner { mock := mocks.NewAssigner(t) - mock.EXPECT().Assign("test-instance", "test-zone", []string{"test-filter"}, "test-order-by").Return(errors.New("error")).Maybe() + mock.EXPECT().Assign(tmock.Anything, "test-instance", "test-zone", []string{"test-filter"}, "test-order-by").Return(errors.New("error")).Maybe() return mock }, node: &types.Node{ diff --git a/go.mod b/go.mod index 885f9e0..70dd964 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,8 @@ go 1.21 require ( cloud.google.com/go/compute/metadata v0.2.3 + github.com/aws/aws-sdk-go-v2/config v1.18.44 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.124.0 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.8.3 @@ -17,6 +19,17 @@ require ( require ( cloud.google.com/go/compute v1.23.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.21.1 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.13.42 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.42 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.36 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.44 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.36 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.15.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.23.1 // indirect + github.com/aws/smithy-go v1.15.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect @@ -36,6 +49,7 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.3.1 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/imdario/mergo v0.3.6 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect diff --git a/go.sum b/go.sum index c7cc230..2547d5f 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,32 @@ cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdi cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/aws/aws-sdk-go-v2 v1.21.1 h1:wjHYshtPpYOZm+/mu3NhVgRRc0baM6LJZOmxPZ5Cwzs= +github.com/aws/aws-sdk-go-v2 v1.21.1/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM= +github.com/aws/aws-sdk-go-v2/config v1.18.44 h1:U10NQ3OxiY0dGGozmVIENIDnCT0W432PWxk2VO8wGnY= +github.com/aws/aws-sdk-go-v2/config v1.18.44/go.mod h1:pHxnQBldd0heEdJmolLBk78D1Bf69YnKLY3LOpFImlU= +github.com/aws/aws-sdk-go-v2/credentials v1.13.42 h1:KMkjpZqcMOwtRHChVlHdNxTUUAC6NC/b58mRZDIdcRg= +github.com/aws/aws-sdk-go-v2/credentials v1.13.42/go.mod h1:7ltKclhvEB8305sBhrpls24HGxORl6qgnQqSJ314Uw8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.12 h1:3j5lrl9kVQrJ1BU4O0z7MQ8sa+UXdiLuo4j0V+odNI8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.12/go.mod h1:JbFpcHDBdsex1zpIKuVRorZSQiZEyc3MykNCcjgz174= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.42 h1:817VqVe6wvwE46xXy6YF5RywvjOX6U2zRQQ6IbQFK0s= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.42/go.mod h1:oDfgXoBBmj+kXnqxDDnIDnC56QBosglKp8ftRCTxR+0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.36 h1:7ZApaXzWbo8slc+W5TynuUlB4z66g44h7uqa3/d/BsY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.36/go.mod h1:rwr4WnmFi3RJO0M4dxbJtgi9BPLMpVBMX1nUte5ha9U= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.44 h1:quOJOqlbSfeJTboXLjYXM1M9T52LBXqLoTPlmsKLpBo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.44/go.mod h1:LNy+P1+1LiRcCsVYr/4zG5n8zWFL0xsvZkOybjbftm8= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.124.0 h1:3VsdIKjFmyXFkKV21tgn49/dxSziWhjnx3YbqrDofXc= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.124.0/go.mod h1:f2AJtWtbonV7cSBVdxfs6e68cponNukbBDvzc4WIASo= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.36 h1:YXlm7LxwNlauqb2OrinWlcvtsflTzP8GaMvYfQBhoT4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.36/go.mod h1:ou9ffqJ9hKOVZmjlC6kQ6oROAyG1M4yBKzR+9BKbDwk= +github.com/aws/aws-sdk-go-v2/service/sso v1.15.1 h1:ZN3bxw9OYC5D6umLw6f57rNJfGfhg1DIAAcKpzyUTOE= +github.com/aws/aws-sdk-go-v2/service/sso v1.15.1/go.mod h1:PieckvBoT5HtyB9AsJRrYZFY2Z+EyfVM/9zG6gbV8DQ= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.2 h1:fSCCJuT5i6ht8TqGdZc5Q5K9pz/atrf7qH4iK5C9XzU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.2/go.mod h1:5eNtr+vNc5vVd92q7SJ+U/HszsIdhZBEyi9dkMRKsp8= +github.com/aws/aws-sdk-go-v2/service/sts v1.23.1 h1:ASNYk1ypWAxRhJjKS0jBnTUeDl7HROOpeSMu1xDA/I8= +github.com/aws/aws-sdk-go-v2/service/sts v1.23.1/go.mod h1:2cnsAhVT3mqusovc2stUSUrSBGTcX9nh8Tu6xh//2eI= +github.com/aws/smithy-go v1.15.0 h1:PS/durmlzvAFpQHDs4wi4sNNP9ExsqZh6IlfdHXgKK8= +github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -61,6 +87,7 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -79,6 +106,10 @@ github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56 github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= diff --git a/internal/address/assigner.go b/internal/address/assigner.go index 1bce70e..c3a24b4 100644 --- a/internal/address/assigner.go +++ b/internal/address/assigner.go @@ -9,7 +9,7 @@ import ( ) type Assigner interface { - Assign(instanceID, zone string, filter []string, orderBy string) error + Assign(ctx context.Context, instanceID, zone string, filter []string, orderBy string) error } type assigner struct { @@ -17,7 +17,7 @@ type assigner struct { func NewAssigner(ctx context.Context, logger *logrus.Entry, provider types.CloudProvider, cfg *config.Config) (Assigner, error) { if provider == types.CloudProviderAWS { - return &awsAssigner{}, nil + return NewAwsAssigner(ctx, logger, cfg.Region) } else if provider == types.CloudProviderAzure { return &azureAssigner{}, nil } else if provider == types.CloudProviderGCP { @@ -26,6 +26,6 @@ func NewAssigner(ctx context.Context, logger *logrus.Entry, provider types.Cloud return &assigner{}, nil } -func (a *assigner) Assign(_, _ string, _ []string, _ string) error { +func (a *assigner) Assign(_ context.Context, _, _ string, _ []string, _ string) error { return nil } diff --git a/internal/address/aws.go b/internal/address/aws.go index a859f05..a7dbee2 100644 --- a/internal/address/aws.go +++ b/internal/address/aws.go @@ -1,8 +1,213 @@ package address +import ( + "context" + "sort" + "strings" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/doitintl/kubeip/internal/cloud" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +const ( + shorthandFilterTokens = 2 +) + type awsAssigner struct { + region string + logger *logrus.Entry + instanceGetter cloud.Ec2InstanceGetter + eipLister cloud.EipLister + eipAssigner cloud.EipAssigner +} + +func NewAwsAssigner(ctx context.Context, logger *logrus.Entry, region string) (Assigner, error) { + // initialize AWS client + cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(region)) + if err != nil { + return nil, errors.Wrap(err, "failed to load AWS config") + } + + // create AWS client for EC2 service in the given region with default config and credentials + client := ec2.NewFromConfig(cfg) + + // initialize AWS instance getter + instanceGetter := cloud.NewEc2InstanceGetter(client) + + // initialize AWS elastic IP lister + eipLister := cloud.NewEipLister(client) + + // initialize AWS elastic IP assigner + eipAssigner := cloud.NewEipAssigner(client) + + return &awsAssigner{ + region: region, + logger: logger, + instanceGetter: instanceGetter, + eipLister: eipLister, + eipAssigner: eipAssigner, + }, nil } -func (a *awsAssigner) Assign(_, _ string, _ []string, _ string) error { +// parseShorthandFilter parses shorthand filter string into filter name and values +// shorthand filter format: Name=string,Values=string,string ... +// https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/describe-addresses.html#options +func parseShorthandFilter(filter string) (string, []string, error) { + // split filter by the first "," + exp := strings.SplitN(filter, ",", shorthandFilterTokens) + if len(exp) != shorthandFilterTokens { + return "", nil, errors.New("invalid filter format; supported format Name=string,Values=string,string,") + } + // get filter name + name := strings.Split(exp[0], "=") + if len(name) != 2 || name[0] != "Name" { + return "", nil, errors.New("invalid filter Name") + } + // get filter values + values := strings.Split(exp[1], "=") + if len(values) != 2 || values[0] != "Values" { + return "", nil, errors.New("invalid filter Values list") + } + listValues := strings.Split(values[1], ",") + return name[1], listValues, nil +} + +// sortAddressesByField sorts addresses by the given field +// if sortBy is Tag:, sort addresses by tag value +func sortAddressesByField(addresses []types.Address, sortBy string) { + // if sortBy is Tag:, sort addresses by tag value + if strings.HasPrefix(sortBy, "Tag:") { + key := strings.TrimPrefix(sortBy, "Tag:") + sort.Slice(addresses, func(i, j int) bool { + if addresses[i].Tags == nil { + return false + } + if addresses[j].Tags == nil { + return true + } + for _, tag := range addresses[i].Tags { + if *tag.Key == key { + for _, tag2 := range addresses[j].Tags { + if *tag2.Key == key { + return *tag.Value < *tag2.Value + } + } + } + } + return false + }) + return // return if sortBy is Tag: + } + // sort addresses by orderBy field + switch sortBy { + case "AllocationId": + sort.Slice(addresses, func(i, j int) bool { + return *addresses[i].AllocationId < *addresses[j].AllocationId + }) + case "AssociationId": + sort.Slice(addresses, func(i, j int) bool { + return *addresses[i].AssociationId < *addresses[j].AssociationId + }) + case "Domain": + sort.Slice(addresses, func(i, j int) bool { + return addresses[i].Domain < addresses[j].Domain + }) + case "InstanceId": + sort.Slice(addresses, func(i, j int) bool { + return *addresses[i].InstanceId < *addresses[j].InstanceId + }) + case "NetworkInterfaceId": + sort.Slice(addresses, func(i, j int) bool { + return *addresses[i].NetworkInterfaceId < *addresses[j].NetworkInterfaceId + }) + case "NetworkInterfaceOwnerId": + sort.Slice(addresses, func(i, j int) bool { + return *addresses[i].NetworkInterfaceOwnerId < *addresses[j].NetworkInterfaceOwnerId + }) + case "PrivateIpAddress": + sort.Slice(addresses, func(i, j int) bool { + return *addresses[i].PrivateIpAddress < *addresses[j].PrivateIpAddress + }) + case "PublicIp": + sort.Slice(addresses, func(i, j int) bool { + return *addresses[i].PublicIp < *addresses[j].PublicIp + }) + case "PublicIpv4Pool": + sort.Slice(addresses, func(i, j int) bool { + return *addresses[i].PublicIpv4Pool < *addresses[j].PublicIpv4Pool + }) + } +} + +func (a *awsAssigner) Assign(ctx context.Context, instanceID, zone string, filter []string, orderBy string) error { + // get elastic IP attached to the instance + filters := make(map[string][]string) + filters["instance-id"] = []string{instanceID} + addresses, err := a.eipLister.List(ctx, filters, true) + if err != nil { + return errors.Wrapf(err, "failed to list elastic IPs attached to instance %s", instanceID) + } + if len(addresses) > 0 { + a.logger.Infof("elastic IP %s is already attached to instance %s", *addresses[0].PublicIp, instanceID) + return nil + } + + // get available elastic IPs + filters = make(map[string][]string) + for _, f := range filter { + name, values, err2 := parseShorthandFilter(f) + if err2 != nil { + return errors.Wrapf(err2, "failed to parse filter %s", f) + } + filters[name] = values + } + addresses, err = a.eipLister.List(context.Background(), filters, false) + if err != nil { + return errors.Wrap(err, "failed to list available elastic IPs") + } + + // if no available elastic IPs, return error + if len(addresses) == 0 { + return errors.Errorf("no available elastic IPs") + } + + // get EC2 instance + instance, err := a.instanceGetter.Get(ctx, instanceID, a.region) + if err != nil { + return errors.Wrapf(err, "failed to get instance %s", instanceID) + } + // get network interface ID + if instance.NetworkInterfaces == nil || len(instance.NetworkInterfaces) == 0 { + return errors.Errorf("no network interfaces found for instance %s", instanceID) + } + // get network interface ID of network interface with public IP address + networkInterfaceID := "" + for _, ni := range instance.NetworkInterfaces { + if ni.Association != nil && ni.Association.PublicIp != nil { + networkInterfaceID = *ni.NetworkInterfaceId + } + } + if networkInterfaceID == "" { + return errors.Errorf("no network interfaces with public IP address found for instance %s", instanceID) + } + + // sort addresses by orderBy field + sortAddressesByField(addresses, orderBy) + + // assign the first available elastic IP to the instance + address := addresses[0] + if err = a.eipAssigner.Assign(ctx, instanceID, networkInterfaceID, &address); err != nil { + return errors.Wrap(err, "failed to assign elastic IP") + } + a.logger.WithFields(logrus.Fields{ + "instance": instanceID, + "address": *address.PublicIp, + "allocation_id": *address.AllocationId, + }).Info("elastic IP assigned to the instance") + return nil } diff --git a/internal/address/azure.go b/internal/address/azure.go index 46ed863..4e19b14 100644 --- a/internal/address/azure.go +++ b/internal/address/azure.go @@ -1,8 +1,10 @@ package address +import "context" + type azureAssigner struct { } -func (a *azureAssigner) Assign(_, _ string, _ []string, _ string) error { +func (a *azureAssigner) Assign(_ context.Context, _, _ string, _ []string, _ string) error { return nil } diff --git a/internal/address/gcp.go b/internal/address/gcp.go index 1f2cbb2..c2ff630 100644 --- a/internal/address/gcp.go +++ b/internal/address/gcp.go @@ -71,13 +71,13 @@ func NewGCPAssigner(ctx context.Context, logger *logrus.Entry, project, region s }, nil } -func (a *gcpAssigner) waitForOperation(op *compute.Operation, zone string, timeout time.Duration) error { +func (a *gcpAssigner) waitForOperation(c context.Context, op *compute.Operation, zone string, timeout time.Duration) error { if op == nil { a.logger.Warn("operation is nil") return nil } // Create a context that will be cancelled with timeout - ctx, cancel := context.WithTimeout(context.Background(), timeout) + ctx, cancel := context.WithTimeout(c, timeout) defer cancel() var err error @@ -100,7 +100,7 @@ func (a *gcpAssigner) waitForOperation(op *compute.Operation, zone string, timeo return nil } -func (a *gcpAssigner) deleteInstanceAddress(instance *compute.Instance, zone string) error { +func (a *gcpAssigner) deleteInstanceAddress(ctx context.Context, instance *compute.Instance, zone string) error { // Check if the instance has at least one network interface if len(instance.NetworkInterfaces) == 0 { a.logger.WithField("instance", instance.Name).Info("instance has no network interfaces") @@ -126,13 +126,13 @@ func (a *gcpAssigner) deleteInstanceAddress(instance *compute.Instance, zone str return errors.Wrapf(err, "failed to delete access config %s from instance %s", accessConfigName, instance.Name) } // wait for operation to complete - if err = a.waitForOperation(op, zone, defaultTimeout); err != nil { + if err = a.waitForOperation(ctx, op, zone, defaultTimeout); err != nil { return errors.Wrapf(err, "failed to wait for operation %s", op.Name) } return nil } -func (a *gcpAssigner) addInstanceAddress(instance *compute.Instance, zone string, address *compute.Address) error { +func (a *gcpAssigner) addInstanceAddress(ctx context.Context, instance *compute.Instance, zone string, address *compute.Address) error { // add instance network interface access config a.logger.WithFields(logrus.Fields{ "instance": instance.Name, @@ -148,13 +148,13 @@ func (a *gcpAssigner) addInstanceAddress(instance *compute.Instance, zone string return errors.Wrapf(err, "failed to add access config %s to instance %s", address.Name, instance.Name) } // wait for operation to complete - if err = a.waitForOperation(op, zone, defaultTimeout); err != nil { + if err = a.waitForOperation(ctx, op, zone, defaultTimeout); err != nil { return errors.Wrapf(err, "failed to wait for operation %s", op.Name) } return nil } -func (a *gcpAssigner) Assign(instanceID, zone string, filter []string, orderBy string) error { +func (a *gcpAssigner) Assign(ctx context.Context, instanceID, zone string, filter []string, orderBy string) error { // check if instance already has a public static IP address assigned instance, err := a.instanceGetter.Get(a.project, zone, instanceID) if err != nil { @@ -188,13 +188,13 @@ func (a *gcpAssigner) Assign(instanceID, zone string, filter []string, orderBy s } // delete current ephemeral public IP address - if err = a.deleteInstanceAddress(instance, zone); err != nil { + if err = a.deleteInstanceAddress(ctx, instance, zone); err != nil { return errors.Wrap(err, "failed to delete current public IP address") } // assign first available static public IP address to the instance address := addresses[0] - if err = a.addInstanceAddress(instance, zone, address); err != nil { + if err = a.addInstanceAddress(ctx, instance, zone, address); err != nil { return errors.Wrap(err, "failed to assign static public IP address") } diff --git a/internal/address/gcp_test.go b/internal/address/gcp_test.go index 80f39fe..8a2b76a 100644 --- a/internal/address/gcp_test.go +++ b/internal/address/gcp_test.go @@ -250,7 +250,7 @@ func Test_gcpAssigner_waitForOperation(t *testing.T) { project: tt.fields.project, logger: logger, } - if err := a.waitForOperation(tt.args.op, tt.args.zone, tt.args.timeout); (err != nil) != tt.wantErr { + if err := a.waitForOperation(context.TODO(), tt.args.op, tt.args.zone, tt.args.timeout); (err != nil) != tt.wantErr { t.Errorf("waitForOperation() error = %v, wantErr %v", err, tt.wantErr) } }) diff --git a/internal/cloud/aws_address.go b/internal/cloud/aws_address.go new file mode 100644 index 0000000..718c63b --- /dev/null +++ b/internal/cloud/aws_address.go @@ -0,0 +1,37 @@ +package cloud + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/pkg/errors" +) + +type EipAssigner interface { + Assign(ctx context.Context, region, instanceID string, address *types.Address) error +} + +type eipAssigner struct { + client *ec2.Client +} + +func NewEipAssigner(client *ec2.Client) EipAssigner { + return &eipAssigner{client: client} +} + +func (a *eipAssigner) Assign(ctx context.Context, instanceID, networkInterfaceID string, address *types.Address) error { + // associate elastic IP with the instance + input := &ec2.AssociateAddressInput{ + AllocationId: address.AllocationId, + InstanceId: &instanceID, + NetworkInterfaceId: &networkInterfaceID, + } + + _, err := a.client.AssociateAddress(ctx, input) + if err != nil { + return errors.Wrap(err, "failed to associate elastic IP with the instance") + } + + return nil +} diff --git a/internal/cloud/aws_instance.go b/internal/cloud/aws_instance.go new file mode 100644 index 0000000..45b521e --- /dev/null +++ b/internal/cloud/aws_instance.go @@ -0,0 +1,40 @@ +package cloud + +import ( + "context" + "fmt" + + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/ec2/types" +) + +type Ec2InstanceGetter interface { + Get(ctx context.Context, instanceID, region string) (*types.Instance, error) +} + +type ec2InstanceGetter struct { + client *ec2.Client +} + +func NewEc2InstanceGetter(client *ec2.Client) Ec2InstanceGetter { + return &ec2InstanceGetter{client: client} +} + +func (g *ec2InstanceGetter) Get(ctx context.Context, instanceID, region string) (*types.Instance, error) { + input := &ec2.DescribeInstancesInput{ + InstanceIds: []string{ + instanceID, + }, + } + + resp, err := g.client.DescribeInstances(ctx, input) + if err != nil { + return nil, fmt.Errorf("failed to describe instances, %v", err) + } + + if len(resp.Reservations) == 0 || len(resp.Reservations[0].Instances) == 0 { + return nil, fmt.Errorf("no instances found for the given id") + } + + return &resp.Reservations[0].Instances[0], nil +} diff --git a/internal/cloud/aws_lister.go b/internal/cloud/aws_lister.go new file mode 100644 index 0000000..64247b6 --- /dev/null +++ b/internal/cloud/aws_lister.go @@ -0,0 +1,53 @@ +package cloud + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/pkg/errors" +) + +type EipLister interface { + List(ctx context.Context, filter map[string][]string, inUse bool) ([]types.Address, error) +} + +type eipLister struct { + client *ec2.Client +} + +func NewEipLister(client *ec2.Client) EipLister { + return &eipLister{client: client} +} + +func (l *eipLister) List(ctx context.Context, filter map[string][]string, inUse bool) ([]types.Address, error) { + // create filter for DescribeAddressesInput + filters := make([]types.Filter, 0, len(filter)+1) + for k, v := range filter { + key := k + filters = append(filters, types.Filter{ + Name: &key, + Values: v, + }) + } + + // list all elastic IPs in the region matching the filter + input := &ec2.DescribeAddressesInput{ + Filters: filters, + } + list, err := l.client.DescribeAddresses(ctx, input) + if err != nil { + return nil, errors.Wrap(err, "failed to list elastic IPs") + } + + filtered := make([]types.Address, 0, len(list.Addresses)) + // API does not support filtering by association ID equal to nil + // filter addresses based on whether they are in use or not + for _, address := range list.Addresses { + if (inUse && address.AssociationId != nil) || (!inUse && address.AssociationId == nil) { + filtered = append(filtered, address) + } + } + + return filtered, nil +} diff --git a/mocks/address/Assigner.go b/mocks/address/Assigner.go index bddc658..290b42a 100644 --- a/mocks/address/Assigner.go +++ b/mocks/address/Assigner.go @@ -2,7 +2,11 @@ package mocks -import mock "github.com/stretchr/testify/mock" +import ( + context "context" + + mock "github.com/stretchr/testify/mock" +) // Assigner is an autogenerated mock type for the Assigner type type Assigner struct { @@ -17,13 +21,13 @@ func (_m *Assigner) EXPECT() *Assigner_Expecter { return &Assigner_Expecter{mock: &_m.Mock} } -// Assign provides a mock function with given fields: instanceID, zone, filter, orderBy -func (_m *Assigner) Assign(instanceID string, zone string, filter []string, orderBy string) error { - ret := _m.Called(instanceID, zone, filter, orderBy) +// Assign provides a mock function with given fields: ctx, instanceID, zone, filter, orderBy +func (_m *Assigner) Assign(ctx context.Context, instanceID string, zone string, filter []string, orderBy string) error { + ret := _m.Called(ctx, instanceID, zone, filter, orderBy) var r0 error - if rf, ok := ret.Get(0).(func(string, string, []string, string) error); ok { - r0 = rf(instanceID, zone, filter, orderBy) + if rf, ok := ret.Get(0).(func(context.Context, string, string, []string, string) error); ok { + r0 = rf(ctx, instanceID, zone, filter, orderBy) } else { r0 = ret.Error(0) } @@ -37,17 +41,18 @@ type Assigner_Assign_Call struct { } // Assign is a helper method to define mock.On call +// - ctx context.Context // - instanceID string // - zone string // - filter []string // - orderBy string -func (_e *Assigner_Expecter) Assign(instanceID interface{}, zone interface{}, filter interface{}, orderBy interface{}) *Assigner_Assign_Call { - return &Assigner_Assign_Call{Call: _e.mock.On("Assign", instanceID, zone, filter, orderBy)} +func (_e *Assigner_Expecter) Assign(ctx interface{}, instanceID interface{}, zone interface{}, filter interface{}, orderBy interface{}) *Assigner_Assign_Call { + return &Assigner_Assign_Call{Call: _e.mock.On("Assign", ctx, instanceID, zone, filter, orderBy)} } -func (_c *Assigner_Assign_Call) Run(run func(instanceID string, zone string, filter []string, orderBy string)) *Assigner_Assign_Call { +func (_c *Assigner_Assign_Call) Run(run func(ctx context.Context, instanceID string, zone string, filter []string, orderBy string)) *Assigner_Assign_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(string), args[1].(string), args[2].([]string), args[3].(string)) + run(args[0].(context.Context), args[1].(string), args[2].(string), args[3].([]string), args[4].(string)) }) return _c } @@ -57,7 +62,7 @@ func (_c *Assigner_Assign_Call) Return(_a0 error) *Assigner_Assign_Call { return _c } -func (_c *Assigner_Assign_Call) RunAndReturn(run func(string, string, []string, string) error) *Assigner_Assign_Call { +func (_c *Assigner_Assign_Call) RunAndReturn(run func(context.Context, string, string, []string, string) error) *Assigner_Assign_Call { _c.Call.Return(run) return _c } diff --git a/mocks/cloud/Ec2InstanceGetter.go b/mocks/cloud/Ec2InstanceGetter.go new file mode 100644 index 0000000..f94bd73 --- /dev/null +++ b/mocks/cloud/Ec2InstanceGetter.go @@ -0,0 +1,93 @@ +// Code generated by mockery v2.30.16. DO NOT EDIT. + +package mocks + +import ( + context "context" + + types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + mock "github.com/stretchr/testify/mock" +) + +// Ec2InstanceGetter is an autogenerated mock type for the Ec2InstanceGetter type +type Ec2InstanceGetter struct { + mock.Mock +} + +type Ec2InstanceGetter_Expecter struct { + mock *mock.Mock +} + +func (_m *Ec2InstanceGetter) EXPECT() *Ec2InstanceGetter_Expecter { + return &Ec2InstanceGetter_Expecter{mock: &_m.Mock} +} + +// Get provides a mock function with given fields: ctx, instanceID, region +func (_m *Ec2InstanceGetter) Get(ctx context.Context, instanceID string, region string) (*types.Instance, error) { + ret := _m.Called(ctx, instanceID, region) + + var r0 *types.Instance + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) (*types.Instance, error)); ok { + return rf(ctx, instanceID, region) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string) *types.Instance); ok { + r0 = rf(ctx, instanceID, region) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.Instance) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { + r1 = rf(ctx, instanceID, region) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Ec2InstanceGetter_Get_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Get' +type Ec2InstanceGetter_Get_Call struct { + *mock.Call +} + +// Get is a helper method to define mock.On call +// - ctx context.Context +// - instanceID string +// - region string +func (_e *Ec2InstanceGetter_Expecter) Get(ctx interface{}, instanceID interface{}, region interface{}) *Ec2InstanceGetter_Get_Call { + return &Ec2InstanceGetter_Get_Call{Call: _e.mock.On("Get", ctx, instanceID, region)} +} + +func (_c *Ec2InstanceGetter_Get_Call) Run(run func(ctx context.Context, instanceID string, region string)) *Ec2InstanceGetter_Get_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string)) + }) + return _c +} + +func (_c *Ec2InstanceGetter_Get_Call) Return(_a0 *types.Instance, _a1 error) *Ec2InstanceGetter_Get_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Ec2InstanceGetter_Get_Call) RunAndReturn(run func(context.Context, string, string) (*types.Instance, error)) *Ec2InstanceGetter_Get_Call { + _c.Call.Return(run) + return _c +} + +// NewEc2InstanceGetter creates a new instance of Ec2InstanceGetter. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewEc2InstanceGetter(t interface { + mock.TestingT + Cleanup(func()) +}) *Ec2InstanceGetter { + mock := &Ec2InstanceGetter{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/cloud/EipAssigner.go b/mocks/cloud/EipAssigner.go new file mode 100644 index 0000000..668862f --- /dev/null +++ b/mocks/cloud/EipAssigner.go @@ -0,0 +1,82 @@ +// Code generated by mockery v2.30.16. DO NOT EDIT. + +package mocks + +import ( + context "context" + + types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + mock "github.com/stretchr/testify/mock" +) + +// EipAssigner is an autogenerated mock type for the EipAssigner type +type EipAssigner struct { + mock.Mock +} + +type EipAssigner_Expecter struct { + mock *mock.Mock +} + +func (_m *EipAssigner) EXPECT() *EipAssigner_Expecter { + return &EipAssigner_Expecter{mock: &_m.Mock} +} + +// Assign provides a mock function with given fields: ctx, region, instanceID, address +func (_m *EipAssigner) Assign(ctx context.Context, region string, instanceID string, address *types.Address) error { + ret := _m.Called(ctx, region, instanceID, address) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, *types.Address) error); ok { + r0 = rf(ctx, region, instanceID, address) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// EipAssigner_Assign_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Assign' +type EipAssigner_Assign_Call struct { + *mock.Call +} + +// Assign is a helper method to define mock.On call +// - ctx context.Context +// - region string +// - instanceID string +// - address *types.Address +func (_e *EipAssigner_Expecter) Assign(ctx interface{}, region interface{}, instanceID interface{}, address interface{}) *EipAssigner_Assign_Call { + return &EipAssigner_Assign_Call{Call: _e.mock.On("Assign", ctx, region, instanceID, address)} +} + +func (_c *EipAssigner_Assign_Call) Run(run func(ctx context.Context, region string, instanceID string, address *types.Address)) *EipAssigner_Assign_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string), args[3].(*types.Address)) + }) + return _c +} + +func (_c *EipAssigner_Assign_Call) Return(_a0 error) *EipAssigner_Assign_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *EipAssigner_Assign_Call) RunAndReturn(run func(context.Context, string, string, *types.Address) error) *EipAssigner_Assign_Call { + _c.Call.Return(run) + return _c +} + +// NewEipAssigner creates a new instance of EipAssigner. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewEipAssigner(t interface { + mock.TestingT + Cleanup(func()) +}) *EipAssigner { + mock := &EipAssigner{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/cloud/EipLister.go b/mocks/cloud/EipLister.go new file mode 100644 index 0000000..1cd651f --- /dev/null +++ b/mocks/cloud/EipLister.go @@ -0,0 +1,93 @@ +// Code generated by mockery v2.30.16. DO NOT EDIT. + +package mocks + +import ( + context "context" + + types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + mock "github.com/stretchr/testify/mock" +) + +// EipLister is an autogenerated mock type for the EipLister type +type EipLister struct { + mock.Mock +} + +type EipLister_Expecter struct { + mock *mock.Mock +} + +func (_m *EipLister) EXPECT() *EipLister_Expecter { + return &EipLister_Expecter{mock: &_m.Mock} +} + +// List provides a mock function with given fields: ctx, filter, inUse +func (_m *EipLister) List(ctx context.Context, filter map[string][]string, inUse bool) ([]types.Address, error) { + ret := _m.Called(ctx, filter, inUse) + + var r0 []types.Address + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, map[string][]string, bool) ([]types.Address, error)); ok { + return rf(ctx, filter, inUse) + } + if rf, ok := ret.Get(0).(func(context.Context, map[string][]string, bool) []types.Address); ok { + r0 = rf(ctx, filter, inUse) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]types.Address) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, map[string][]string, bool) error); ok { + r1 = rf(ctx, filter, inUse) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// EipLister_List_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'List' +type EipLister_List_Call struct { + *mock.Call +} + +// List is a helper method to define mock.On call +// - ctx context.Context +// - filter map[string][]string +// - inUse bool +func (_e *EipLister_Expecter) List(ctx interface{}, filter interface{}, inUse interface{}) *EipLister_List_Call { + return &EipLister_List_Call{Call: _e.mock.On("List", ctx, filter, inUse)} +} + +func (_c *EipLister_List_Call) Run(run func(ctx context.Context, filter map[string][]string, inUse bool)) *EipLister_List_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(map[string][]string), args[2].(bool)) + }) + return _c +} + +func (_c *EipLister_List_Call) Return(_a0 []types.Address, _a1 error) *EipLister_List_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *EipLister_List_Call) RunAndReturn(run func(context.Context, map[string][]string, bool) ([]types.Address, error)) *EipLister_List_Call { + _c.Call.Return(run) + return _c +} + +// NewEipLister creates a new instance of EipLister. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewEipLister(t interface { + mock.TestingT + Cleanup(func()) +}) *EipLister { + mock := &EipLister{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} From 3bee1dbac81416d7406be51815702ac8cc9f796e Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Thu, 12 Oct 2023 10:12:47 +0300 Subject: [PATCH 27/66] remove obsolete code --- internal/address/assigner.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/internal/address/assigner.go b/internal/address/assigner.go index c3a24b4..68f0ccd 100644 --- a/internal/address/assigner.go +++ b/internal/address/assigner.go @@ -2,6 +2,7 @@ package address import ( "context" + "errors" "github.com/doitintl/kubeip/internal/config" "github.com/doitintl/kubeip/internal/types" @@ -12,9 +13,6 @@ type Assigner interface { Assign(ctx context.Context, instanceID, zone string, filter []string, orderBy string) error } -type assigner struct { -} - func NewAssigner(ctx context.Context, logger *logrus.Entry, provider types.CloudProvider, cfg *config.Config) (Assigner, error) { if provider == types.CloudProviderAWS { return NewAwsAssigner(ctx, logger, cfg.Region) @@ -23,9 +21,5 @@ func NewAssigner(ctx context.Context, logger *logrus.Entry, provider types.Cloud } else if provider == types.CloudProviderGCP { return NewGCPAssigner(ctx, logger, cfg.Project, cfg.Region) } - return &assigner{}, nil -} - -func (a *assigner) Assign(_ context.Context, _, _ string, _ []string, _ string) error { - return nil + return nil, errors.New("unknown cloud provider") } From abe16ea996b233f60528034439870b2f59bdf726 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Thu, 12 Oct 2023 10:50:21 +0300 Subject: [PATCH 28/66] test address list sort --- internal/address/aws.go | 42 ++-- internal/address/aws_test.go | 417 +++++++++++++++++++++++++++++++++++ 2 files changed, 440 insertions(+), 19 deletions(-) create mode 100644 internal/address/aws_test.go diff --git a/internal/address/aws.go b/internal/address/aws.go index a7dbee2..dc79951 100644 --- a/internal/address/aws.go +++ b/internal/address/aws.go @@ -76,30 +76,34 @@ func parseShorthandFilter(filter string) (string, []string, error) { return name[1], listValues, nil } +func sortAddressesByTag(addresses []types.Address, key string) { + sort.Slice(addresses, func(i, j int) bool { + if addresses[i].Tags == nil { + return false + } + if addresses[j].Tags == nil { + return true + } + for _, tag := range addresses[i].Tags { + if *tag.Key == key { + for _, tag2 := range addresses[j].Tags { + if *tag2.Key == key { + return *tag.Value < *tag2.Value + } + } + } + } + return false + }) +} + // sortAddressesByField sorts addresses by the given field // if sortBy is Tag:, sort addresses by tag value func sortAddressesByField(addresses []types.Address, sortBy string) { // if sortBy is Tag:, sort addresses by tag value if strings.HasPrefix(sortBy, "Tag:") { key := strings.TrimPrefix(sortBy, "Tag:") - sort.Slice(addresses, func(i, j int) bool { - if addresses[i].Tags == nil { - return false - } - if addresses[j].Tags == nil { - return true - } - for _, tag := range addresses[i].Tags { - if *tag.Key == key { - for _, tag2 := range addresses[j].Tags { - if *tag2.Key == key { - return *tag.Value < *tag2.Value - } - } - } - } - return false - }) + sortAddressesByTag(addresses, key) return // return if sortBy is Tag: } // sort addresses by orderBy field @@ -143,7 +147,7 @@ func sortAddressesByField(addresses []types.Address, sortBy string) { } } -func (a *awsAssigner) Assign(ctx context.Context, instanceID, zone string, filter []string, orderBy string) error { +func (a *awsAssigner) Assign(ctx context.Context, instanceID, _ string, filter []string, orderBy string) error { // get elastic IP attached to the instance filters := make(map[string][]string) filters["instance-id"] = []string{instanceID} diff --git a/internal/address/aws_test.go b/internal/address/aws_test.go new file mode 100644 index 0000000..12cd97c --- /dev/null +++ b/internal/address/aws_test.go @@ -0,0 +1,417 @@ +package address + +import ( + "reflect" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ec2/types" +) + +func Test_sortAddressesByTag(t *testing.T) { + type args struct { + addresses []types.Address + key string + } + tests := []struct { + name string + args args + want []types.Address + }{ + { + name: "Test case 1: Sort addresses by tag value", + args: args{ + addresses: []types.Address{ + { + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("B"), + }, + }, + }, + { + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("A"), + }, + }, + }, + }, + key: "Name", + }, + want: []types.Address{ + { + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("A"), + }, + }, + }, + { + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("B"), + }, + }, + }, + }, + }, + { + name: "Test case 2: Addresses with no tags", + args: args{ + addresses: []types.Address{ + {}, + {}, + }, + key: "Name", + }, + want: []types.Address{ + {}, + {}, + }, + }, + { + name: "Test case 3: Key not found in tags", + args: args{ + addresses: []types.Address{ + { + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("B"), + }, + }, + }, + { + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("A"), + }, + }, + }, + }, + key: "NonExistentKey", + }, + want: []types.Address{ + { + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("B"), + }, + }, + }, + { + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("A"), + }, + }, + }, + }, + }, + { + name: "Test case 4: One address with tags, one without", + args: args{ + addresses: []types.Address{ + { + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("B"), + }, + }, + }, + {}, + }, + key: "Name", + }, + want: []types.Address{ + { + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("B"), + }, + }, + }, + {}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sortAddressesByTag(tt.args.addresses, tt.args.key) + if !reflect.DeepEqual(tt.args.addresses, tt.want) { + t.Errorf("sortAddressesByTag() = %v, want %v", tt.args.addresses, tt.want) + } + }) + } +} + +func Test_sortAddressesByField(t *testing.T) { + type args struct { + addresses []types.Address + sortBy string + } + tests := []struct { + name string + args args + want []types.Address + }{ + { + name: "Test case 1: Sort addresses by tag value", + args: args{ + addresses: []types.Address{ + { + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("B"), + }, + }, + }, + { + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("A"), + }, + }, + }, + }, + sortBy: "Tag:Name", + }, + want: []types.Address{ + { + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("A"), + }, + }, + }, + { + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("B"), + }, + }, + }, + }, + }, + { + name: "Test case 2: Sort addresses by AllocationId", + args: args{ + addresses: []types.Address{ + { + AllocationId: aws.String("b"), + }, + { + AllocationId: aws.String("a"), + }, + }, + sortBy: "AllocationId", + }, + want: []types.Address{ + { + AllocationId: aws.String("a"), + }, + { + AllocationId: aws.String("b"), + }, + }, + }, + { + name: "Test case 3: Sort addresses by PublicIp", + args: args{ + addresses: []types.Address{ + { + PublicIp: aws.String("192.168.1.2"), + }, + { + PublicIp: aws.String("192.168.1.1"), + }, + }, + sortBy: "PublicIp", + }, + want: []types.Address{ + { + PublicIp: aws.String("192.168.1.1"), + }, + { + PublicIp: aws.String("192.168.1.2"), + }, + }, + }, + { + name: "Test case 4: Sort addresses by InstanceId", + args: args{ + addresses: []types.Address{ + { + InstanceId: aws.String("i-0abcd1234efgh5678"), + }, + { + InstanceId: aws.String("i-0abcd1234efgh5679"), + }, + }, + sortBy: "InstanceId", + }, + want: []types.Address{ + { + InstanceId: aws.String("i-0abcd1234efgh5678"), + }, + { + InstanceId: aws.String("i-0abcd1234efgh5679"), + }, + }, + }, + { + name: "Test case 5: Sort addresses by Domain", + args: args{ + addresses: []types.Address{ + { + Domain: types.DomainTypeVpc, + }, + { + Domain: types.DomainTypeStandard, + }, + }, + sortBy: "Domain", + }, + want: []types.Address{ + { + Domain: types.DomainTypeStandard, + }, + { + Domain: types.DomainTypeVpc, + }, + }, + }, + { + name: "Test case 6: Sort addresses by NetworkInterfaceId", + args: args{ + addresses: []types.Address{ + { + NetworkInterfaceId: aws.String("eni-0abcd1234efgh5679"), + }, + { + NetworkInterfaceId: aws.String("eni-0abcd1234efgh5678"), + }, + }, + sortBy: "NetworkInterfaceId", + }, + want: []types.Address{ + { + NetworkInterfaceId: aws.String("eni-0abcd1234efgh5678"), + }, + { + NetworkInterfaceId: aws.String("eni-0abcd1234efgh5679"), + }, + }, + }, + { + name: "Test case 7: Sort addresses by NetworkInterfaceOwnerId", + args: args{ + addresses: []types.Address{ + { + NetworkInterfaceOwnerId: aws.String("123456789013"), + }, + { + NetworkInterfaceOwnerId: aws.String("123456789012"), + }, + }, + sortBy: "NetworkInterfaceOwnerId", + }, + want: []types.Address{ + { + NetworkInterfaceOwnerId: aws.String("123456789012"), + }, + { + NetworkInterfaceOwnerId: aws.String("123456789013"), + }, + }, + }, + { + name: "Test case 8: Sort addresses by AssociationId", + args: args{ + addresses: []types.Address{ + { + AssociationId: aws.String("b"), + }, + { + AssociationId: aws.String("a"), + }, + }, + sortBy: "AssociationId", + }, + want: []types.Address{ + { + AssociationId: aws.String("a"), + }, + { + AssociationId: aws.String("b"), + }, + }, + }, + { + name: "Test case 9: Sort addresses by PrivateIpAddress", + args: args{ + addresses: []types.Address{ + { + PrivateIpAddress: aws.String("10.10.0.3"), + }, + { + PrivateIpAddress: aws.String("10.10.0.1"), + }, + }, + sortBy: "PrivateIpAddress", + }, + want: []types.Address{ + { + PrivateIpAddress: aws.String("10.10.0.1"), + }, + { + PrivateIpAddress: aws.String("10.10.0.3"), + }, + }, + }, + { + name: "Test case 10: Sort addresses by PublicIpv4Pool", + args: args{ + addresses: []types.Address{ + { + PublicIpv4Pool: aws.String("amazon"), + }, + { + PublicIpv4Pool: aws.String("aws"), + }, + }, + sortBy: "PublicIpv4Pool", + }, + want: []types.Address{ + { + PublicIpv4Pool: aws.String("amazon"), + }, + { + PublicIpv4Pool: aws.String("aws"), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sortAddressesByField(tt.args.addresses, tt.args.sortBy) + if !reflect.DeepEqual(tt.args.addresses, tt.want) { + t.Errorf("sortAddressesByField() = %v, want %v", tt.args.addresses, tt.want) + } + }) + } +} From f393e7e3dd09f643656bee43bc7ea33bb214da35 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Thu, 12 Oct 2023 11:50:18 +0300 Subject: [PATCH 29/66] test aws eip assign --- internal/address/aws_test.go | 153 +++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) diff --git a/internal/address/aws_test.go b/internal/address/aws_test.go index 12cd97c..036f3fb 100644 --- a/internal/address/aws_test.go +++ b/internal/address/aws_test.go @@ -1,11 +1,16 @@ package address import ( + "context" "reflect" "testing" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/doitintl/kubeip/internal/cloud" + mocks "github.com/doitintl/kubeip/mocks/cloud" + "github.com/sirupsen/logrus" + tmock "github.com/stretchr/testify/mock" ) func Test_sortAddressesByTag(t *testing.T) { @@ -415,3 +420,151 @@ func Test_sortAddressesByField(t *testing.T) { }) } } + +func Test_awsAssigner_Assign(t *testing.T) { + type args struct { + ctx context.Context + instanceID string + filter []string + orderBy string + } + type fields struct { + region string + logger *logrus.Entry + instanceGetterFn func(t *testing.T, args *args) cloud.Ec2InstanceGetter + eipListerFn func(t *testing.T, args *args) cloud.EipLister + eipAssignerFn func(t *testing.T, args *args) cloud.EipAssigner + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "assign EIP to instance", + fields: fields{ + region: "us-east-1", + logger: logrus.NewEntry(logrus.New()), + instanceGetterFn: func(t *testing.T, args *args) cloud.Ec2InstanceGetter { + mock := mocks.NewEc2InstanceGetter(t) + mock.EXPECT().Get(args.ctx, args.instanceID, "us-east-1").Return(&types.Instance{ + InstanceId: aws.String(args.instanceID), + NetworkInterfaces: []types.InstanceNetworkInterface{ + { + Association: &types.InstanceNetworkInterfaceAssociation{ + PublicIp: aws.String("135.64.10.1"), + }, + NetworkInterfaceId: aws.String("eni-0abcd1234efgh5678"), + }, + }, + }, nil) + return mock + }, + eipListerFn: func(t *testing.T, args *args) cloud.EipLister { + mock := mocks.NewEipLister(t) + mock.EXPECT().List(args.ctx, map[string][]string{ + "instance-id": {args.instanceID}, + }, true).Return([]types.Address{}, nil).Once() + mock.EXPECT().List(args.ctx, map[string][]string{ + "tag:env": {"test"}, + "tag:kubeip": {"reserved"}, + }, false).Return([]types.Address{ + { + AllocationId: aws.String("eipalloc-0abcd1234efgh5678"), + PublicIp: aws.String("100.0.0.1"), + Tags: []types.Tag{ + { + Key: aws.String("env"), + Value: aws.String("test"), + }, + { + Key: aws.String("kubeip"), + Value: aws.String("reserved"), + }, + }, + }, + { + AllocationId: aws.String("eipalloc-0abcd1234efgh5679"), + PublicIp: aws.String("100.0.0.2"), + Tags: []types.Tag{ + { + Key: aws.String("env"), + Value: aws.String("test"), + }, + { + Key: aws.String("kubeip"), + Value: aws.String("reserved"), + }, + }, + }, + }, nil).Once() + return mock + }, + eipAssignerFn: func(t *testing.T, args *args) cloud.EipAssigner { + mock := mocks.NewEipAssigner(t) + mock.EXPECT().Assign(args.ctx, args.instanceID, "eni-0abcd1234efgh5678", tmock.Anything).Return(nil) + return mock + }, + }, + args: args{ + ctx: context.Background(), + instanceID: "i-0abcd1234efgh5678", + filter: []string{ + "Name=tag:env,Values=test", + "Name=tag:kubeip,Values=reserved", + }, + orderBy: "PublicIp", + }, + }, + { + name: "instance already has EIP assigned", + fields: fields{ + region: "us-east-1", + logger: logrus.NewEntry(logrus.New()), + instanceGetterFn: func(t *testing.T, args *args) cloud.Ec2InstanceGetter { + return nil + }, + eipAssignerFn: func(t *testing.T, args *args) cloud.EipAssigner { + return nil + }, + eipListerFn: func(t *testing.T, args *args) cloud.EipLister { + mock := mocks.NewEipLister(t) + mock.EXPECT().List(args.ctx, map[string][]string{ + "instance-id": {args.instanceID}, + }, true).Return([]types.Address{ + { + AllocationId: aws.String("eipalloc-0abcd1234efgh5678"), + PublicIp: aws.String("100.0.0.1"), + Tags: []types.Tag{ + { + Key: aws.String("env"), + Value: aws.String("test"), + }, + }, + }, + }, nil) + return mock + }, + }, + args: args{ + ctx: context.Background(), + instanceID: "i-0abcd1234efgh5678", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + a := &awsAssigner{ + region: tt.fields.region, + logger: tt.fields.logger, + instanceGetter: tt.fields.instanceGetterFn(t, &tt.args), + eipLister: tt.fields.eipListerFn(t, &tt.args), + eipAssigner: tt.fields.eipAssignerFn(t, &tt.args), + } + if err := a.Assign(tt.args.ctx, tt.args.instanceID, "", tt.args.filter, tt.args.orderBy); (err != nil) != tt.wantErr { + t.Errorf("Assign() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} From b5e7a98197371d9a3cd33027ad9fde76727a37a8 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Thu, 12 Oct 2023 13:18:06 +0300 Subject: [PATCH 30/66] release static IP on graceful shutdown (optional) --- cmd/main.go | 15 +++++++++++- internal/address/assigner.go | 1 + internal/address/aws.go | 27 +++++++++++++++++++++ internal/address/azure.go | 4 ++++ internal/address/gcp.go | 31 +++++++++++++++++++++++- internal/cloud/aws_address.go | 15 ++++++++++++ internal/config/config.go | 3 +++ mocks/address/Assigner.go | 44 +++++++++++++++++++++++++++++++++++ mocks/cloud/EipAssigner.go | 43 ++++++++++++++++++++++++++++++++++ 9 files changed, 181 insertions(+), 2 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index f5e3306..9a92bd3 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -102,7 +102,6 @@ func assignAddress(c context.Context, log *logrus.Entry, assigner address.Assign case <-ticker.C: continue case <-ctx.Done(): - log.Infof("kubeip agent stopped") return errors.Wrap(err, "context is done") } } @@ -157,12 +156,19 @@ func run(c context.Context, log *logrus.Entry, cfg *config.Config) error { } case <-ctx.Done(): log.Infof("kubeip agent stopped") + if cfg.ReleaseOnExit { + log.Infof("releasing static public IP address") + if err = assigner.Unassign(ctx, n.Instance, n.Zone); err != nil { + return errors.Wrap(err, "releasing static public IP address") + } + } } return nil } func runCmd(c *cli.Context) error { + // setup signal handler for graceful shutdown: SIGTERM, SIGINT ctx := signals.SetupSignalHandler() log := prepareLogger(c.String("log-level"), c.Bool("json")) cfg := config.NewConfig(c) @@ -234,6 +240,13 @@ func main() { EnvVars: []string{"RETRY_ATTEMPTS"}, Category: "Configuration", }, + &cli.BoolFlag{ + Name: "release-on-exit", + Usage: "release the static public IP address on exit", + EnvVars: []string{"RELEASE_ON_EXIT"}, + Category: "Configuration", + Value: true, + }, &cli.StringFlag{ Name: "log-level", Usage: "set log level (debug, info(*), warning, error, fatal, panic)", diff --git a/internal/address/assigner.go b/internal/address/assigner.go index 68f0ccd..2f64732 100644 --- a/internal/address/assigner.go +++ b/internal/address/assigner.go @@ -11,6 +11,7 @@ import ( type Assigner interface { Assign(ctx context.Context, instanceID, zone string, filter []string, orderBy string) error + Unassign(ctx context.Context, instanceID, zone string) error } func NewAssigner(ctx context.Context, logger *logrus.Entry, provider types.CloudProvider, cfg *config.Config) (Assigner, error) { diff --git a/internal/address/aws.go b/internal/address/aws.go index dc79951..0f7f2ae 100644 --- a/internal/address/aws.go +++ b/internal/address/aws.go @@ -215,3 +215,30 @@ func (a *awsAssigner) Assign(ctx context.Context, instanceID, _ string, filter [ return nil } + +func (a *awsAssigner) Unassign(ctx context.Context, instanceID, _ string) error { + // get elastic IP attached to the instance + filters := make(map[string][]string) + filters["instance-id"] = []string{instanceID} + addresses, err := a.eipLister.List(ctx, filters, true) + if err != nil { + return errors.Wrapf(err, "failed to list elastic IPs attached to instance %s", instanceID) + } + if len(addresses) == 0 { + a.logger.Infof("no elastic IP attached to instance %s", instanceID) + return nil + } + + // unassign elastic IP from the instance + address := addresses[0] + if err = a.eipAssigner.Unassign(ctx, &address); err != nil { + return errors.Wrap(err, "failed to unassign elastic IP") + } + a.logger.WithFields(logrus.Fields{ + "instance": instanceID, + "address": *address.PublicIp, + "allocation_id": *address.AllocationId, + }).Info("elastic IP unassigned from the instance") + + return nil +} diff --git a/internal/address/azure.go b/internal/address/azure.go index 4e19b14..79d870f 100644 --- a/internal/address/azure.go +++ b/internal/address/azure.go @@ -8,3 +8,7 @@ type azureAssigner struct { func (a *azureAssigner) Assign(_ context.Context, _, _ string, _ []string, _ string) error { return nil } + +func (a *azureAssigner) Unassign(_ context.Context, _, _ string) error { + return nil +} diff --git a/internal/address/gcp.go b/internal/address/gcp.go index c2ff630..1d07b9b 100644 --- a/internal/address/gcp.go +++ b/internal/address/gcp.go @@ -120,7 +120,7 @@ func (a *gcpAssigner) deleteInstanceAddress(ctx context.Context, instance *compu a.logger.WithFields(logrus.Fields{ "instance": instance.Name, "address": accessConfig.NatIP, - }).Infof("deleting ephemeral public IP address from instance") + }).Infof("deleting public IP address from instance") op, err := a.addressManager.DeleteAccessConfig(a.project, zone, instance.Name, accessConfigName, networkInterface.Name) if err != nil { return errors.Wrapf(err, "failed to delete access config %s from instance %s", accessConfigName, instance.Name) @@ -233,3 +233,32 @@ func (a *gcpAssigner) listAddresses(filter []string, orderBy, status string) ([] call = call.PageToken(list.NextPageToken) } } + +func (a *gcpAssigner) Unassign(ctx context.Context, instanceID, zone string) error { + // get the instance details + instance, err := a.instanceGetter.Get(a.project, zone, instanceID) + if err != nil { + return errors.Wrapf(err, "failed to get instance %s", instanceID) + } + // list all assigned addresses + assigned, err := a.listAddresses(nil, "", inUseStatus) + if err != nil { + return errors.Wrap(err, "failed to list assigned addresses") + } + // if there are assigned addresses, check if they are assigned to the instance + if len(assigned) > 0 { + for _, address := range assigned { + for _, user := range address.Users { + if user == instance.SelfLink { + // release/remove current static public IP address + if err = a.deleteInstanceAddress(ctx, instance, zone); err != nil { + return errors.Wrap(err, "failed to delete current public IP address") + } + // break the loop after deleting the address + return nil + } + } + } + } + return nil +} diff --git a/internal/cloud/aws_address.go b/internal/cloud/aws_address.go index 718c63b..e7300a1 100644 --- a/internal/cloud/aws_address.go +++ b/internal/cloud/aws_address.go @@ -10,6 +10,7 @@ import ( type EipAssigner interface { Assign(ctx context.Context, region, instanceID string, address *types.Address) error + Unassign(ctx context.Context, address *types.Address) error } type eipAssigner struct { @@ -35,3 +36,17 @@ func (a *eipAssigner) Assign(ctx context.Context, instanceID, networkInterfaceID return nil } + +func (a *eipAssigner) Unassign(ctx context.Context, address *types.Address) error { + // disassociate elastic IP from the instance + input := &ec2.DisassociateAddressInput{ + AssociationId: address.AssociationId, + } + + _, err := a.client.DisassociateAddress(ctx, input) + if err != nil { + return errors.Wrap(err, "failed to disassociate elastic IP from the instance") + } + + return nil +} diff --git a/internal/config/config.go b/internal/config/config.go index f765cc5..c50f41a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -25,6 +25,8 @@ type Config struct { RetryInterval time.Duration `json:"retry-interval"` // Retry attempts RetryAttempts int `json:"retry-attempts"` + // ReleaseOnExit releases the IP address on exit + ReleaseOnExit bool `json:"release-on-exit"` } func NewConfig(c *cli.Context) *Config { @@ -38,5 +40,6 @@ func NewConfig(c *cli.Context) *Config { cfg.OrderBy = c.String("order-by") cfg.Project = c.String("project") cfg.Region = c.String("region") + cfg.ReleaseOnExit = c.Bool("release-on-exit") return &cfg } diff --git a/mocks/address/Assigner.go b/mocks/address/Assigner.go index 290b42a..15140f7 100644 --- a/mocks/address/Assigner.go +++ b/mocks/address/Assigner.go @@ -67,6 +67,50 @@ func (_c *Assigner_Assign_Call) RunAndReturn(run func(context.Context, string, s return _c } +// Unassign provides a mock function with given fields: ctx, instanceID, zone +func (_m *Assigner) Unassign(ctx context.Context, instanceID string, zone string) error { + ret := _m.Called(ctx, instanceID, zone) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) error); ok { + r0 = rf(ctx, instanceID, zone) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Assigner_Unassign_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Unassign' +type Assigner_Unassign_Call struct { + *mock.Call +} + +// Unassign is a helper method to define mock.On call +// - ctx context.Context +// - instanceID string +// - zone string +func (_e *Assigner_Expecter) Unassign(ctx interface{}, instanceID interface{}, zone interface{}) *Assigner_Unassign_Call { + return &Assigner_Unassign_Call{Call: _e.mock.On("Unassign", ctx, instanceID, zone)} +} + +func (_c *Assigner_Unassign_Call) Run(run func(ctx context.Context, instanceID string, zone string)) *Assigner_Unassign_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string)) + }) + return _c +} + +func (_c *Assigner_Unassign_Call) Return(_a0 error) *Assigner_Unassign_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Assigner_Unassign_Call) RunAndReturn(run func(context.Context, string, string) error) *Assigner_Unassign_Call { + _c.Call.Return(run) + return _c +} + // NewAssigner creates a new instance of Assigner. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewAssigner(t interface { diff --git a/mocks/cloud/EipAssigner.go b/mocks/cloud/EipAssigner.go index 668862f..802e028 100644 --- a/mocks/cloud/EipAssigner.go +++ b/mocks/cloud/EipAssigner.go @@ -67,6 +67,49 @@ func (_c *EipAssigner_Assign_Call) RunAndReturn(run func(context.Context, string return _c } +// Unassign provides a mock function with given fields: ctx, address +func (_m *EipAssigner) Unassign(ctx context.Context, address *types.Address) error { + ret := _m.Called(ctx, address) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, *types.Address) error); ok { + r0 = rf(ctx, address) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// EipAssigner_Unassign_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Unassign' +type EipAssigner_Unassign_Call struct { + *mock.Call +} + +// Unassign is a helper method to define mock.On call +// - ctx context.Context +// - address *types.Address +func (_e *EipAssigner_Expecter) Unassign(ctx interface{}, address interface{}) *EipAssigner_Unassign_Call { + return &EipAssigner_Unassign_Call{Call: _e.mock.On("Unassign", ctx, address)} +} + +func (_c *EipAssigner_Unassign_Call) Run(run func(ctx context.Context, address *types.Address)) *EipAssigner_Unassign_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*types.Address)) + }) + return _c +} + +func (_c *EipAssigner_Unassign_Call) Return(_a0 error) *EipAssigner_Unassign_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *EipAssigner_Unassign_Call) RunAndReturn(run func(context.Context, *types.Address) error) *EipAssigner_Unassign_Call { + _c.Call.Return(run) + return _c +} + // NewEipAssigner creates a new instance of EipAssigner. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewEipAssigner(t interface { From 18a21b4426b5b8cc7f6a260500b1d7fd57f89a19 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Thu, 12 Oct 2023 16:25:03 +0300 Subject: [PATCH 31/66] test gcp static ip assign --- internal/address/gcp_test.go | 165 +++++++++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) diff --git a/internal/address/gcp_test.go b/internal/address/gcp_test.go index 8a2b76a..493fab7 100644 --- a/internal/address/gcp_test.go +++ b/internal/address/gcp_test.go @@ -256,3 +256,168 @@ func Test_gcpAssigner_waitForOperation(t *testing.T) { }) } } + +func Test_gcpAssigner_deleteInstanceAddress(t *testing.T) { + type args struct { + ctx context.Context + instance *compute.Instance + zone string + } + type fields struct { + addressManagerFn func(t *testing.T, args *args) cloud.AddressManager + project string + region string + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "delete instance address successfully", + fields: fields{ + project: "test-project", + region: "test-region", + addressManagerFn: func(t *testing.T, args *args) cloud.AddressManager { + mock := mocks.NewAddressManager(t) + networkInterfaceName := args.instance.NetworkInterfaces[0].Name + accessConfigName := args.instance.NetworkInterfaces[0].AccessConfigs[0].Name + mock.EXPECT().DeleteAccessConfig("test-project", "", args.instance.Name, accessConfigName, networkInterfaceName).Return(&compute.Operation{Name: "test-operation", Status: "DONE"}, nil) + return mock + }, + }, + args: args{ + ctx: context.TODO(), + instance: &compute.Instance{ + Name: "test-instance", + Zone: "test-zone", + NetworkInterfaces: []*compute.NetworkInterface{ + { + Name: "test-network-interface", + AccessConfigs: []*compute.AccessConfig{ + {Name: "test-access-config", NatIP: "100.0.0.1"}, + }, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + logger := logrus.NewEntry(logrus.New()) + a := &gcpAssigner{ + addressManager: tt.fields.addressManagerFn(t, &tt.args), + project: tt.fields.project, + region: tt.fields.region, + logger: logger, + } + if err := a.deleteInstanceAddress(tt.args.ctx, tt.args.instance, tt.args.zone); (err != nil) != tt.wantErr { + t.Errorf("deleteInstanceAddress() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_gcpAssigner_Assign(t *testing.T) { + type fields struct { + listerFn func(t *testing.T) cloud.Lister + addressManagerFn func(t *testing.T) cloud.AddressManager + instanceGetterFn func(t *testing.T) cloud.InstanceGetter + project string + region string + } + type args struct { + ctx context.Context + instanceID string + zone string + filter []string + orderBy string + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "assign address successfully", + fields: fields{ + project: "test-project", + region: "test-region", + listerFn: func(t *testing.T) cloud.Lister { + mock := mocks.NewLister(t) + mockCall := mocks.NewListCall(t) + mock.EXPECT().List("test-project", "test-region").Return(mockCall) + mockCall.EXPECT().Filter("(status=IN_USE) (addressType=EXTERNAL)").Return(mockCall).Once() + mockCall.EXPECT().Do().Return(&compute.AddressList{ + Items: []*compute.Address{ + {Name: "test-address-1", Status: inUseStatus, Address: "100.0.0.1", NetworkTier: "PREMIUM", AddressType: "EXTERNAL", Users: []string{"self-link-test-instance-1"}}, + {Name: "test-address-2", Status: inUseStatus, Address: "100.0.0.2", NetworkTier: "PREMIUM", AddressType: "EXTERNAL", Users: []string{"self-link-test-instance-2"}}, + }, + }, nil).Once() + mockCall.EXPECT().Filter("(status=RESERVED) (addressType=EXTERNAL) (test-filter-1) (test-filter-2)").Return(mockCall).Once() + mockCall.EXPECT().OrderBy("test-order-by").Return(mockCall).Once() + mockCall.EXPECT().Do().Return(&compute.AddressList{ + Items: []*compute.Address{ + {Name: "test-address-3", Status: reservedStatus, Address: "100.0.0.3", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, + {Name: "test-address-4", Status: reservedStatus, Address: "100.0.0.4", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, + }, + }, nil).Once() + return mock + }, + instanceGetterFn: func(t *testing.T) cloud.InstanceGetter { + mock := mocks.NewInstanceGetter(t) + mock.EXPECT().Get("test-project", "test-zone", "test-instance-0").Return(&compute.Instance{ + Name: "test-instance-0", + Zone: "test-zone", + NetworkInterfaces: []*compute.NetworkInterface{ + { + Name: "test-network-interface", + AccessConfigs: []*compute.AccessConfig{ + {Name: "test-access-config", NatIP: "200.0.0.1", Type: accessConfigType, Kind: accessConfigKind}, + }, + }, + }, + }, nil) + return mock + }, + addressManagerFn: func(t *testing.T) cloud.AddressManager { + mock := mocks.NewAddressManager(t) + mock.EXPECT().DeleteAccessConfig("test-project", "test-zone", "test-instance-0", "test-access-config", "test-network-interface").Return(&compute.Operation{Name: "test-operation", Status: "DONE"}, nil) + mock.EXPECT().AddAccessConfig("test-project", "test-zone", "test-instance-0", defaultNetworkInterface, &compute.AccessConfig{ + Name: "test-address-3", + Type: accessConfigType, + Kind: accessConfigKind, + NatIP: "100.0.0.3", + }).Return(&compute.Operation{Name: "test-operation", Status: "DONE"}, nil) + return mock + }, + }, + args: args{ + ctx: context.TODO(), + instanceID: "test-instance-0", + zone: "test-zone", + filter: []string{"test-filter-1", "test-filter-2"}, + orderBy: "test-order-by", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + logger := logrus.NewEntry(logrus.New()) + a := &gcpAssigner{ + lister: tt.fields.listerFn(t), + addressManager: tt.fields.addressManagerFn(t), + instanceGetter: tt.fields.instanceGetterFn(t), + project: tt.fields.project, + region: tt.fields.region, + logger: logger, + } + if err := a.Assign(tt.args.ctx, tt.args.instanceID, tt.args.zone, tt.args.filter, tt.args.orderBy); (err != nil) != tt.wantErr { + t.Errorf("Assign() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} From 20dec2d9c1d8f7598137401356d18b78437bef55 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Thu, 12 Oct 2023 16:29:29 +0300 Subject: [PATCH 32/66] fix linter error --- internal/address/assigner.go | 6 +++++- internal/cloud/aws_instance.go | 6 +++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/internal/address/assigner.go b/internal/address/assigner.go index 2f64732..88dd12c 100644 --- a/internal/address/assigner.go +++ b/internal/address/assigner.go @@ -9,6 +9,10 @@ import ( "github.com/sirupsen/logrus" ) +var ( + ErrUnknownCloudProvider = errors.New("unknown cloud provider") +) + type Assigner interface { Assign(ctx context.Context, instanceID, zone string, filter []string, orderBy string) error Unassign(ctx context.Context, instanceID, zone string) error @@ -22,5 +26,5 @@ func NewAssigner(ctx context.Context, logger *logrus.Entry, provider types.Cloud } else if provider == types.CloudProviderGCP { return NewGCPAssigner(ctx, logger, cfg.Project, cfg.Region) } - return nil, errors.New("unknown cloud provider") + return nil, ErrUnknownCloudProvider } diff --git a/internal/cloud/aws_instance.go b/internal/cloud/aws_instance.go index 45b521e..f665ab2 100644 --- a/internal/cloud/aws_instance.go +++ b/internal/cloud/aws_instance.go @@ -2,10 +2,10 @@ package cloud import ( "context" - "fmt" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/pkg/errors" ) type Ec2InstanceGetter interface { @@ -29,11 +29,11 @@ func (g *ec2InstanceGetter) Get(ctx context.Context, instanceID, region string) resp, err := g.client.DescribeInstances(ctx, input) if err != nil { - return nil, fmt.Errorf("failed to describe instances, %v", err) + return nil, errors.Wrap(err, "failed to describe instances, %v") } if len(resp.Reservations) == 0 || len(resp.Reservations[0].Instances) == 0 { - return nil, fmt.Errorf("no instances found for the given id") + return nil, errors.Wrap(err, "no instances found for the given id") } return &resp.Reservations[0].Instances[0], nil From 9beeab9013268f880a5b80fd4c59c78bf6403076 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Thu, 12 Oct 2023 19:23:03 +0300 Subject: [PATCH 33/66] README draft --- README.md | 186 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 181 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 7bcec55..22592a4 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,184 @@ -![ci](https://github.com/doitintl/kubeip/workflows/ci/badge.svg) [![Go Report Card](https://goreportcard.com/badge/github.com/doitintl/kubeip)](https://goreportcard.com/report/github.com/doitintl/kubeip) ![Docker Pulls](https://img.shields.io/docker/pulls/doitintl/kubeip) +![build](https://github.com/doitintl/kubeip/workflows/build/badge.svg) [![Go Report Card](https://goreportcard.com/badge/github.com/doitintl/kubeip)](https://goreportcard.com/report/github.com/doitintl/kubeip) ![Docker Pulls](https://img.shields.io/docker/pulls/doitintl/kubeip-agent) -# What is kubeIP? +# KubeIP v2 -Many applications need to be whitelisted by users based on a Source IP Address. As of today, cloud-manages Kubernetes Engines do not support -assigning a static pool of IP addresses to the Kubernetes cluster. Using kubeIP, this problem is solved by assigning Kubernetes nodes -external IP addresses from a predefined list. +Welcome to KubeIP v2, a complete overhaul of the popular [DoiT](https://www.doit.com/) +KubeIP [v1](https://github.com/doitintl/kubeip/tree/v1-main) open-source project, originally developed +by [Aviv Laufer](https://github.com/avivl). +KubeIP v2 expands its support beyond Google Cloud (as in v1) to include AWS, and it's designed to be extendable to other cloud providers +that allow assigning static public IP to VMs. We've also transitioned from a Kubernetes controller to a standard DaemonSet, enhancing +reliability and ease of use. + +## What KubeIP does? + +KubeIP is a tool that assigns a static public IP to any node it operates on. The IP is assigned to the node's primary network interface, +selected from a list of reserved static IPs using platform-supported filtering. + +## How to use KubeIP? + +Deploy KubeIP as a DaemonSet on your desired nodes using standard Kubernetes selectors. Once deployed, KubeIP will assign a static public IP +to each node it operates on. If no static public IP is available, KubeIP will wait until one becomes available. When a node is deleted, +KubeIP will release the static public IP. + +### AWS + +Make sure that KubeIP DaemonSet is deployed on nodes that have a public IP (node in public subnet) and uses Kubernetes service +account [bound](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) +to IAM role with the following permissions: + +```yaml +Version: '2012-10-17' +Statement: + - Effect: Allow + Action: + - ec2:AssociateAddress + - ec2:DisassociateAddress + - ec2:DescribeInstances + - ec2:DescribeAddresses + Resource: '*' +``` + +KubeIP supports filtering of reserved Elastic IPs using tags. To use this feature, add the `filter` flag (or set `FILTER` environment +variable) to the KubeIP DaemonSet: + +```yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kubeip +spec: + selector: + matchLabels: + app: kubeip + template: + metadata: + labels: + app: kubeip + spec: + serviceAccountName: kubeip-sa + nodeSelector: + kubeip.com/public: "true" + containers: + - name: kubeip + image: doitintl/kubeip-agent + env: + - name: FILTER + value: "Name=tag:env,Values=dev;Name=tag:app,Values=streamer" +``` + +KubeIP AWS filter supports the same filter syntax as the AWS `describe-addresses` command. For more information, +see [describe-addresses](https://docs.aws.amazon.com/cli/latest/reference/ec2/describe-addresses.html#options). If you specify multiple +filters, they are joined with an `AND`, and the request returns only results that match all the specified filters. Multiple filters must be +separated by semicolons (`;`). + +### Google Cloud + +Ensure that the KubeIP DaemonSet is deployed on nodes with a public IP (nodes in a public subnet) and uses a Kubernetes service +account [bound](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity) to an IAM role with the following permissions: + +```yaml +title: "KubeIP Role" +description: "KubeIP required permissions" +stage: "GA" +includedPermissions: + - compute.instances.addAccessConfig + - compute.instances.deleteAccessConfig + - compute.instances.get + - compute.addresses.list + - compute.zoneOperations.get + - compute.projects.get +``` + +KubeIP Google Cloud filter supports the same filter syntax as the Google Cloud `gcloud compute addresses list` command. For more +information, see [gcloud topic filter](https://cloud.google.com/sdk/gcloud/reference/topic/filters). If you specify multiple filters, they +are joined with an `AND`, and the request returns only results that match all the specified filters. Multiple filters must be separated by +semicolons (`;`). + +To use this feature, add the `filter` flag (or set `FILTER` environment variable) to the KubeIP DaemonSet: + +```yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kubeip +spec: + selector: + matchLabels: + app: kubeip + template: + metadata: + labels: + app: kubeip + spec: + serviceAccountName: kubeip-sa + nodeSelector: + kubeip.com/public: "true" + containers: + - name: kubeip + image: doitintl/kubeip-agent + env: + - name: FILTER + value: "labels.env=dev;labels.app=streamer" +``` + +## How to contribute to KubeIP? + +KubeIP is an open-source project, and we welcome your contributions! + +## How to build KubeIP? + +KubeIP is written in Go and can be built using standard Go tools. To build KubeIP, run the following command: + +```shell +make build +``` + +## How to run KubeIP? + +KubeIP is a standard command-line application. To explore the available options, run the following command: + +```shell +kubeip-agent run --help +``` + +```text +NAME: + kubeip-agent run - run agent + +USAGE: + kubeip-agent run [command options] [arguments...] + +OPTIONS: + Configuration + + --filter value [ --filter value ] filter for the IP addresses [$FILTER] + --kubeconfig value path to Kubernetes configuration file (not needed if running in node) [$KUBECONFIG] + --node-name value Kubernetes node name (not needed if running in node) [$NODE_NAME] + --order-by value order by for the IP addresses [$ORDER_BY] + --project value name of the GCP project or the AWS account ID (not needed if running in node) [$PROJECT] + --region value name of the GCP region or the AWS region (not needed if running in node) [$REGION] + --release-on-exit release the static public IP address on exit (default: true) [$RELEASE_ON_EXIT] + --retry-attempts value number of attempts to assign the static public IP address (default: 10) [$RETRY_ATTEMPTS] + --retry-interval value when the agent fails to assign the static public IP address, it will retry after this interval (default: 5m0s) [$RETRY_INTERVAL] + + Development + + --develop-mode enable develop mode (default: false) [$DEV_MODE] + + Logging + + --json produce log in JSON format: Logstash and Splunk friendly (default: false) [$LOG_JSON] + --log-level value set log level (debug, info(*), warning, error, fatal, panic) (default: "info") [$LOG_LEVEL] +``` + +## How to test KubeIP? + +To test KubeIP, create a pool of reserved static public IPs, ensuring that the pool has enough IPs to assign to all nodes that KubeIP will +operate on. Use labels to filter the pool of reserved static public IPs. + +Next, create a Kubernetes cluster and deploy KubeIP as a DaemonSet on your desired nodes. Ensure that the nodes have a public IP (nodes in a +public subnet). Configure KubeIP to use the pool of reserved static public IPs, using filters and order by. + +Finally, scale the number of nodes in the cluster and verify that KubeIP assigns a static public IP to each node. Scale down the number of +nodes in the cluster and verify that KubeIP releases the static public IP addresses. From c5f2a53b4deab3b2d749d70eb4ae67d1f4ff4088 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Fri, 13 Oct 2023 19:47:47 +0300 Subject: [PATCH 34/66] AWS TF example (draft) --- .gitignore | 8 ++++++++ Dockerfile | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5f6b96d..665715a 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,14 @@ .in .out +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + + # Output of the go coverage tool, specifically when used with LiteIDE *.out diff --git a/Dockerfile b/Dockerfile index 46f24d3..226da46 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,4 +34,5 @@ COPY --from=builder /usr/share/zoneinfo /usr/share/zoneinfo # copy the binary to the production image from the builder stage COPY --from=builder /app/.bin/kubeip /kubeip -ENTRYPOINT ["/kubeip"] \ No newline at end of file +ENTRYPOINT ["/kubeip"] +CMD ["run"] \ No newline at end of file From a7c746fc90afc4a11a515d743d2f7ca1b9229493 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Fri, 13 Oct 2023 19:47:54 +0300 Subject: [PATCH 35/66] AWS TF example (draft) --- examples/aws/.terraform.lock.hcl | 105 ++++++++++++++++++++++ examples/aws/eks.tf | 146 +++++++++++++++++++++++++++++++ examples/aws/variables.tf | 14 +++ 3 files changed, 265 insertions(+) create mode 100644 examples/aws/.terraform.lock.hcl create mode 100644 examples/aws/eks.tf create mode 100644 examples/aws/variables.tf diff --git a/examples/aws/.terraform.lock.hcl b/examples/aws/.terraform.lock.hcl new file mode 100644 index 0000000..3e5955b --- /dev/null +++ b/examples/aws/.terraform.lock.hcl @@ -0,0 +1,105 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.21.0" + constraints = ">= 5.0.0" + hashes = [ + "h1:bCRZDV8QYPpl+zIU7karO77B4x2cvQ7UHKAGNvTFWQs=", + "zh:1ba1411e4f8c047950db94c236f146d4590790320c68320b4e56082d8746a507", + "zh:3185e4a34cfcad35dcf11439290a4bd0ad52d462eca2ab5d4940488a2db72833", + "zh:3c6b901f874b4d9a85301a653d0bd507b052992bd84fc81100f4e5f73b1adab7", + "zh:45d3fdbbc5804f295576b7155fdca527dedff17a014ed40c215af3bc60c329db", + "zh:47b64b453d2c373062e47a54f3df33335dc29bce6ddbbf2da9e7be768c560abe", + "zh:5cdf57ffd465288d9732d14ba13b377a8d389e0ba0ce3ac4773fd6fdfc09d6a1", + "zh:81ec4c662581a2446c78da7b27d7e0d5c2e4d50925294789ec13661817f4b5a4", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:ac248464fd4ce1f020c05f27e3182532a7d1af4b8185a4b4be8b906b30b0ca5a", + "zh:bbbedc6b6eaffcce0b31b397d607464f0c21c1b9406182163d504d3f392cc68d", + "zh:c2afc111f9503829ed055e2ae91d873670c57bd16acc1a3246ac3957f6998d4e", + "zh:cd3c8175b2152848113482da70e5b9c7cb4c951f2046fc0b832715300bd88b97", + "zh:cf89b0c09d426d489f9477209d4084e64ad1b598036284fa688b41de626b58e6", + "zh:d9d127637c3b9ff6e2d0a2c30f54bd48ab1de34f725a5df1a6a3d039b021e636", + "zh:dccca1090e4054d6558218406385fb0421ab4ac3b75e121641973be481a81f01", + ] +} + +provider "registry.terraform.io/hashicorp/cloudinit" { + version = "2.3.2" + constraints = ">= 2.0.0" + hashes = [ + "h1:ocyv0lvfyvzW4krenxV5CL4Jq5DiA3EUfoy8DR6zFMw=", + "zh:2487e498736ed90f53de8f66fe2b8c05665b9f8ff1506f751c5ee227c7f457d1", + "zh:3d8627d142942336cf65eea6eb6403692f47e9072ff3fa11c3f774a3b93130b3", + "zh:434b643054aeafb5df28d5529b72acc20c6f5ded24decad73b98657af2b53f4f", + "zh:436aa6c2b07d82aa6a9dd746a3e3a627f72787c27c80552ceda6dc52d01f4b6f", + "zh:458274c5aabe65ef4dbd61d43ce759287788e35a2da004e796373f88edcaa422", + "zh:54bc70fa6fb7da33292ae4d9ceef5398d637c7373e729ed4fce59bd7b8d67372", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:893ba267e18749c1a956b69be569f0d7bc043a49c3a0eb4d0d09a8e8b2ca3136", + "zh:95493b7517bce116f75cdd4c63b7c82a9d0d48ec2ef2f5eb836d262ef96d0aa7", + "zh:9ae21ab393be52e3e84e5cce0ef20e690d21f6c10ade7d9d9d22b39851bfeddc", + "zh:cc3b01ac2472e6d59358d54d5e4945032efbc8008739a6d4946ca1b621a16040", + "zh:f23bfe9758f06a1ec10ea3a81c9deedf3a7b42963568997d84a5153f35c5839a", + ] +} + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.23.0" + constraints = ">= 2.10.0" + hashes = [ + "h1:arTzD0XG/DswGCAx9JEttkSKe9RyyFW9W7UWcXF13dU=", + "zh:10488a12525ed674359585f83e3ee5e74818b5c98e033798351678b21b2f7d89", + "zh:1102ba5ca1a595f880e67102bbf999cc8b60203272a078a5b1e896d173f3f34b", + "zh:1347cf958ed3f3f80b3c7b3e23ddda3d6c6573a81847a8ee92b7df231c238bf6", + "zh:2cb18e9f5156bc1b1ee6bc580a709f7c2737d142722948f4a6c3c8efe757fa8d", + "zh:5506aa6f28dcca2a265ccf8e34478b5ec2cb43b867fe6d93b0158f01590fdadd", + "zh:6217a20686b631b1dcb448ee4bc795747ebc61b56fbe97a1ad51f375ebb0d996", + "zh:8accf916c00579c22806cb771e8909b349ffb7eb29d9c5468d0a3f3166c7a84a", + "zh:9379b0b54a0fa030b19c7b9356708ec8489e194c3b5e978df2d31368563308e5", + "zh:aa99c580890691036c2931841e88e7ee80d59ae52289c8c2c28ea0ac23e31520", + "zh:c57376d169875990ac68664d227fb69cd0037b92d0eba6921d757c3fd1879080", + "zh:e6068e3f94f6943b5586557b73f109debe19d1a75ca9273a681d22d1ce066579", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/time" { + version = "0.9.1" + constraints = ">= 0.9.0" + hashes = [ + "h1:VxyoYYOCaJGDmLz4TruZQTSfQhvwEcMxvcKclWdnpbs=", + "zh:00a1476ecf18c735cc08e27bfa835c33f8ac8fa6fa746b01cd3bcbad8ca84f7f", + "zh:3007f8fc4a4f8614c43e8ef1d4b0c773a5de1dcac50e701d8abc9fdc8fcb6bf5", + "zh:5f79d0730fdec8cb148b277de3f00485eff3e9cf1ff47fb715b1c969e5bbd9d4", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:8c8094689a2bed4bb597d24a418bbbf846e15507f08be447d0a5acea67c2265a", + "zh:a6d9206e95d5681229429b406bc7a9ba4b2d9b67470bda7df88fa161508ace57", + "zh:aa299ec058f23ebe68976c7581017de50da6204883950de228ed9246f309e7f1", + "zh:b129f00f45fba1991db0aa954a6ba48d90f64a738629119bfb8e9a844b66e80b", + "zh:ef6cecf5f50cda971c1b215847938ced4cb4a30a18095509c068643b14030b00", + "zh:f1f46a4f6c65886d2dd27b66d92632232adc64f92145bf8403fe64d5ffa5caea", + "zh:f79d6155cda7d559c60d74883a24879a01c4d5f6fd7e8d1e3250f3cd215fb904", + "zh:fd59fa73074805c3575f08cd627eef7acda14ab6dac2c135a66e7a38d262201c", + ] +} + +provider "registry.terraform.io/hashicorp/tls" { + version = "4.0.4" + constraints = ">= 3.0.0" + hashes = [ + "h1:GZcFizg5ZT2VrpwvxGBHQ/hO9r6g0vYdQqx3bFD3anY=", + "zh:23671ed83e1fcf79745534841e10291bbf34046b27d6e68a5d0aab77206f4a55", + "zh:45292421211ffd9e8e3eb3655677700e3c5047f71d8f7650d2ce30242335f848", + "zh:59fedb519f4433c0fdb1d58b27c210b27415fddd0cd73c5312530b4309c088be", + "zh:5a8eec2409a9ff7cd0758a9d818c74bcba92a240e6c5e54b99df68fff312bbd5", + "zh:5e6a4b39f3171f53292ab88058a59e64825f2b842760a4869e64dc1dc093d1fe", + "zh:810547d0bf9311d21c81cc306126d3547e7bd3f194fc295836acf164b9f8424e", + "zh:824a5f3617624243bed0259d7dd37d76017097dc3193dac669be342b90b2ab48", + "zh:9361ccc7048be5dcbc2fafe2d8216939765b3160bd52734f7a9fd917a39ecbd8", + "zh:aa02ea625aaf672e649296bce7580f62d724268189fe9ad7c1b36bb0fa12fa60", + "zh:c71b4cd40d6ec7815dfeefd57d88bc592c0c42f5e5858dcc88245d371b4b8b1e", + "zh:dabcd52f36b43d250a3d71ad7abfa07b5622c69068d989e60b79b2bb4f220316", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/examples/aws/eks.tf b/examples/aws/eks.tf new file mode 100644 index 0000000..f1e4598 --- /dev/null +++ b/examples/aws/eks.tf @@ -0,0 +1,146 @@ +provider "aws" { + region = "us-west-2" # replace with your region +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + + name = var.vpc_name + cidr = "10.0.0.0/16" + azs = ["us-west-2a", "us-west-2b", "us-west-2c"] + private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] + public_subnets = ["10.0.101.0/24", "10.0.102.0/24", "10.0.103.0/24"] + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true + + tags = { + App = "kubeip" + Env = "demo" + } +} + +module "eks" { + source = "terraform-aws-modules/eks/aws" + + cluster_name = var.cluster_name + cluster_version = var.kubernetes_version + + cluster_endpoint_public_access = true + + vpc_id = module.vpc.vpc_id + subnet_ids = concat(module.vpc.private_subnets, module.vpc.public_subnets) + + cluster_addons = { + coredns = { + most_recent = true + } + kube-proxy = { + most_recent = true + } + vpc-cni = { + most_recent = true + } + } + + eks_managed_node_groups = { + eks_nodes_public = { + desired_size = 1 + max_size = 5 + min_size = 1 + + instance_types = ["t4g.micro", "t4g.small"] + capacity_type = "SPOT" + platform = "bottlerocket" + + taints = [ + { + key = "kubeip" + value = "use" + effect = "NO_SCHEDULE" + } + ] + + labels = { + nodegroup = "public" + kubeip = "use" + } + + tags = { + Environment = "demo" + Name = "public-node-group" + public = "true" + kubeip = "use" + } + + subnet_ids = module.vpc.public_subnets + } + + eks_nodes_private = { + desired_size = 1 + max_size = 5 + min_size = 1 + + instance_types = ["t4g.micro", "t4g.small"] + capacity_type = "SPOT" + platform = "bottlerocket" + + labels = { + nodegroup = "private" + kubeip = "ignore" + } + + tags = { + Environment = "demo" + Name = "private-node-group" + } + + subnet_ids = module.vpc.private_subnets + } + } + + # aws-auth configmap + manage_aws_auth_configmap = true + + tags = { + App = "kubeip" + Environment = "demo" + } +} + +resource "aws_iam_policy" "kubeip-policy" { + name = "kubeip-policy" + description = "KubeIP policy" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "ec2:AssociateAddress", + "ec2:DisassociateAddress", + "ec2:DescribeInstances", + "ec2:DescribeAddresses" + ] + Effect = "Allow" + Resource = "*" + }, + ] + }) +} + +module "kubeip_eks_role" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + role_name = "kubeip-eks-role" + + role_policy_arns = { + "kubeip-policy" = aws_iam_policy.kubeip-policy.arn + } + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["kube-system:kubeip-sa"] + } + } +} diff --git a/examples/aws/variables.tf b/examples/aws/variables.tf new file mode 100644 index 0000000..36f5e49 --- /dev/null +++ b/examples/aws/variables.tf @@ -0,0 +1,14 @@ +variable "cluster_name" { + type = string + default = "kubeip-demo" +} + +variable "vpc_name" { + type = string + default = "kubeip-demo" +} + +variable "kubernetes_version" { + type = string + default = "1.28" +} \ No newline at end of file From 475fcdd3cd34ff617b95f1d787f8960446836b75 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Fri, 13 Oct 2023 20:00:04 +0300 Subject: [PATCH 36/66] cleanup tf script --- examples/aws/eks.tf | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/examples/aws/eks.tf b/examples/aws/eks.tf index f1e4598..516f482 100644 --- a/examples/aws/eks.tf +++ b/examples/aws/eks.tf @@ -31,18 +31,6 @@ module "eks" { vpc_id = module.vpc.vpc_id subnet_ids = concat(module.vpc.private_subnets, module.vpc.public_subnets) - cluster_addons = { - coredns = { - most_recent = true - } - kube-proxy = { - most_recent = true - } - vpc-cni = { - most_recent = true - } - } - eks_managed_node_groups = { eks_nodes_public = { desired_size = 1 From 8bbcdaf1f1e7e6e0dd275d50c42f7720b0139c72 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Sun, 15 Oct 2023 12:54:30 +0300 Subject: [PATCH 37/66] eks and gke TF draft --- examples/aws/eks.tf | 12 +- examples/aws/variables.tf | 5 + examples/gcp/.terraform.lock.hcl | 120 ++++++++++++++++++ examples/gcp/gke.tf | 202 +++++++++++++++++++++++++++++++ examples/gcp/variables.tf | 18 +++ 5 files changed, 356 insertions(+), 1 deletion(-) create mode 100644 examples/gcp/.terraform.lock.hcl create mode 100644 examples/gcp/gke.tf create mode 100644 examples/gcp/variables.tf diff --git a/examples/aws/eks.tf b/examples/aws/eks.tf index 516f482..a53a063 100644 --- a/examples/aws/eks.tf +++ b/examples/aws/eks.tf @@ -1,5 +1,5 @@ provider "aws" { - region = "us-west-2" # replace with your region + region = var.region } module "vpc" { @@ -132,3 +132,13 @@ module "kubeip_eks_role" { } } } + +# 5 elastic IPs in the same region +resource "aws_eip" "kubeip" { + count = 5 + + tags = { + Name = "kubeip-${count.index}" + kubeip = "reserved" + } +} diff --git a/examples/aws/variables.tf b/examples/aws/variables.tf index 36f5e49..9d49338 100644 --- a/examples/aws/variables.tf +++ b/examples/aws/variables.tf @@ -1,3 +1,8 @@ +variable "region" { + type = string + default = "us-west-2" +} + variable "cluster_name" { type = string default = "kubeip-demo" diff --git a/examples/gcp/.terraform.lock.hcl b/examples/gcp/.terraform.lock.hcl new file mode 100644 index 0000000..e98080c --- /dev/null +++ b/examples/gcp/.terraform.lock.hcl @@ -0,0 +1,120 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/external" { + version = "2.3.1" + constraints = ">= 2.2.2" + hashes = [ + "h1:gznGscVJ0USxy4CdihpjRKPsKvyGr/zqPvBoFLJTQDc=", + "zh:001e2886dc81fc98cf17cf34c0d53cb2dae1e869464792576e11b0f34ee92f54", + "zh:2eeac58dd75b1abdf91945ac4284c9ccb2bfb17fa9bdb5f5d408148ff553b3ee", + "zh:2fc39079ba61411a737df2908942e6970cb67ed2f4fb19090cd44ce2082903dd", + "zh:472a71c624952cff7aa98a7b967f6c7bb53153dbd2b8f356ceb286e6743bb4e2", + "zh:4cff06d31272aac8bc35e9b7faec42cf4554cbcbae1092eaab6ab7f643c215d9", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7ed16ccd2049fa089616b98c0bd57219f407958f318f3c697843e2397ddf70df", + "zh:842696362c92bf2645eb85c739410fd51376be6c488733efae44f4ce688da50e", + "zh:8985129f2eccfd7f1841ce06f3bf2bbede6352ec9e9f926fbaa6b1a05313b326", + "zh:a5f0602d8ec991a5411ef42f872aa90f6347e93886ce67905c53cfea37278e05", + "zh:bf4ab82cbe5256dcef16949973bf6aa1a98c2c73a98d6a44ee7bc40809d002b8", + "zh:e70770be62aa70198fa899526d671643ff99eecf265bf1a50e798fc3480bd417", + ] +} + +provider "registry.terraform.io/hashicorp/google" { + version = "4.84.0" + constraints = ">= 3.39.0, >= 3.53.0, < 5.0.0, < 6.0.0" + hashes = [ + "h1:fybaK74buTd4Ys2CUZm6jw7NXtSqtcLoW2jeNB4Ff2E=", + "zh:0b3e945fa76876c312bdddca7b18c93b734998febb616b2ebb84a0a299ae97c2", + "zh:1d47d00730fab764bddb6d548fed7e124739b0bcebb9f3b3c6aa247de55fb804", + "zh:29bff92b4375a35a7729248b3bc5db8991ca1b9ba640fc25b13700e12f99c195", + "zh:382353516e7d408a81f1a09a36f9015429be73ca3665367119aad88713209d9a", + "zh:78afa20e25a690d076eeaafd7879993ef9763a8a1b6762e2cbe42330464cc1fa", + "zh:8f6422e94de865669b33a2d9fb95a3e392e841988e890f7379a206e9d47e3415", + "zh:be5c7b52c893b971c860146aec643f7007f34430106f101eab686ed81eccbd26", + "zh:bfc37b641bf3378183eb3b8735554c3949a5cfaa8f76403d7eff38de1474b6d9", + "zh:c834f88dc8eb21af992871ed13a221015ae3b051aeca7386662071026f1546b4", + "zh:f3296c8c0d57dc28e23cf91717484264531655ac478d994584ebc73f70679471", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f8efe114ff4891776f48f7d2620b8d6963d3ddac6e42ce25bc761343da964c24", + ] +} + +provider "registry.terraform.io/hashicorp/google-beta" { + version = "5.1.0" + hashes = [ + "h1:d0u9a3m826V3QZ7XsycvlJGduERSi1eunLoHWZjRIb0=", + "zh:3369d685b81dfad4ee307c9729ea20f6593f1dee23bc785c167c6a0b6843f208", + "zh:40cef4bbbcefc4944843915a626ab39b734ba0e7dcbe57ea9faed1e935b73efb", + "zh:50dea4f57191c8a91dcbc9db3a09381899b43e06c3f2e6767792d8ce7711e8a1", + "zh:5406877b75fdf94daeed74a69e65f6e02c40880cf22f5d91c75ca69c3c7f435a", + "zh:7d9074317ca61384a86468d40e2f30f67eec5e44e87d2eac752cdaaed0a45e83", + "zh:9188b5492c70f3826f65134f7c74ce74a933ced6e28426e9d6a9358d8c33b13d", + "zh:b06dabf01ca9f9a0cf2c0613d00a212ae2b8c2b7d3e78057f52856e385483c87", + "zh:b7ac631dbd6efea37ca94bae7d0476a13243d884a8bd8eb2d39e3398c1e9b9ad", + "zh:c927efccfab1e3afb1fdc3ba141d0e04f67fffadb55346b2b4b272a1e358fe8a", + "zh:d418d657c7b95762b6d5caae993ccc18bf54c63dec08c5a03f0aeb53403440f4", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f9e96ead5152fbb2df8571ee87810e7c3638df877ba5124e2b092faf4a3a641e", + ] +} + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.23.0" + hashes = [ + "h1:arTzD0XG/DswGCAx9JEttkSKe9RyyFW9W7UWcXF13dU=", + "zh:10488a12525ed674359585f83e3ee5e74818b5c98e033798351678b21b2f7d89", + "zh:1102ba5ca1a595f880e67102bbf999cc8b60203272a078a5b1e896d173f3f34b", + "zh:1347cf958ed3f3f80b3c7b3e23ddda3d6c6573a81847a8ee92b7df231c238bf6", + "zh:2cb18e9f5156bc1b1ee6bc580a709f7c2737d142722948f4a6c3c8efe757fa8d", + "zh:5506aa6f28dcca2a265ccf8e34478b5ec2cb43b867fe6d93b0158f01590fdadd", + "zh:6217a20686b631b1dcb448ee4bc795747ebc61b56fbe97a1ad51f375ebb0d996", + "zh:8accf916c00579c22806cb771e8909b349ffb7eb29d9c5468d0a3f3166c7a84a", + "zh:9379b0b54a0fa030b19c7b9356708ec8489e194c3b5e978df2d31368563308e5", + "zh:aa99c580890691036c2931841e88e7ee80d59ae52289c8c2c28ea0ac23e31520", + "zh:c57376d169875990ac68664d227fb69cd0037b92d0eba6921d757c3fd1879080", + "zh:e6068e3f94f6943b5586557b73f109debe19d1a75ca9273a681d22d1ce066579", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.1" + constraints = ">= 2.1.0" + hashes = [ + "h1:ydA0/SNRVB1o95btfshvYsmxA+jZFRZcvKzZSB+4S1M=", + "zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840", + "zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb", + "zh:63cff4de03af983175a7e37e52d4bd89d990be256b16b5c7f919aff5ad485aa5", + "zh:74cb22c6700e48486b7cabefa10b33b801dfcab56f1a6ac9b6624531f3d36ea3", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:79e553aff77f1cfa9012a2218b8238dd672ea5e1b2924775ac9ac24d2a75c238", + "zh:a1e06ddda0b5ac48f7e7c7d59e1ab5a4073bbcf876c73c0299e4610ed53859dc", + "zh:c37a97090f1a82222925d45d84483b2aa702ef7ab66532af6cbcfb567818b970", + "zh:e4453fbebf90c53ca3323a92e7ca0f9961427d2f0ce0d2b65523cc04d5d999c2", + "zh:e80a746921946d8b6761e77305b752ad188da60688cfd2059322875d363be5f5", + "zh:fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f", + "zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.5.1" + constraints = ">= 2.1.0" + hashes = [ + "h1:IL9mSatmwov+e0+++YX2V6uel+dV6bn+fC/cnGDK3Ck=", + "zh:04e3fbd610cb52c1017d282531364b9c53ef72b6bc533acb2a90671957324a64", + "zh:119197103301ebaf7efb91df8f0b6e0dd31e6ff943d231af35ee1831c599188d", + "zh:4d2b219d09abf3b1bb4df93d399ed156cadd61f44ad3baf5cf2954df2fba0831", + "zh:6130bdde527587bbe2dcaa7150363e96dbc5250ea20154176d82bc69df5d4ce3", + "zh:6cc326cd4000f724d3086ee05587e7710f032f94fc9af35e96a386a1c6f2214f", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:b6d88e1d28cf2dfa24e9fdcc3efc77adcdc1c3c3b5c7ce503a423efbdd6de57b", + "zh:ba74c592622ecbcef9dc2a4d81ed321c4e44cddf7da799faa324da9bf52a22b2", + "zh:c7c5cde98fe4ef1143bd1b3ec5dc04baf0d4cc3ca2c5c7d40d17c0e9b2076865", + "zh:dac4bad52c940cd0dfc27893507c1e92393846b024c5a9db159a93c534a3da03", + "zh:de8febe2a2acd9ac454b844a4106ed295ae9520ef54dc8ed2faf29f12716b602", + "zh:eab0d0495e7e711cca367f7d4df6e322e6c562fc52151ec931176115b83ed014", + ] +} diff --git a/examples/gcp/gke.tf b/examples/gcp/gke.tf new file mode 100644 index 0000000..e134539 --- /dev/null +++ b/examples/gcp/gke.tf @@ -0,0 +1,202 @@ +# Set the provider and credentials +provider "google" { + project = var.project_id + region = var.region +} + +# Create custom IAM Role +resource "google_project_iam_custom_role" "kubeip_role" { + role_id = "kubeip_role" + title = "KubeIP Role" + description = "KubeIP Role" + stage = "GA" + permissions = [ + "compute.instances.addAccessConfig", + "compute.instances.deleteAccessConfig", + "compute.instances.get", + "compute.addresses.list", + "compute.zoneOperations.get", + "compute.projects.get" + ] +} + +# Create custom IAM service account +resource "google_service_account" "kubeip_service_account" { + account_id = "kubeip-service-account" + display_name = "KubeIP Service Account" +} + +# Bind custom IAM Role to custom IAM service account +resource "google_project_iam_member" "kubeip_role_binding" { + role = google_project_iam_custom_role.kubeip_role.id + member = "serviceAccount:${google_service_account.kubeip_service_account.email}" + project = var.project_id +} + +# Create a VPC network +resource "google_compute_network" "vpc" { + name = var.vpc_name +} + +# Create a public subnet +resource "google_compute_subnetwork" "public_subnet" { + name = "public-subnet" + network = google_compute_network.vpc.id + ip_cidr_range = "10.0.1.0/24" +} + +# Create a private subnet +resource "google_compute_subnetwork" "private_subnet" { + name = "private-subnet" + network = google_compute_network.vpc.id + ip_cidr_range = "10.0.2.0/24" +} + +# Create GKE cluster +resource "google_container_cluster" "kubeip_cluster" { + name = var.cluster_name + location = var.region + + initial_node_count = 1 + remove_default_node_pool = true + + # Enable VPC-native + network = google_compute_network.vpc.id + subnetwork = google_compute_subnetwork.public_subnet.id +} + +# Create node pools +resource "google_container_node_pool" "public_node_pool" { + name = "public-node-pool" + location = google_container_cluster.kubeip_cluster.location + cluster = google_container_cluster.kubeip_cluster.name + node_count = 1 + autoscaling { + min_node_count = 1 + max_node_count = 3 + location_policy = "ANY" + } + node_config { + machine_type = "e2-medium" + spot = true + labels = { + nodegroup = "public" + kubeip = "use" + } + resource_labels = { + Environment = "demo" + kubeip = "use" + public = "true" + } + } + node_locations = [google_compute_subnetwork.public_subnet.region] +} + +resource "google_container_node_pool" "private_node_pool" { + name = "private-node-pool" + location = google_container_cluster.kubeip_cluster.location + cluster = google_container_cluster.kubeip_cluster.name + node_count = 1 + autoscaling { + min_node_count = 1 + max_node_count = 3 + location_policy = "ANY" + } + node_config { + machine_type = "e2-medium" + spot = true + labels = { + nodegroup = "private" + kubeip = "ignore" + } + resource_labels = { + Environment = "demo" + kubeip = "ignore" + public = "false" + } + } + network_config { + enable_private_nodes = true + } + node_locations = [google_compute_subnetwork.private_subnet.region] +} + +# Create static public IP addresses +resource "google_compute_address" "static_ip" { + provider = google-beta + project = var.project_id + count = 5 + name = "static-ip-${count.index}" + address_type = "EXTERNAL" + region = google_container_cluster.kubeip_cluster.location + purpose = "GCE_ENDPOINT" + labels = { + kubeip = "reserved" + } +} + +# Create Kubernetes service account in kube-system namespace +resource "kubernetes_service_account" "kubeip_service_account" { + metadata { + name = "kubeip-service-account" + namespace = "kube-system" + annotations = { + "iam.gke.io/gcp-service-account" = google_service_account.kubeip_service_account.email + } + } + depends_on = [ + google_service_account.kubeip_service_account, + google_container_cluster.kubeip_cluster + ] +} + +# Configure Workload Identity for Kube IP service account and Kubernetes service account +module "kubeip-workload-identity" { + source = "terraform-google-modules/kubernetes-engine/google//modules/workload-identity" + project_id = var.project_id + cluster_name = var.cluster_name + location = var.region + name = google_service_account.kubeip_service_account.name + namespace = "kube-system" + depends_on = [ + google_service_account.kubeip_service_account, + google_service_account.kubeip_service_account, + google_container_cluster.kubeip_cluster + ] +} + + +# Deploy KubeIP DaemonSet +#resource "kubernetes_daemonset" "kubeip_daemonset" { +# metadata { +# name = "kubeip-agent" +# namespace = "kube-system" +# labels = { +# app = "kubeip" +# } +# } +# spec { +# selector { +# match_labels = { +# app = "kubeip" +# } +# } +# template { +# metadata { +# labels = { +# app = "kubeip" +# } +# } +# spec { +# service_account_name = "kubeip-service-account" +# container { +# name = "kubeip-agent" +# image = "doitintl/kubeip-agent" +# } +# node_selector = { +# nodegroup = "public" +# } +# } +# } +# } +#} diff --git a/examples/gcp/variables.tf b/examples/gcp/variables.tf new file mode 100644 index 0000000..a41b088 --- /dev/null +++ b/examples/gcp/variables.tf @@ -0,0 +1,18 @@ +variable "project_id" { + type = string +} + +variable "region" { + type = string + default = "us-central1" +} + +variable "cluster_name" { + type = string + default = "kubeip-demo" +} + +variable "vpc_name" { + type = string + default = "kubeip-demo" +} \ No newline at end of file From 409e79f34dede77a2b6aad8c8bf9ce76e382336e Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Sun, 15 Oct 2023 18:50:45 +0300 Subject: [PATCH 38/66] fix docker exec --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 226da46..ec90421 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,5 +34,5 @@ COPY --from=builder /usr/share/zoneinfo /usr/share/zoneinfo # copy the binary to the production image from the builder stage COPY --from=builder /app/.bin/kubeip /kubeip -ENTRYPOINT ["/kubeip"] +ENTRYPOINT ["/kubeip-agent"] CMD ["run"] \ No newline at end of file From 164722711ae566998b6c9ae795629e65703bf3fd Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Sun, 15 Oct 2023 18:52:18 +0300 Subject: [PATCH 39/66] work on gke example --- examples/aws/eks.tf | 8 -- examples/gcp/.terraform.lock.hcl | 60 --------- examples/gcp/gke.tf | 205 ++++++++++++++++++------------- examples/gcp/variables.tf | 5 + 4 files changed, 126 insertions(+), 152 deletions(-) diff --git a/examples/aws/eks.tf b/examples/aws/eks.tf index a53a063..7f731cc 100644 --- a/examples/aws/eks.tf +++ b/examples/aws/eks.tf @@ -41,14 +41,6 @@ module "eks" { capacity_type = "SPOT" platform = "bottlerocket" - taints = [ - { - key = "kubeip" - value = "use" - effect = "NO_SCHEDULE" - } - ] - labels = { nodegroup = "public" kubeip = "use" diff --git a/examples/gcp/.terraform.lock.hcl b/examples/gcp/.terraform.lock.hcl index e98080c..271117d 100644 --- a/examples/gcp/.terraform.lock.hcl +++ b/examples/gcp/.terraform.lock.hcl @@ -1,26 +1,6 @@ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. -provider "registry.terraform.io/hashicorp/external" { - version = "2.3.1" - constraints = ">= 2.2.2" - hashes = [ - "h1:gznGscVJ0USxy4CdihpjRKPsKvyGr/zqPvBoFLJTQDc=", - "zh:001e2886dc81fc98cf17cf34c0d53cb2dae1e869464792576e11b0f34ee92f54", - "zh:2eeac58dd75b1abdf91945ac4284c9ccb2bfb17fa9bdb5f5d408148ff553b3ee", - "zh:2fc39079ba61411a737df2908942e6970cb67ed2f4fb19090cd44ce2082903dd", - "zh:472a71c624952cff7aa98a7b967f6c7bb53153dbd2b8f356ceb286e6743bb4e2", - "zh:4cff06d31272aac8bc35e9b7faec42cf4554cbcbae1092eaab6ab7f643c215d9", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:7ed16ccd2049fa089616b98c0bd57219f407958f318f3c697843e2397ddf70df", - "zh:842696362c92bf2645eb85c739410fd51376be6c488733efae44f4ce688da50e", - "zh:8985129f2eccfd7f1841ce06f3bf2bbede6352ec9e9f926fbaa6b1a05313b326", - "zh:a5f0602d8ec991a5411ef42f872aa90f6347e93886ce67905c53cfea37278e05", - "zh:bf4ab82cbe5256dcef16949973bf6aa1a98c2c73a98d6a44ee7bc40809d002b8", - "zh:e70770be62aa70198fa899526d671643ff99eecf265bf1a50e798fc3480bd417", - ] -} - provider "registry.terraform.io/hashicorp/google" { version = "4.84.0" constraints = ">= 3.39.0, >= 3.53.0, < 5.0.0, < 6.0.0" @@ -78,43 +58,3 @@ provider "registry.terraform.io/hashicorp/kubernetes" { "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", ] } - -provider "registry.terraform.io/hashicorp/null" { - version = "3.2.1" - constraints = ">= 2.1.0" - hashes = [ - "h1:ydA0/SNRVB1o95btfshvYsmxA+jZFRZcvKzZSB+4S1M=", - "zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840", - "zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb", - "zh:63cff4de03af983175a7e37e52d4bd89d990be256b16b5c7f919aff5ad485aa5", - "zh:74cb22c6700e48486b7cabefa10b33b801dfcab56f1a6ac9b6624531f3d36ea3", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:79e553aff77f1cfa9012a2218b8238dd672ea5e1b2924775ac9ac24d2a75c238", - "zh:a1e06ddda0b5ac48f7e7c7d59e1ab5a4073bbcf876c73c0299e4610ed53859dc", - "zh:c37a97090f1a82222925d45d84483b2aa702ef7ab66532af6cbcfb567818b970", - "zh:e4453fbebf90c53ca3323a92e7ca0f9961427d2f0ce0d2b65523cc04d5d999c2", - "zh:e80a746921946d8b6761e77305b752ad188da60688cfd2059322875d363be5f5", - "zh:fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f", - "zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694", - ] -} - -provider "registry.terraform.io/hashicorp/random" { - version = "3.5.1" - constraints = ">= 2.1.0" - hashes = [ - "h1:IL9mSatmwov+e0+++YX2V6uel+dV6bn+fC/cnGDK3Ck=", - "zh:04e3fbd610cb52c1017d282531364b9c53ef72b6bc533acb2a90671957324a64", - "zh:119197103301ebaf7efb91df8f0b6e0dd31e6ff943d231af35ee1831c599188d", - "zh:4d2b219d09abf3b1bb4df93d399ed156cadd61f44ad3baf5cf2954df2fba0831", - "zh:6130bdde527587bbe2dcaa7150363e96dbc5250ea20154176d82bc69df5d4ce3", - "zh:6cc326cd4000f724d3086ee05587e7710f032f94fc9af35e96a386a1c6f2214f", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:b6d88e1d28cf2dfa24e9fdcc3efc77adcdc1c3c3b5c7ce503a423efbdd6de57b", - "zh:ba74c592622ecbcef9dc2a4d81ed321c4e44cddf7da799faa324da9bf52a22b2", - "zh:c7c5cde98fe4ef1143bd1b3ec5dc04baf0d4cc3ca2c5c7d40d17c0e9b2076865", - "zh:dac4bad52c940cd0dfc27893507c1e92393846b024c5a9db159a93c534a3da03", - "zh:de8febe2a2acd9ac454b844a4106ed295ae9520ef54dc8ed2faf29f12716b602", - "zh:eab0d0495e7e711cca367f7d4df6e322e6c562fc52151ec931176115b83ed014", - ] -} diff --git a/examples/gcp/gke.tf b/examples/gcp/gke.tf index e134539..888ac26 100644 --- a/examples/gcp/gke.tf +++ b/examples/gcp/gke.tf @@ -1,3 +1,10 @@ +# Save state to local file +terraform { + backend "local" { + path = "terraform.tfstate" + } +} + # Set the provider and credentials provider "google" { project = var.project_id @@ -26,32 +33,34 @@ resource "google_service_account" "kubeip_service_account" { display_name = "KubeIP Service Account" } -# Bind custom IAM Role to custom IAM service account +# Bind custom IAM Role to kubeip IAM service account resource "google_project_iam_member" "kubeip_role_binding" { role = google_project_iam_custom_role.kubeip_role.id member = "serviceAccount:${google_service_account.kubeip_service_account.email}" project = var.project_id } +# Bind workload identity to kubeip IAM service account +resource "google_service_account_iam_member" "kubeip_workload_identity_binding" { + service_account_id = google_service_account.kubeip_service_account.name + role = "roles/iam.workloadIdentityUser" + member = "serviceAccount:${var.project_id}.svc.id.goog[kube-system/kubeip-service-account]" +} + # Create a VPC network resource "google_compute_network" "vpc" { - name = var.vpc_name + name = var.vpc_name + auto_create_subnetworks = false } # Create a public subnet -resource "google_compute_subnetwork" "public_subnet" { - name = "public-subnet" +resource "google_compute_subnetwork" "kubeip_subnet" { + name = "kubeip-subnet" network = google_compute_network.vpc.id + region = var.region ip_cidr_range = "10.0.1.0/24" } -# Create a private subnet -resource "google_compute_subnetwork" "private_subnet" { - name = "private-subnet" - network = google_compute_network.vpc.id - ip_cidr_range = "10.0.2.0/24" -} - # Create GKE cluster resource "google_container_cluster" "kubeip_cluster" { name = var.cluster_name @@ -60,57 +69,78 @@ resource "google_container_cluster" "kubeip_cluster" { initial_node_count = 1 remove_default_node_pool = true - # Enable VPC-native network = google_compute_network.vpc.id - subnetwork = google_compute_subnetwork.public_subnet.id + subnetwork = google_compute_subnetwork.kubeip_subnet.id + + # Enable Workload Identity + workload_identity_config { + workload_pool = "${var.project_id}.svc.id.goog" + } } # Create node pools resource "google_container_node_pool" "public_node_pool" { - name = "public-node-pool" - location = google_container_cluster.kubeip_cluster.location - cluster = google_container_cluster.kubeip_cluster.name - node_count = 1 + name = "public-node-pool" + location = google_container_cluster.kubeip_cluster.location + cluster = google_container_cluster.kubeip_cluster.name autoscaling { - min_node_count = 1 - max_node_count = 3 - location_policy = "ANY" + total_min_node_count = 1 + total_max_node_count = 5 + location_policy = "ANY" } node_config { - machine_type = "e2-medium" + machine_type = var.machine_type spot = true - labels = { + oauth_scopes = [ + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + ] + metadata = { + disable-legacy-endpoints = "true" + } + workload_metadata_config { + mode = "GKE_METADATA" + } + labels = { nodegroup = "public" kubeip = "use" } resource_labels = { - Environment = "demo" + environment = "demo" kubeip = "use" public = "true" } } - node_locations = [google_compute_subnetwork.public_subnet.region] } resource "google_container_node_pool" "private_node_pool" { - name = "private-node-pool" - location = google_container_cluster.kubeip_cluster.location - cluster = google_container_cluster.kubeip_cluster.name - node_count = 1 + name = "private-node-pool" + location = google_container_cluster.kubeip_cluster.location + cluster = google_container_cluster.kubeip_cluster.name autoscaling { - min_node_count = 1 - max_node_count = 3 - location_policy = "ANY" + total_min_node_count = 1 + total_max_node_count = 5 + location_policy = "ANY" } node_config { - machine_type = "e2-medium" + machine_type = var.machine_type spot = true - labels = { + oauth_scopes = [ + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + ] + metadata = { + disable-legacy-endpoints = "true" + } + workload_metadata_config { + mode = "GKE_METADATA" + } + labels = { nodegroup = "private" kubeip = "ignore" } resource_labels = { - Environment = "demo" + environment = "demo" kubeip = "ignore" public = "false" } @@ -118,7 +148,6 @@ resource "google_container_node_pool" "private_node_pool" { network_config { enable_private_nodes = true } - node_locations = [google_compute_subnetwork.private_subnet.region] } # Create static public IP addresses @@ -129,12 +158,22 @@ resource "google_compute_address" "static_ip" { name = "static-ip-${count.index}" address_type = "EXTERNAL" region = google_container_cluster.kubeip_cluster.location - purpose = "GCE_ENDPOINT" labels = { - kubeip = "reserved" + environment = "demo" + kubeip = "reserved" } } +data "google_client_config" "provider" {} + +provider "kubernetes" { + host = "https://${google_container_cluster.kubeip_cluster.endpoint}" + token = data.google_client_config.provider.access_token + cluster_ca_certificate = base64decode( + google_container_cluster.kubeip_cluster.master_auth[0].cluster_ca_certificate, + ) +} + # Create Kubernetes service account in kube-system namespace resource "kubernetes_service_account" "kubeip_service_account" { metadata { @@ -150,53 +189,51 @@ resource "kubernetes_service_account" "kubeip_service_account" { ] } -# Configure Workload Identity for Kube IP service account and Kubernetes service account -module "kubeip-workload-identity" { - source = "terraform-google-modules/kubernetes-engine/google//modules/workload-identity" - project_id = var.project_id - cluster_name = var.cluster_name - location = var.region - name = google_service_account.kubeip_service_account.name - namespace = "kube-system" - depends_on = [ - google_service_account.kubeip_service_account, - google_service_account.kubeip_service_account, + +# Deploy KubeIP DaemonSet +resource "kubernetes_daemonset" "kubeip_daemonset" { + metadata { + name = "kubeip-agent" + namespace = "kube-system" + labels = { + app = "kubeip" + } + } + spec { + selector { + match_labels = { + app = "kubeip" + } + } + template { + metadata { + labels = { + app = "kubeip" + } + } + spec { + service_account_name = "kubeip-service-account" + container { + name = "kubeip-agent" + image = "doitintl/kubeip-agent" + env { + name = "FILTER" + value = "label.kubeip=reserved;labels.environment=demo" + } + env { + name = "LOG_LEVEL" + value = "debug" + } + } + node_selector = { + nodegroup = "public" + kubeip = "use" + } + } + } + } + depends_on = [ + kubernetes_service_account.kubeip_service_account, google_container_cluster.kubeip_cluster ] } - - -# Deploy KubeIP DaemonSet -#resource "kubernetes_daemonset" "kubeip_daemonset" { -# metadata { -# name = "kubeip-agent" -# namespace = "kube-system" -# labels = { -# app = "kubeip" -# } -# } -# spec { -# selector { -# match_labels = { -# app = "kubeip" -# } -# } -# template { -# metadata { -# labels = { -# app = "kubeip" -# } -# } -# spec { -# service_account_name = "kubeip-service-account" -# container { -# name = "kubeip-agent" -# image = "doitintl/kubeip-agent" -# } -# node_selector = { -# nodegroup = "public" -# } -# } -# } -# } -#} diff --git a/examples/gcp/variables.tf b/examples/gcp/variables.tf index a41b088..e68d489 100644 --- a/examples/gcp/variables.tf +++ b/examples/gcp/variables.tf @@ -15,4 +15,9 @@ variable "cluster_name" { variable "vpc_name" { type = string default = "kubeip-demo" +} + +variable "machine_type" { + type = string + default = "e2-standard-2" } \ No newline at end of file From 9c4201c5907061b1560e4be63ea87a57f02843d5 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Sun, 15 Oct 2023 19:13:52 +0300 Subject: [PATCH 40/66] create k8s resources on eks --- examples/aws/eks.tf | 66 +++++++++++++++++++++++++++++++++++++++++++++ examples/gcp/gke.tf | 1 - 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/examples/aws/eks.tf b/examples/aws/eks.tf index 7f731cc..a358a97 100644 --- a/examples/aws/eks.tf +++ b/examples/aws/eks.tf @@ -134,3 +134,69 @@ resource "aws_eip" "kubeip" { kubeip = "reserved" } } + +data "aws_eks_cluster_auth" "kubeip_cluster_auth" { + name = module.eks.cluster_name +} + +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + token = data.aws_eks_cluster_auth.kubeip_cluster_auth.token +} + +resource "kubernetes_service_account" "kubeip_service_account" { + metadata { + name = "kubeip-service-account" + namespace = "kube-system" + annotations = { + "eks.amazonaws.com/role-arn" = module.kubeip_eks_role.iam_role_arn + } + } + depends_on = [module.eks] +} + +# Deploy KubeIP DaemonSet +resource "kubernetes_daemonset" "kubeip_daemonset" { + metadata { + name = "kubeip-agent" + namespace = "kube-system" + labels = { + app = "kubeip" + } + } + spec { + selector { + match_labels = { + app = "kubeip" + } + } + template { + metadata { + labels = { + app = "kubeip" + } + } + spec { + service_account_name = "kubeip-service-account" + container { + name = "kubeip-agent" + image = "doitintl/kubeip-agent" + env { + name = "FILTER" + value = "label.kubeip=reserved;labels.environment=demo" + } + env { + name = "LOG_LEVEL" + value = "debug" + } + } + node_selector = { + nodegroup = "public" + kubeip = "use" + } + } + } + } + depends_on = [kubernetes_service_account.kubeip_service_account] +} diff --git a/examples/gcp/gke.tf b/examples/gcp/gke.tf index 888ac26..e76f138 100644 --- a/examples/gcp/gke.tf +++ b/examples/gcp/gke.tf @@ -189,7 +189,6 @@ resource "kubernetes_service_account" "kubeip_service_account" { ] } - # Deploy KubeIP DaemonSet resource "kubernetes_daemonset" "kubeip_daemonset" { metadata { From b8d38c61b70ec9afebc994a67ac3b53d5a9f5011 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Mon, 16 Oct 2023 12:16:58 +0300 Subject: [PATCH 41/66] fix daemonset --- Dockerfile | 2 +- examples/gcp/gke.tf | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ec90421..9ecbf4a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,7 +32,7 @@ COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certifi # copy timezone settings COPY --from=builder /usr/share/zoneinfo /usr/share/zoneinfo # copy the binary to the production image from the builder stage -COPY --from=builder /app/.bin/kubeip /kubeip +COPY --from=builder /app/.bin/kubeip-agent /kubeip-agent ENTRYPOINT ["/kubeip-agent"] CMD ["run"] \ No newline at end of file diff --git a/examples/gcp/gke.tf b/examples/gcp/gke.tf index e76f138..aa0b967 100644 --- a/examples/gcp/gke.tf +++ b/examples/gcp/gke.tf @@ -223,11 +223,26 @@ resource "kubernetes_daemonset" "kubeip_daemonset" { name = "LOG_LEVEL" value = "debug" } + volume_mount { + mount_path = "/etc/podinfo" + name = "podinfo" + } } node_selector = { nodegroup = "public" kubeip = "use" } + volume { + name = "podinfo" + downward_api { + items { + path = "nodeName" + field_ref { + field_path = "spec.nodeName" + } + } + } + } } } } From 0e012c86e9a20cc3e9bec00d9efc5dcb687fa967 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Mon, 16 Oct 2023 16:23:29 +0300 Subject: [PATCH 42/66] made gke demo to work --- README.md | 37 +++++++++++++++++++ examples/aws/eks.tf | 48 +++++++++++++++++++++++- examples/gcp/gke.tf | 78 +++++++++++++++++++++++++++------------ examples/gcp/variables.tf | 2 +- roles.yaml | 16 -------- 5 files changed, 140 insertions(+), 41 deletions(-) delete mode 100644 roles.yaml diff --git a/README.md b/README.md index 22592a4..69804b6 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,42 @@ Deploy KubeIP as a DaemonSet on your desired nodes using standard Kubernetes sel to each node it operates on. If no static public IP is available, KubeIP will wait until one becomes available. When a node is deleted, KubeIP will release the static public IP. +### Kubernetes Service Account + +KubeIP requires a Kubernetes service account with the following permissions: + +```yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubeip-service-account + namespace: kube-system +--- + +piVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubeip-cluster-role +rules: + - apiGroups: [ "" ] + resources: [ "nodes" ] + verbs: [ "get" ] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubeip-cluster-role-binding +subjects: + - kind: ServiceAccount + name: kubeip-service-account + namespace: kube-system +roleRef: + kind: ClusterRole + name: kubeip-cluster-role + apiGroup: rbac.authorization.k8s.io +``` + ### AWS Make sure that KubeIP DaemonSet is deployed on nodes that have a public IP (node in public subnet) and uses Kubernetes service @@ -87,6 +123,7 @@ includedPermissions: - compute.instances.get - compute.addresses.list - compute.zoneOperations.get + - compute.subnetworks.useExternalIp - compute.projects.get ``` diff --git a/examples/aws/eks.tf b/examples/aws/eks.tf index a358a97..6104967 100644 --- a/examples/aws/eks.tf +++ b/examples/aws/eks.tf @@ -156,6 +156,44 @@ resource "kubernetes_service_account" "kubeip_service_account" { depends_on = [module.eks] } +# Create cluster role with get node permission +resource "kubernetes_cluster_role" "kubeip_cluster_role" { + metadata { + name = "kubeip-cluster-role" + } + rule { + api_groups = ["*"] + resources = ["nodes"] + verbs = ["get"] + } + depends_on = [ + kubernetes_service_account.kubeip_service_account, + module.eks + ] +} + +# Bind cluster role to kubeip service account +resource "kubernetes_cluster_role_binding" "kubeip_cluster_role_binding" { + metadata { + name = "kubeip-cluster-role-binding" + } + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = kubernetes_cluster_role.kubeip_cluster_role.metadata[0].name + } + subject { + kind = "ServiceAccount" + name = kubernetes_service_account.kubeip_service_account.metadata[0].name + namespace = kubernetes_service_account.kubeip_service_account.metadata[0].namespace + } + depends_on = [ + kubernetes_service_account.kubeip_service_account, + kubernetes_cluster_role.kubeip_cluster_role + ] +} + + # Deploy KubeIP DaemonSet resource "kubernetes_daemonset" "kubeip_daemonset" { metadata { @@ -182,9 +220,17 @@ resource "kubernetes_daemonset" "kubeip_daemonset" { container { name = "kubeip-agent" image = "doitintl/kubeip-agent" + env { + name = "NODE_NAME" + value_from { + field_ref { + field_path = "spec.nodeName" + } + } + } env { name = "FILTER" - value = "label.kubeip=reserved;labels.environment=demo" + value = "Name=tag:kubeip,Values=reserved;Name=tag:environment,Values=demo" } env { name = "LOG_LEVEL" diff --git a/examples/gcp/gke.tf b/examples/gcp/gke.tf index aa0b967..cf36425 100644 --- a/examples/gcp/gke.tf +++ b/examples/gcp/gke.tf @@ -15,7 +15,7 @@ provider "google" { resource "google_project_iam_custom_role" "kubeip_role" { role_id = "kubeip_role" title = "KubeIP Role" - description = "KubeIP Role" + description = "KubeIP required permissions" stage = "GA" permissions = [ "compute.instances.addAccessConfig", @@ -23,6 +23,8 @@ resource "google_project_iam_custom_role" "kubeip_role" { "compute.instances.get", "compute.addresses.list", "compute.zoneOperations.get", + "compute.zoneOperations.list", + "compute.subnetworks.useExternalIp", "compute.projects.get" ] } @@ -84,9 +86,9 @@ resource "google_container_node_pool" "public_node_pool" { location = google_container_cluster.kubeip_cluster.location cluster = google_container_cluster.kubeip_cluster.name autoscaling { - total_min_node_count = 1 - total_max_node_count = 5 - location_policy = "ANY" + min_node_count = 1 + max_node_count = 2 + location_policy = "ANY" } node_config { machine_type = var.machine_type @@ -118,9 +120,9 @@ resource "google_container_node_pool" "private_node_pool" { location = google_container_cluster.kubeip_cluster.location cluster = google_container_cluster.kubeip_cluster.name autoscaling { - total_min_node_count = 1 - total_max_node_count = 5 - location_policy = "ANY" + min_node_count = 1 + max_node_count = 2 + location_policy = "ANY" } node_config { machine_type = var.machine_type @@ -189,6 +191,43 @@ resource "kubernetes_service_account" "kubeip_service_account" { ] } +# Create cluster role with get node permission +resource "kubernetes_cluster_role" "kubeip_cluster_role" { + metadata { + name = "kubeip-cluster-role" + } + rule { + api_groups = ["*"] + resources = ["nodes"] + verbs = ["get"] + } + depends_on = [ + kubernetes_service_account.kubeip_service_account, + google_container_cluster.kubeip_cluster + ] +} + +# Bind cluster role to kubeip service account +resource "kubernetes_cluster_role_binding" "kubeip_cluster_role_binding" { + metadata { + name = "kubeip-cluster-role-binding" + } + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = kubernetes_cluster_role.kubeip_cluster_role.metadata[0].name + } + subject { + kind = "ServiceAccount" + name = kubernetes_service_account.kubeip_service_account.metadata[0].name + namespace = kubernetes_service_account.kubeip_service_account.metadata[0].namespace + } + depends_on = [ + kubernetes_service_account.kubeip_service_account, + kubernetes_cluster_role.kubeip_cluster_role + ] +} + # Deploy KubeIP DaemonSet resource "kubernetes_daemonset" "kubeip_daemonset" { metadata { @@ -215,34 +254,27 @@ resource "kubernetes_daemonset" "kubeip_daemonset" { container { name = "kubeip-agent" image = "doitintl/kubeip-agent" + env { + name = "NODE_NAME" + value_from { + field_ref { + field_path = "spec.nodeName" + } + } + } env { name = "FILTER" - value = "label.kubeip=reserved;labels.environment=demo" + value = "labels.kubeip=reserved;labels.environment=demo" } env { name = "LOG_LEVEL" value = "debug" } - volume_mount { - mount_path = "/etc/podinfo" - name = "podinfo" - } } node_selector = { nodegroup = "public" kubeip = "use" } - volume { - name = "podinfo" - downward_api { - items { - path = "nodeName" - field_ref { - field_path = "spec.nodeName" - } - } - } - } } } } diff --git a/examples/gcp/variables.tf b/examples/gcp/variables.tf index e68d489..3bd185f 100644 --- a/examples/gcp/variables.tf +++ b/examples/gcp/variables.tf @@ -19,5 +19,5 @@ variable "vpc_name" { variable "machine_type" { type = string - default = "e2-standard-2" + default = "e2-medium" } \ No newline at end of file diff --git a/roles.yaml b/roles.yaml deleted file mode 100644 index c2e031c..0000000 --- a/roles.yaml +++ /dev/null @@ -1,16 +0,0 @@ -title: "kubeip" -description: "required permissions to run KubeIP" -stage: "GA" -includedPermissions: -- compute.addresses.list -- compute.instances.addAccessConfig -- compute.instances.deleteAccessConfig -- compute.instances.get -- compute.instances.list -- compute.projects.get -- container.clusters.get -- container.clusters.list -- resourcemanager.projects.get -- compute.networks.useExternalIp -- compute.subnetworks.useExternalIp -- compute.addresses.use From c2a5f830e69623bf43acd11265b109f0fceebd13 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Mon, 16 Oct 2023 17:31:47 +0300 Subject: [PATCH 43/66] private GCP API access; do not fail on wait --- README.md | 1 + examples/gcp/gke.tf | 10 ++++++---- internal/address/gcp.go | 6 ++++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 69804b6..7d7a8ab 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,7 @@ includedPermissions: - compute.instances.deleteAccessConfig - compute.instances.get - compute.addresses.list + - compute.addresses.use - compute.zoneOperations.get - compute.subnetworks.useExternalIp - compute.projects.get diff --git a/examples/gcp/gke.tf b/examples/gcp/gke.tf index cf36425..5c8446d 100644 --- a/examples/gcp/gke.tf +++ b/examples/gcp/gke.tf @@ -22,6 +22,7 @@ resource "google_project_iam_custom_role" "kubeip_role" { "compute.instances.deleteAccessConfig", "compute.instances.get", "compute.addresses.list", + "compute.addresses.use", "compute.zoneOperations.get", "compute.zoneOperations.list", "compute.subnetworks.useExternalIp", @@ -57,10 +58,11 @@ resource "google_compute_network" "vpc" { # Create a public subnet resource "google_compute_subnetwork" "kubeip_subnet" { - name = "kubeip-subnet" - network = google_compute_network.vpc.id - region = var.region - ip_cidr_range = "10.0.1.0/24" + name = "kubeip-subnet" + network = google_compute_network.vpc.id + region = var.region + ip_cidr_range = "10.0.1.0/24" + private_ip_google_access = true } # Create GKE cluster diff --git a/internal/address/gcp.go b/internal/address/gcp.go index 1d07b9b..d847aa2 100644 --- a/internal/address/gcp.go +++ b/internal/address/gcp.go @@ -127,7 +127,8 @@ func (a *gcpAssigner) deleteInstanceAddress(ctx context.Context, instance *compu } // wait for operation to complete if err = a.waitForOperation(ctx, op, zone, defaultTimeout); err != nil { - return errors.Wrapf(err, "failed to wait for operation %s", op.Name) + // log error and continue + a.logger.WithError(err).Errorf("failed to wait for operation %s", op.Name) } return nil } @@ -149,7 +150,8 @@ func (a *gcpAssigner) addInstanceAddress(ctx context.Context, instance *compute. } // wait for operation to complete if err = a.waitForOperation(ctx, op, zone, defaultTimeout); err != nil { - return errors.Wrapf(err, "failed to wait for operation %s", op.Name) + // log error and continue + a.logger.WithError(err).Errorf("failed to wait for operation %s", op.Name) } return nil } From 9dbb3bddf890cd6bb8f64128478a2aae8b5481cf Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Mon, 16 Oct 2023 17:34:25 +0300 Subject: [PATCH 44/66] mod tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 70dd964..16ad7d7 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.21 require ( cloud.google.com/go/compute/metadata v0.2.3 + github.com/aws/aws-sdk-go-v2 v1.21.1 github.com/aws/aws-sdk-go-v2/config v1.18.44 github.com/aws/aws-sdk-go-v2/service/ec2 v1.124.0 github.com/pkg/errors v0.9.1 @@ -19,7 +20,6 @@ require ( require ( cloud.google.com/go/compute v1.23.0 // indirect - github.com/aws/aws-sdk-go-v2 v1.21.1 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.13.42 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.12 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.42 // indirect From 305e33d66610be2bc0381c341683beb0aa87012c Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Mon, 16 Oct 2023 17:40:42 +0300 Subject: [PATCH 45/66] remove debug from build --- makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/makefile b/makefile index 8d8d17b..8a3fb5c 100644 --- a/makefile +++ b/makefile @@ -46,7 +46,8 @@ setup-mockery: build: ; $(info $(M) building $(GOOS)/$(GOARCH) binary...) @ ## build with local Go SDK $(GOBUILD) -v \ - -ldflags '-X main.version=$(VERSION) -X main.buildDate=$(DATE) -X main.gitCommit=$(COMMIT) -X main.gitBranch=$(BRANCH)' \ + -tags release \ + -ldflags '-s -w -X main.version=$(VERSION) -X main.buildDate=$(DATE) -X main.gitCommit=$(COMMIT) -X main.gitBranch=$(BRANCH)' \ -o $(BIN)/$(BINARY_NAME) ./cmd/. lint: setup-lint; $(info $(M) running golangci-lint ...) @ ## run golangci-lint linters From 59f9d441fde13f100e4f0f311cb55864ba019571 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Mon, 16 Oct 2023 18:42:33 +0300 Subject: [PATCH 46/66] debug log and vpc-native for gke --- cmd/main.go | 1 + examples/gcp/gke.tf | 29 ++++++++++++++++++++++------- examples/gcp/variables.tf | 25 +++++++++++++++++++++++++ internal/types/node.go | 10 +++++++++- 4 files changed, 57 insertions(+), 8 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 9a92bd3..e07e550 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -134,6 +134,7 @@ func run(c context.Context, log *logrus.Entry, cfg *config.Config) error { if err != nil { return errors.Wrap(err, "getting node") } + log.WithField("node", n).Debug("node discovery done") // assign static public IP address with retry (interval and attempts) assigner, err := address.NewAssigner(ctx, log, n.Cloud, cfg) diff --git a/examples/gcp/gke.tf b/examples/gcp/gke.tf index 5c8446d..6a17779 100644 --- a/examples/gcp/gke.tf +++ b/examples/gcp/gke.tf @@ -61,8 +61,16 @@ resource "google_compute_subnetwork" "kubeip_subnet" { name = "kubeip-subnet" network = google_compute_network.vpc.id region = var.region - ip_cidr_range = "10.0.1.0/24" + ip_cidr_range = var.subnet_range private_ip_google_access = true + secondary_ip_range { + range_name = var.services_range_name + ip_cidr_range = var.services_range + } + secondary_ip_range { + range_name = var.pods_range_name + ip_cidr_range = var.pods_range + } } # Create GKE cluster @@ -76,6 +84,11 @@ resource "google_container_cluster" "kubeip_cluster" { network = google_compute_network.vpc.id subnetwork = google_compute_subnetwork.kubeip_subnet.id + ip_allocation_policy { + services_secondary_range_name = var.services_range_name + cluster_secondary_range_name = var.pods_range_name + } + # Enable Workload Identity workload_identity_config { workload_pool = "${var.project_id}.svc.id.goog" @@ -84,9 +97,10 @@ resource "google_container_cluster" "kubeip_cluster" { # Create node pools resource "google_container_node_pool" "public_node_pool" { - name = "public-node-pool" - location = google_container_cluster.kubeip_cluster.location - cluster = google_container_cluster.kubeip_cluster.name + name = "public-node-pool" + location = google_container_cluster.kubeip_cluster.location + cluster = google_container_cluster.kubeip_cluster.name + initial_node_count = 1 autoscaling { min_node_count = 1 max_node_count = 2 @@ -118,9 +132,10 @@ resource "google_container_node_pool" "public_node_pool" { } resource "google_container_node_pool" "private_node_pool" { - name = "private-node-pool" - location = google_container_cluster.kubeip_cluster.location - cluster = google_container_cluster.kubeip_cluster.name + name = "private-node-pool" + location = google_container_cluster.kubeip_cluster.location + cluster = google_container_cluster.kubeip_cluster.name + initial_node_count = 1 autoscaling { min_node_count = 1 max_node_count = 2 diff --git a/examples/gcp/variables.tf b/examples/gcp/variables.tf index 3bd185f..e03524b 100644 --- a/examples/gcp/variables.tf +++ b/examples/gcp/variables.tf @@ -17,6 +17,31 @@ variable "vpc_name" { default = "kubeip-demo" } +variable "subnet_range" { + type = string + default = "10.128.0.0/20" +} + +variable "pods_range" { + type = string + default = "10.128.64.0/18" +} + +variable "pods_range_name" { + type = string + default = "pods-range" +} + +variable "services_range_name" { + type = string + default = "services-range" +} + +variable "services_range" { + type = string + default = "10.128.32.0/20" +} + variable "machine_type" { type = string default = "e2-medium" diff --git a/internal/types/node.go b/internal/types/node.go index 0632d06..286cf1f 100644 --- a/internal/types/node.go +++ b/internal/types/node.go @@ -1,6 +1,9 @@ package types -import "net" +import ( + "fmt" + "net" +) type CloudProvider string @@ -20,3 +23,8 @@ type Node struct { ExternalIPs []net.IP InternalIPs []net.IP } + +// Stringer interface: all fields with name and value +func (n *Node) String() string { + return fmt.Sprintf("%+v", *n) +} From 4bd6b26caba275ec8e31fa95823cd18991409a03 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Tue, 17 Oct 2023 10:30:56 +0300 Subject: [PATCH 47/66] add ephemeral public IP on Unassign after removing static public IP --- internal/address/gcp.go | 31 +++++++++++++++++++++++++++---- internal/address/gcp_test.go | 2 +- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/internal/address/gcp.go b/internal/address/gcp.go index d847aa2..cb32654 100644 --- a/internal/address/gcp.go +++ b/internal/address/gcp.go @@ -19,6 +19,7 @@ const ( reservedStatus = "RESERVED" // static IP addresses that are reserved but not currently in use defaultTimeout = 10 * time.Minute defaultNetworkInterface = "nic0" + defaultNetworkName = "External NAT" accessConfigType = "ONE_TO_ONE_NAT" accessConfigKind = "compute#accessConfig" ) @@ -71,6 +72,17 @@ func NewGCPAssigner(ctx context.Context, logger *logrus.Entry, project, region s }, nil } +func joinErrorMessages(operationError *compute.OperationError) string { + if operationError == nil || len(operationError.Errors) == 0 { + return "" + } + messages := make([]string, 0, len(operationError.Errors)) + for _, errorItem := range operationError.Errors { + messages = append(messages, errorItem.Message) + } + return strings.Join(messages, "; ") +} + func (a *gcpAssigner) waitForOperation(c context.Context, op *compute.Operation, zone string, timeout time.Duration) error { if op == nil { a.logger.Warn("operation is nil") @@ -94,7 +106,7 @@ func (a *gcpAssigner) waitForOperation(c context.Context, op *compute.Operation, } // If the operation has an error, return it if op != nil && op.Error != nil { - return errors.Errorf("operation %s failed with error %v", op.Name, op.Error.Errors) + return errors.Errorf("operation %s failed with error %v", op.Name, joinErrorMessages(op.Error)) } } return nil @@ -134,19 +146,26 @@ func (a *gcpAssigner) deleteInstanceAddress(ctx context.Context, instance *compu } func (a *gcpAssigner) addInstanceAddress(ctx context.Context, instance *compute.Instance, zone string, address *compute.Address) error { + // empty address means ephemeral public IP address + natIP := "" + name := defaultNetworkName + if address != nil { + natIP = address.Address + name = address.Name + } // add instance network interface access config a.logger.WithFields(logrus.Fields{ "instance": instance.Name, "address": address.Address, }).Infof("adding reserved public IP address to instance") op, err := a.addressManager.AddAccessConfig(a.project, zone, instance.Name, defaultNetworkInterface, &compute.AccessConfig{ - Name: address.Name, + Name: name, Type: accessConfigType, Kind: accessConfigKind, - NatIP: address.Address, + NatIP: natIP, }) if err != nil { - return errors.Wrapf(err, "failed to add access config %s to instance %s", address.Name, instance.Name) + return errors.Wrapf(err, "failed to add access config %s to instance %s", name, instance.Name) } // wait for operation to complete if err = a.waitForOperation(ctx, op, zone, defaultTimeout); err != nil { @@ -256,6 +275,10 @@ func (a *gcpAssigner) Unassign(ctx context.Context, instanceID, zone string) err if err = a.deleteInstanceAddress(ctx, instance, zone); err != nil { return errors.Wrap(err, "failed to delete current public IP address") } + // assign ephemeral public IP address to the instance (pass nil address) + if err = a.addInstanceAddress(ctx, instance, zone, nil); err != nil { + return errors.Wrap(err, "failed to assign ephemeral public IP address") + } // break the loop after deleting the address return nil } diff --git a/internal/address/gcp_test.go b/internal/address/gcp_test.go index 493fab7..496f4bd 100644 --- a/internal/address/gcp_test.go +++ b/internal/address/gcp_test.go @@ -342,7 +342,7 @@ func Test_gcpAssigner_Assign(t *testing.T) { wantErr bool }{ { - name: "assign address successfully", + name: "assign static IP address successfully", fields: fields{ project: "test-project", region: "test-region", From 80cbaabac4297df609f16cff7f4c85f8eb662499 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Tue, 17 Oct 2023 12:46:55 +0300 Subject: [PATCH 48/66] moer robust ip assignment --- cmd/main.go | 4 +- cmd/main_test.go | 2 +- internal/address/gcp.go | 101 +++++++++++++++++++++++++++-------- internal/address/gcp_test.go | 1 - 4 files changed, 81 insertions(+), 27 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index e07e550..8e40689 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -36,8 +36,8 @@ var ( const ( // DefaultRetryInterval is the default retry interval - defaultRetryInterval = 5 * time.Minute - defaultRetryAttempts = 10 + defaultRetryInterval = time.Minute + defaultRetryAttempts = 60 ) func prepareLogger(level string, json bool) *logrus.Entry { diff --git a/cmd/main_test.go b/cmd/main_test.go index 3bd62b6..dda1580 100644 --- a/cmd/main_test.go +++ b/cmd/main_test.go @@ -101,7 +101,7 @@ func Test_assignAddress(t *testing.T) { name: "error after a few retries and context is done", args: args{ c: func() context.Context { - ctx, _ := context.WithTimeout(context.Background(), 10*time.Millisecond) + ctx, _ := context.WithTimeout(context.Background(), 10*time.Millisecond) //nolint:govet return ctx }(), assignerFn: func(t *testing.T) address.Assigner { diff --git a/internal/address/gcp.go b/internal/address/gcp.go index cb32654..5daf2aa 100644 --- a/internal/address/gcp.go +++ b/internal/address/gcp.go @@ -3,6 +3,7 @@ package address import ( "context" "fmt" + "math/rand" "strings" "time" @@ -34,6 +35,38 @@ type gcpAssigner struct { logger *logrus.Entry } +type operationError struct { + name string + err *compute.OperationError +} + +func newOperationError(name string, err *compute.OperationError) *operationError { + return &operationError{name: name, err: err} +} + +func isOperationError(err error) bool { + _, ok := err.(*operationError) + return ok +} + +func joinErrorMessages(operationError *compute.OperationError) string { + if operationError == nil || len(operationError.Errors) == 0 { + return "" + } + messages := make([]string, 0, len(operationError.Errors)) + for _, errorItem := range operationError.Errors { + messages = append(messages, errorItem.Message) + } + return strings.Join(messages, "; ") +} + +func (e *operationError) Error() string { + if e.err == nil { + return "" + } + return fmt.Sprintf("operation %s failed with error %v", e.name, joinErrorMessages(e.err)) +} + func NewGCPAssigner(ctx context.Context, logger *logrus.Entry, project, region string) (Assigner, error) { // initialize Google Cloud client client, err := compute.NewService(ctx) @@ -72,17 +105,6 @@ func NewGCPAssigner(ctx context.Context, logger *logrus.Entry, project, region s }, nil } -func joinErrorMessages(operationError *compute.OperationError) string { - if operationError == nil || len(operationError.Errors) == 0 { - return "" - } - messages := make([]string, 0, len(operationError.Errors)) - for _, errorItem := range operationError.Errors { - messages = append(messages, errorItem.Message) - } - return strings.Join(messages, "; ") -} - func (a *gcpAssigner) waitForOperation(c context.Context, op *compute.Operation, zone string, timeout time.Duration) error { if op == nil { a.logger.Warn("operation is nil") @@ -106,7 +128,7 @@ func (a *gcpAssigner) waitForOperation(c context.Context, op *compute.Operation, } // If the operation has an error, return it if op != nil && op.Error != nil { - return errors.Errorf("operation %s failed with error %v", op.Name, joinErrorMessages(op.Error)) + return newOperationError(op.Name, op.Error) } } return nil @@ -139,8 +161,12 @@ func (a *gcpAssigner) deleteInstanceAddress(ctx context.Context, instance *compu } // wait for operation to complete if err = a.waitForOperation(ctx, op, zone, defaultTimeout); err != nil { - // log error and continue - a.logger.WithError(err).Errorf("failed to wait for operation %s", op.Name) + // return error if operation failed + if isOperationError(err) { + return err + } + // log error and continue (ignore non-operation errors) + a.logger.WithError(err).Errorf("failed waiting for operation %s", op.Name) } return nil } @@ -149,15 +175,19 @@ func (a *gcpAssigner) addInstanceAddress(ctx context.Context, instance *compute. // empty address means ephemeral public IP address natIP := "" name := defaultNetworkName + kind := "ephemeral" if address != nil { natIP = address.Address name = address.Name + kind = "static" } // add instance network interface access config a.logger.WithFields(logrus.Fields{ - "instance": instance.Name, - "address": address.Address, - }).Infof("adding reserved public IP address to instance") + "instance": instance.Name, + "address-name": name, + "address-ip": natIP, + "kind": kind, + }).Info("adding public IP address to instance") op, err := a.addressManager.AddAccessConfig(a.project, zone, instance.Name, defaultNetworkInterface, &compute.AccessConfig{ Name: name, Type: accessConfigType, @@ -169,13 +199,24 @@ func (a *gcpAssigner) addInstanceAddress(ctx context.Context, instance *compute. } // wait for operation to complete if err = a.waitForOperation(ctx, op, zone, defaultTimeout); err != nil { - // log error and continue - a.logger.WithError(err).Errorf("failed to wait for operation %s", op.Name) + // return error if operation failed + if isOperationError(err) { + return err + } + // log error and continue (ignore non-operation errors) + a.logger.WithError(err).Errorf("failed waiting for operation %s", op.Name) } return nil } func (a *gcpAssigner) Assign(ctx context.Context, instanceID, zone string, filter []string, orderBy string) error { + // insert random delay (0-60s) to reduce collisions + // between multiple kubeip instances running in the same cluster + r := rand.New(rand.NewSource(time.Now().UnixNano())) //nolint:gosec + random := time.Duration(r.Intn(61)) * time.Second //nolint:gomnd + a.logger.WithField("delay", random).Debug("inserting random delay") + time.Sleep(random) + // check if instance already has a public static IP address assigned instance, err := a.instanceGetter.Get(a.project, zone, instanceID) if err != nil { @@ -207,18 +248,32 @@ func (a *gcpAssigner) Assign(ctx context.Context, instanceID, zone string, filte if len(addresses) == 0 { return errors.Errorf("no available addresses") } + // log available addresses IPs + ips := make([]string, 0, len(addresses)) + for _, address := range addresses { + ips = append(ips, address.Address) + } + a.logger.WithField("addresses", ips).Debugf("found %d available addresses", len(addresses)) // delete current ephemeral public IP address if err = a.deleteInstanceAddress(ctx, instance, zone); err != nil { return errors.Wrap(err, "failed to delete current public IP address") } - // assign first available static public IP address to the instance - address := addresses[0] - if err = a.addInstanceAddress(ctx, instance, zone, address); err != nil { + // try to assign all available addresses until one succeeds + // due to concurrency, it is possible that another kubeip instance will assign the same address + for _, address := range addresses { + if err = a.addInstanceAddress(ctx, instance, zone, address); err != nil { + a.logger.WithError(err).Errorf("failed to assign static public IP address %s", address.Address) + a.logger.Debug("trying next address") + continue + } + // break the loop after assigning the address + break + } + if err != nil { return errors.Wrap(err, "failed to assign static public IP address") } - return nil } diff --git a/internal/address/gcp_test.go b/internal/address/gcp_test.go index 496f4bd..633709c 100644 --- a/internal/address/gcp_test.go +++ b/internal/address/gcp_test.go @@ -128,7 +128,6 @@ func Test_gcpAssigner_waitForOperation(t *testing.T) { type fields struct { waiterFn func(t *testing.T) cloud.ZoneWaiter project string - logger *logrus.Entry } type args struct { op *compute.Operation From c026a8294b76f291f8bf64bdd33191bcc632252b Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Tue, 17 Oct 2023 13:05:59 +0300 Subject: [PATCH 49/66] check if address already assigned --- README.md | 1 + examples/aws/eks.tf | 3 +- examples/gcp/gke.tf | 4 ++- internal/address/gcp.go | 8 ----- internal/cloud/gcp_address.go | 5 ++++ mocks/cloud/AddressManager.go | 56 +++++++++++++++++++++++++++++++++++ 6 files changed, 67 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 7d7a8ab..711303d 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,7 @@ includedPermissions: - compute.instances.addAccessConfig - compute.instances.deleteAccessConfig - compute.instances.get + - compute.addresses.get - compute.addresses.list - compute.addresses.use - compute.zoneOperations.get diff --git a/examples/aws/eks.tf b/examples/aws/eks.tf index 6104967..ff83515 100644 --- a/examples/aws/eks.tf +++ b/examples/aws/eks.tf @@ -216,7 +216,8 @@ resource "kubernetes_daemonset" "kubeip_daemonset" { } } spec { - service_account_name = "kubeip-service-account" + service_account_name = "kubeip-service-account" + termination_grace_period_seconds = 30 container { name = "kubeip-agent" image = "doitintl/kubeip-agent" diff --git a/examples/gcp/gke.tf b/examples/gcp/gke.tf index 6a17779..8b138c5 100644 --- a/examples/gcp/gke.tf +++ b/examples/gcp/gke.tf @@ -21,6 +21,7 @@ resource "google_project_iam_custom_role" "kubeip_role" { "compute.instances.addAccessConfig", "compute.instances.deleteAccessConfig", "compute.instances.get", + "compute.addresses.get", "compute.addresses.list", "compute.addresses.use", "compute.zoneOperations.get", @@ -267,7 +268,8 @@ resource "kubernetes_daemonset" "kubeip_daemonset" { } } spec { - service_account_name = "kubeip-service-account" + service_account_name = "kubeip-service-account" + termination_grace_period_seconds = 30 container { name = "kubeip-agent" image = "doitintl/kubeip-agent" diff --git a/internal/address/gcp.go b/internal/address/gcp.go index 5daf2aa..9f20360 100644 --- a/internal/address/gcp.go +++ b/internal/address/gcp.go @@ -3,7 +3,6 @@ package address import ( "context" "fmt" - "math/rand" "strings" "time" @@ -210,13 +209,6 @@ func (a *gcpAssigner) addInstanceAddress(ctx context.Context, instance *compute. } func (a *gcpAssigner) Assign(ctx context.Context, instanceID, zone string, filter []string, orderBy string) error { - // insert random delay (0-60s) to reduce collisions - // between multiple kubeip instances running in the same cluster - r := rand.New(rand.NewSource(time.Now().UnixNano())) //nolint:gosec - random := time.Duration(r.Intn(61)) * time.Second //nolint:gomnd - a.logger.WithField("delay", random).Debug("inserting random delay") - time.Sleep(random) - // check if instance already has a public static IP address assigned instance, err := a.instanceGetter.Get(a.project, zone, instanceID) if err != nil { diff --git a/internal/cloud/gcp_address.go b/internal/cloud/gcp_address.go index 4b8adb7..90532cc 100644 --- a/internal/cloud/gcp_address.go +++ b/internal/cloud/gcp_address.go @@ -7,6 +7,7 @@ import ( type AddressManager interface { AddAccessConfig(project string, zone string, instance string, networkInterface string, accessconfig *compute.AccessConfig) (*compute.Operation, error) DeleteAccessConfig(project string, zone string, instance string, accessConfig string, networkInterface string) (*compute.Operation, error) + GetAddress(project, region, name string) (*compute.Address, error) } type addressManager struct { @@ -24,3 +25,7 @@ func (m *addressManager) AddAccessConfig(project, zone, instance, networkInterfa func (m *addressManager) DeleteAccessConfig(project, zone, instance, accessConfig, networkInterface string) (*compute.Operation, error) { return m.client.Instances.DeleteAccessConfig(project, zone, instance, accessConfig, networkInterface).Do() //nolint:wrapcheck } + +func (m *addressManager) GetAddress(project, region, name string) (*compute.Address, error) { + return m.client.Addresses.Get(project, region, name).Do() //nolint:wrapcheck +} diff --git a/mocks/cloud/AddressManager.go b/mocks/cloud/AddressManager.go index b925b2c..617d6d7 100644 --- a/mocks/cloud/AddressManager.go +++ b/mocks/cloud/AddressManager.go @@ -136,6 +136,62 @@ func (_c *AddressManager_DeleteAccessConfig_Call) RunAndReturn(run func(string, return _c } +// GetAddress provides a mock function with given fields: project, region, name +func (_m *AddressManager) GetAddress(project string, region string, name string) (*compute.Address, error) { + ret := _m.Called(project, region, name) + + var r0 *compute.Address + var r1 error + if rf, ok := ret.Get(0).(func(string, string, string) (*compute.Address, error)); ok { + return rf(project, region, name) + } + if rf, ok := ret.Get(0).(func(string, string, string) *compute.Address); ok { + r0 = rf(project, region, name) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*compute.Address) + } + } + + if rf, ok := ret.Get(1).(func(string, string, string) error); ok { + r1 = rf(project, region, name) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// AddressManager_GetAddress_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetAddress' +type AddressManager_GetAddress_Call struct { + *mock.Call +} + +// GetAddress is a helper method to define mock.On call +// - project string +// - region string +// - name string +func (_e *AddressManager_Expecter) GetAddress(project interface{}, region interface{}, name interface{}) *AddressManager_GetAddress_Call { + return &AddressManager_GetAddress_Call{Call: _e.mock.On("GetAddress", project, region, name)} +} + +func (_c *AddressManager_GetAddress_Call) Run(run func(project string, region string, name string)) *AddressManager_GetAddress_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string), args[1].(string), args[2].(string)) + }) + return _c +} + +func (_c *AddressManager_GetAddress_Call) Return(_a0 *compute.Address, _a1 error) *AddressManager_GetAddress_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *AddressManager_GetAddress_Call) RunAndReturn(run func(string, string, string) (*compute.Address, error)) *AddressManager_GetAddress_Call { + _c.Call.Return(run) + return _c +} + // NewAddressManager creates a new instance of AddressManager. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewAddressManager(t interface { From f1ac54c27eb69f10cd88144e942ac8e38d6a77fb Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Tue, 17 Oct 2023 13:25:50 +0300 Subject: [PATCH 50/66] check if address already assigned --- internal/address/gcp.go | 26 ++++++++++++++++++++++++-- internal/address/gcp_test.go | 1 + 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/internal/address/gcp.go b/internal/address/gcp.go index 9f20360..2ab3c67 100644 --- a/internal/address/gcp.go +++ b/internal/address/gcp.go @@ -208,7 +208,15 @@ func (a *gcpAssigner) addInstanceAddress(ctx context.Context, instance *compute. return nil } -func (a *gcpAssigner) Assign(ctx context.Context, instanceID, zone string, filter []string, orderBy string) error { +func (a *gcpAssigner) forceCheckAddressAssigned(region, addressName string) (bool, error) { + address, err := a.addressManager.GetAddress(a.project, region, addressName) + if err != nil { + return false, errors.Wrapf(err, "failed to get address %s", addressName) + } + return address.Status == inUseStatus, nil +} + +func (a *gcpAssigner) Assign(ctx context.Context, instanceID, zone string, filter []string, orderBy string) error { //nonlint:gocyclo // check if instance already has a public static IP address assigned instance, err := a.instanceGetter.Get(a.project, zone, instanceID) if err != nil { @@ -255,12 +263,26 @@ func (a *gcpAssigner) Assign(ctx context.Context, instanceID, zone string, filte // try to assign all available addresses until one succeeds // due to concurrency, it is possible that another kubeip instance will assign the same address for _, address := range addresses { + // force check if address is already assigned (reduce the chance of assigning the same address by multiple kubeip instances) + var addressAssigned bool + addressAssigned, err = a.forceCheckAddressAssigned(a.region, address.Name) + if err != nil { + a.logger.WithError(err).Errorf("failed to check if address %s is assigned", address.Address) + a.logger.Debug("trying next address") + continue + } + if addressAssigned { + a.logger.WithField("address", address.Address).Debug("address is already assigned") + a.logger.Debug("trying next address") + continue + } + // assign address to the instance and try the next address if it fails if err = a.addInstanceAddress(ctx, instance, zone, address); err != nil { a.logger.WithError(err).Errorf("failed to assign static public IP address %s", address.Address) a.logger.Debug("trying next address") continue } - // break the loop after assigning the address + // break the loop after successfully assigning an address break } if err != nil { diff --git a/internal/address/gcp_test.go b/internal/address/gcp_test.go index 633709c..761eda8 100644 --- a/internal/address/gcp_test.go +++ b/internal/address/gcp_test.go @@ -391,6 +391,7 @@ func Test_gcpAssigner_Assign(t *testing.T) { Kind: accessConfigKind, NatIP: "100.0.0.3", }).Return(&compute.Operation{Name: "test-operation", Status: "DONE"}, nil) + mock.EXPECT().GetAddress("test-project", "test-region", "test-address-3").Return(&compute.Address{Name: "test-address-3", Status: reservedStatus}, nil) return mock }, }, From 65c1ff313a42579a589ac2f339591271ec00e5c0 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Tue, 17 Oct 2023 14:01:31 +0300 Subject: [PATCH 51/66] retry assign ephemeral ip on shutdown --- internal/address/gcp.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/internal/address/gcp.go b/internal/address/gcp.go index 2ab3c67..010b8c9 100644 --- a/internal/address/gcp.go +++ b/internal/address/gcp.go @@ -345,11 +345,15 @@ func (a *gcpAssigner) Unassign(ctx context.Context, instanceID, zone string) err return errors.Wrap(err, "failed to delete current public IP address") } // assign ephemeral public IP address to the instance (pass nil address) - if err = a.addInstanceAddress(ctx, instance, zone, nil); err != nil { - return errors.Wrap(err, "failed to assign ephemeral public IP address") + for { + // retry until ephemeral public IP address is assigned + // sometime this operation fails and needs to be retried + if err = a.addInstanceAddress(ctx, instance, zone, nil); err != nil { + a.logger.WithError(err).Error("failed to assign ephemeral public IP address, retrying") + continue + } + return nil } - // break the loop after deleting the address - return nil } } } From 715d0c5b0ef47720f368658306184873b571ddc4 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Wed, 18 Oct 2023 15:55:04 +0300 Subject: [PATCH 52/66] TF for AWS working draft --- examples/aws/eks.tf | 57 ++++++++++++++++++++------------------- examples/aws/variables.tf | 20 ++++++++++++++ 2 files changed, 49 insertions(+), 28 deletions(-) diff --git a/examples/aws/eks.tf b/examples/aws/eks.tf index ff83515..68feed9 100644 --- a/examples/aws/eks.tf +++ b/examples/aws/eks.tf @@ -5,19 +5,28 @@ provider "aws" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - name = var.vpc_name - cidr = "10.0.0.0/16" - azs = ["us-west-2a", "us-west-2b", "us-west-2c"] - private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] - public_subnets = ["10.0.101.0/24", "10.0.102.0/24", "10.0.103.0/24"] - enable_nat_gateway = true - single_nat_gateway = true - enable_dns_hostnames = true + name = var.vpc_name + cidr = var.vpc_cidr + azs = var.availability_zones + private_subnets = var.private_cidr_ranges + public_subnets = var.public_cidr_ranges + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true + map_public_ip_on_launch = true tags = { App = "kubeip" Env = "demo" } + public_subnet_tags = { + public = "true" + environment = "demo" + } + private_subnet_tags = { + public = "false" + environment = "demo" + } } module "eks" { @@ -37,9 +46,8 @@ module "eks" { max_size = 5 min_size = 1 - instance_types = ["t4g.micro", "t4g.small"] + instance_types = ["t3a.small", "t3a.medium"] capacity_type = "SPOT" - platform = "bottlerocket" labels = { nodegroup = "public" @@ -47,8 +55,8 @@ module "eks" { } tags = { - Environment = "demo" Name = "public-node-group" + environment = "demo" public = "true" kubeip = "use" } @@ -61,9 +69,8 @@ module "eks" { max_size = 5 min_size = 1 - instance_types = ["t4g.micro", "t4g.small"] + instance_types = ["t3a.small", "t3a.medium"] capacity_type = "SPOT" - platform = "bottlerocket" labels = { nodegroup = "private" @@ -71,26 +78,18 @@ module "eks" { } tags = { - Environment = "demo" Name = "private-node-group" + environment = "demo" } subnet_ids = module.vpc.private_subnets } } - - # aws-auth configmap - manage_aws_auth_configmap = true - - tags = { - App = "kubeip" - Environment = "demo" - } } resource "aws_iam_policy" "kubeip-policy" { name = "kubeip-policy" - description = "KubeIP policy" + description = "KubeIP required permissions" policy = jsonencode({ Version = "2012-10-17" @@ -120,18 +119,20 @@ module "kubeip_eks_role" { oidc_providers = { main = { provider_arn = module.eks.oidc_provider_arn - namespace_service_accounts = ["kube-system:kubeip-sa"] + namespace_service_accounts = ["kube-system:kubeip-service-account"] } } } -# 5 elastic IPs in the same region +# 3 elastic IPs in the same region resource "aws_eip" "kubeip" { - count = 5 + // default EIP limit is 5 (make sure to increase it if you need more) + count = 3 tags = { - Name = "kubeip-${count.index}" - kubeip = "reserved" + Name = "kubeip-${count.index}" + environment = "demo" + kubeip = "reserved" } } diff --git a/examples/aws/variables.tf b/examples/aws/variables.tf index 9d49338..d8a590b 100644 --- a/examples/aws/variables.tf +++ b/examples/aws/variables.tf @@ -3,6 +3,11 @@ variable "region" { default = "us-west-2" } +variable "availability_zones" { + type = list(string) + default = ["us-west-2a", "us-west-2b", "us-west-2c"] +} + variable "cluster_name" { type = string default = "kubeip-demo" @@ -13,6 +18,21 @@ variable "vpc_name" { default = "kubeip-demo" } +variable "vpc_cidr" { + type = string + default = "10.0.0.0/16" +} + +variable "private_cidr_ranges" { + type = list(string) + default = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] +} + +variable "public_cidr_ranges" { + type = list(string) + default = ["10.0.101.0/24", "10.0.102.0/24", "10.0.103.0/24"] +} + variable "kubernetes_version" { type = string default = "1.28" From 88a25b898dbbc7e4fb1549a73542c72a4cdc2456 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Wed, 18 Oct 2023 17:39:54 +0300 Subject: [PATCH 53/66] working EKS example --- cmd/main.go | 3 +- examples/aws/eks.tf | 4 +- internal/address/aws.go | 76 ++++++++++++++++++++++++++++++----- internal/address/aws_test.go | 20 ++++++++- internal/cloud/aws_address.go | 5 +-- mocks/cloud/EipAssigner.go | 23 +++++------ 6 files changed, 102 insertions(+), 29 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 8e40689..ee94de0 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -159,7 +159,8 @@ func run(c context.Context, log *logrus.Entry, cfg *config.Config) error { log.Infof("kubeip agent stopped") if cfg.ReleaseOnExit { log.Infof("releasing static public IP address") - if err = assigner.Unassign(ctx, n.Instance, n.Zone); err != nil { + // use a different context for releasing the static public IP address since the main context is canceled + if err = assigner.Unassign(context.Background(), n.Instance, n.Zone); err != nil { return errors.Wrap(err, "releasing static public IP address") } } diff --git a/examples/aws/eks.tf b/examples/aws/eks.tf index 68feed9..9e6aef2 100644 --- a/examples/aws/eks.tf +++ b/examples/aws/eks.tf @@ -42,7 +42,7 @@ module "eks" { eks_managed_node_groups = { eks_nodes_public = { - desired_size = 1 + desired_size = 3 max_size = 5 min_size = 1 @@ -127,7 +127,7 @@ module "kubeip_eks_role" { # 3 elastic IPs in the same region resource "aws_eip" "kubeip" { // default EIP limit is 5 (make sure to increase it if you need more) - count = 3 + count = 5 tags = { Name = "kubeip-${count.index}" diff --git a/internal/address/aws.go b/internal/address/aws.go index 0f7f2ae..81cbe25 100644 --- a/internal/address/aws.go +++ b/internal/address/aws.go @@ -147,6 +147,24 @@ func sortAddressesByField(addresses []types.Address, sortBy string) { } } +func (a *awsAssigner) forceCheckAddressAssigned(ctx context.Context, allocationID string) (bool, error) { + // get elastic IP attached to the allocation ID + filters := make(map[string][]string) + filters["allocation-id"] = []string{allocationID} + addresses, err := a.eipLister.List(ctx, filters, true) + if err != nil { + return false, errors.Wrapf(err, "failed to list elastic IPs by allocation-id %s", allocationID) + } + if len(addresses) == 0 { + return false, nil + } + // check if the first address (and the only) is assigned + if addresses[0].AssociationId != nil { + return true, nil + } + return false, nil +} + func (a *awsAssigner) Assign(ctx context.Context, instanceID, _ string, filter []string, orderBy string) error { // get elastic IP attached to the instance filters := make(map[string][]string) @@ -179,6 +197,13 @@ func (a *awsAssigner) Assign(ctx context.Context, instanceID, _ string, filter [ return errors.Errorf("no available elastic IPs") } + // log available addresses IPs + ips := make([]string, 0, len(addresses)) + for _, address := range addresses { + ips = append(ips, *address.PublicIp) + } + a.logger.WithField("addresses", ips).Debugf("found %d available addresses", len(addresses)) + // get EC2 instance instance, err := a.instanceGetter.Get(ctx, instanceID, a.region) if err != nil { @@ -202,17 +227,48 @@ func (a *awsAssigner) Assign(ctx context.Context, instanceID, _ string, filter [ // sort addresses by orderBy field sortAddressesByField(addresses, orderBy) - // assign the first available elastic IP to the instance - address := addresses[0] - if err = a.eipAssigner.Assign(ctx, instanceID, networkInterfaceID, &address); err != nil { - return errors.Wrap(err, "failed to assign elastic IP") + // try to assign all available addresses until one succeeds + // due to concurrency, it is possible that another kubeip instance will assign the same address + for i := range addresses { + // force check if address is already assigned (reduce the chance of assigning the same address by multiple kubeip instances) + var addressAssigned bool + addressAssigned, err = a.forceCheckAddressAssigned(ctx, *addresses[i].AllocationId) + if err != nil { + a.logger.WithError(err).Errorf("failed to check if address %s is assigned", *addresses[i].PublicIp) + a.logger.Debug("trying next address") + continue + } + if addressAssigned { + a.logger.WithField("address", addresses[i].PublicIp).Debug("address is already assigned") + a.logger.Debug("trying next address") + continue + } + a.logger.WithFields(logrus.Fields{ + "instance": instanceID, + "address": *addresses[i].PublicIp, + "allocation_id": *addresses[i].AllocationId, + "networkInterfaceID": networkInterfaceID, + }).Debug("assigning elastic IP to the instance") + if err = a.eipAssigner.Assign(ctx, networkInterfaceID, &addresses[i]); err != nil { + a.logger.WithFields(logrus.Fields{ + "instance": instanceID, + "address": *addresses[i].PublicIp, + "allocation_id": *addresses[i].AllocationId, + "networkInterfaceID": networkInterfaceID, + }).Debug("failed to assign elastic IP to the instance") + a.logger.Debug("trying next address") + continue + } + a.logger.WithFields(logrus.Fields{ + "instance": instanceID, + "address": *addresses[i].PublicIp, + "allocation_id": *addresses[i].AllocationId, + }).Info("elastic IP assigned to the instance") + break + } + if err != nil { + return errors.Wrap(err, "failed to assign elastic IP address") } - a.logger.WithFields(logrus.Fields{ - "instance": instanceID, - "address": *address.PublicIp, - "allocation_id": *address.AllocationId, - }).Info("elastic IP assigned to the instance") - return nil } diff --git a/internal/address/aws_test.go b/internal/address/aws_test.go index 036f3fb..39a79b2 100644 --- a/internal/address/aws_test.go +++ b/internal/address/aws_test.go @@ -499,11 +499,29 @@ func Test_awsAssigner_Assign(t *testing.T) { }, }, }, nil).Once() + mock.EXPECT().List(args.ctx, map[string][]string{ + "allocation-id": {"eipalloc-0abcd1234efgh5678"}, + }, true).Return([]types.Address{ + { + AllocationId: aws.String("eipalloc-0abcd1234efgh5678"), + PublicIp: aws.String("100.0.0.1"), + Tags: []types.Tag{ + { + Key: aws.String("env"), + Value: aws.String("test"), + }, + { + Key: aws.String("kubeip"), + Value: aws.String("reserved"), + }, + }, + }, + }, nil).Once() return mock }, eipAssignerFn: func(t *testing.T, args *args) cloud.EipAssigner { mock := mocks.NewEipAssigner(t) - mock.EXPECT().Assign(args.ctx, args.instanceID, "eni-0abcd1234efgh5678", tmock.Anything).Return(nil) + mock.EXPECT().Assign(args.ctx, "eni-0abcd1234efgh5678", tmock.Anything).Return(nil) return mock }, }, diff --git a/internal/cloud/aws_address.go b/internal/cloud/aws_address.go index e7300a1..795f97a 100644 --- a/internal/cloud/aws_address.go +++ b/internal/cloud/aws_address.go @@ -9,7 +9,7 @@ import ( ) type EipAssigner interface { - Assign(ctx context.Context, region, instanceID string, address *types.Address) error + Assign(ctx context.Context, networkInterfaceID string, address *types.Address) error Unassign(ctx context.Context, address *types.Address) error } @@ -21,11 +21,10 @@ func NewEipAssigner(client *ec2.Client) EipAssigner { return &eipAssigner{client: client} } -func (a *eipAssigner) Assign(ctx context.Context, instanceID, networkInterfaceID string, address *types.Address) error { +func (a *eipAssigner) Assign(ctx context.Context, networkInterfaceID string, address *types.Address) error { // associate elastic IP with the instance input := &ec2.AssociateAddressInput{ AllocationId: address.AllocationId, - InstanceId: &instanceID, NetworkInterfaceId: &networkInterfaceID, } diff --git a/mocks/cloud/EipAssigner.go b/mocks/cloud/EipAssigner.go index 802e028..5462058 100644 --- a/mocks/cloud/EipAssigner.go +++ b/mocks/cloud/EipAssigner.go @@ -22,13 +22,13 @@ func (_m *EipAssigner) EXPECT() *EipAssigner_Expecter { return &EipAssigner_Expecter{mock: &_m.Mock} } -// Assign provides a mock function with given fields: ctx, region, instanceID, address -func (_m *EipAssigner) Assign(ctx context.Context, region string, instanceID string, address *types.Address) error { - ret := _m.Called(ctx, region, instanceID, address) +// Assign provides a mock function with given fields: ctx, networkInterfaceID, address +func (_m *EipAssigner) Assign(ctx context.Context, networkInterfaceID string, address *types.Address) error { + ret := _m.Called(ctx, networkInterfaceID, address) var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string, string, *types.Address) error); ok { - r0 = rf(ctx, region, instanceID, address) + if rf, ok := ret.Get(0).(func(context.Context, string, *types.Address) error); ok { + r0 = rf(ctx, networkInterfaceID, address) } else { r0 = ret.Error(0) } @@ -43,16 +43,15 @@ type EipAssigner_Assign_Call struct { // Assign is a helper method to define mock.On call // - ctx context.Context -// - region string -// - instanceID string +// - networkInterfaceID string // - address *types.Address -func (_e *EipAssigner_Expecter) Assign(ctx interface{}, region interface{}, instanceID interface{}, address interface{}) *EipAssigner_Assign_Call { - return &EipAssigner_Assign_Call{Call: _e.mock.On("Assign", ctx, region, instanceID, address)} +func (_e *EipAssigner_Expecter) Assign(ctx interface{}, networkInterfaceID interface{}, address interface{}) *EipAssigner_Assign_Call { + return &EipAssigner_Assign_Call{Call: _e.mock.On("Assign", ctx, networkInterfaceID, address)} } -func (_c *EipAssigner_Assign_Call) Run(run func(ctx context.Context, region string, instanceID string, address *types.Address)) *EipAssigner_Assign_Call { +func (_c *EipAssigner_Assign_Call) Run(run func(ctx context.Context, networkInterfaceID string, address *types.Address)) *EipAssigner_Assign_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(string), args[2].(string), args[3].(*types.Address)) + run(args[0].(context.Context), args[1].(string), args[2].(*types.Address)) }) return _c } @@ -62,7 +61,7 @@ func (_c *EipAssigner_Assign_Call) Return(_a0 error) *EipAssigner_Assign_Call { return _c } -func (_c *EipAssigner_Assign_Call) RunAndReturn(run func(context.Context, string, string, *types.Address) error) *EipAssigner_Assign_Call { +func (_c *EipAssigner_Assign_Call) RunAndReturn(run func(context.Context, string, *types.Address) error) *EipAssigner_Assign_Call { _c.Call.Return(run) return _c } From 4e29f25780ba5186719e99eea870fbbde54fc96b Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Thu, 19 Oct 2023 10:43:24 +0300 Subject: [PATCH 54/66] attach/dettach eip to eth0 only --- internal/address/aws.go | 11 +++++---- internal/address/aws_test.go | 6 +++-- internal/cloud/aws_address.go | 13 +++++----- mocks/cloud/EipAssigner.go | 45 +++++++++++++++++------------------ 4 files changed, 39 insertions(+), 36 deletions(-) diff --git a/internal/address/aws.go b/internal/address/aws.go index 81cbe25..8f769b2 100644 --- a/internal/address/aws.go +++ b/internal/address/aws.go @@ -213,11 +213,13 @@ func (a *awsAssigner) Assign(ctx context.Context, instanceID, _ string, filter [ if instance.NetworkInterfaces == nil || len(instance.NetworkInterfaces) == 0 { return errors.Errorf("no network interfaces found for instance %s", instanceID) } - // get network interface ID of network interface with public IP address + // get primary network interface ID with public IP address (DeviceIndex == 0) networkInterfaceID := "" for _, ni := range instance.NetworkInterfaces { - if ni.Association != nil && ni.Association.PublicIp != nil { + if ni.Association != nil && ni.Association.PublicIp != nil && + ni.Attachment != nil && ni.Attachment.DeviceIndex != nil && *ni.Attachment.DeviceIndex == 0 { networkInterfaceID = *ni.NetworkInterfaceId + break } } if networkInterfaceID == "" { @@ -249,7 +251,7 @@ func (a *awsAssigner) Assign(ctx context.Context, instanceID, _ string, filter [ "allocation_id": *addresses[i].AllocationId, "networkInterfaceID": networkInterfaceID, }).Debug("assigning elastic IP to the instance") - if err = a.eipAssigner.Assign(ctx, networkInterfaceID, &addresses[i]); err != nil { + if err = a.eipAssigner.Assign(ctx, networkInterfaceID, *addresses[i].AllocationId); err != nil { a.logger.WithFields(logrus.Fields{ "instance": instanceID, "address": *addresses[i].PublicIp, @@ -287,13 +289,14 @@ func (a *awsAssigner) Unassign(ctx context.Context, instanceID, _ string) error // unassign elastic IP from the instance address := addresses[0] - if err = a.eipAssigner.Unassign(ctx, &address); err != nil { + if err = a.eipAssigner.Unassign(ctx, *address.AssociationId); err != nil { return errors.Wrap(err, "failed to unassign elastic IP") } a.logger.WithFields(logrus.Fields{ "instance": instanceID, "address": *address.PublicIp, "allocation_id": *address.AllocationId, + "associationId": *address.AssociationId, }).Info("elastic IP unassigned from the instance") return nil diff --git a/internal/address/aws_test.go b/internal/address/aws_test.go index 39a79b2..9aab050 100644 --- a/internal/address/aws_test.go +++ b/internal/address/aws_test.go @@ -10,7 +10,6 @@ import ( "github.com/doitintl/kubeip/internal/cloud" mocks "github.com/doitintl/kubeip/mocks/cloud" "github.com/sirupsen/logrus" - tmock "github.com/stretchr/testify/mock" ) func Test_sortAddressesByTag(t *testing.T) { @@ -455,6 +454,9 @@ func Test_awsAssigner_Assign(t *testing.T) { Association: &types.InstanceNetworkInterfaceAssociation{ PublicIp: aws.String("135.64.10.1"), }, + Attachment: &types.InstanceNetworkInterfaceAttachment{ + DeviceIndex: aws.Int32(0), + }, NetworkInterfaceId: aws.String("eni-0abcd1234efgh5678"), }, }, @@ -521,7 +523,7 @@ func Test_awsAssigner_Assign(t *testing.T) { }, eipAssignerFn: func(t *testing.T, args *args) cloud.EipAssigner { mock := mocks.NewEipAssigner(t) - mock.EXPECT().Assign(args.ctx, "eni-0abcd1234efgh5678", tmock.Anything).Return(nil) + mock.EXPECT().Assign(args.ctx, "eni-0abcd1234efgh5678", "eipalloc-0abcd1234efgh5678").Return(nil) return mock }, }, diff --git a/internal/cloud/aws_address.go b/internal/cloud/aws_address.go index 795f97a..0947edb 100644 --- a/internal/cloud/aws_address.go +++ b/internal/cloud/aws_address.go @@ -4,13 +4,12 @@ import ( "context" "github.com/aws/aws-sdk-go-v2/service/ec2" - "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/pkg/errors" ) type EipAssigner interface { - Assign(ctx context.Context, networkInterfaceID string, address *types.Address) error - Unassign(ctx context.Context, address *types.Address) error + Assign(ctx context.Context, networkInterfaceID, allocationID string) error + Unassign(ctx context.Context, associationID string) error } type eipAssigner struct { @@ -21,10 +20,10 @@ func NewEipAssigner(client *ec2.Client) EipAssigner { return &eipAssigner{client: client} } -func (a *eipAssigner) Assign(ctx context.Context, networkInterfaceID string, address *types.Address) error { +func (a *eipAssigner) Assign(ctx context.Context, networkInterfaceID, allocationID string) error { // associate elastic IP with the instance input := &ec2.AssociateAddressInput{ - AllocationId: address.AllocationId, + AllocationId: &allocationID, NetworkInterfaceId: &networkInterfaceID, } @@ -36,10 +35,10 @@ func (a *eipAssigner) Assign(ctx context.Context, networkInterfaceID string, add return nil } -func (a *eipAssigner) Unassign(ctx context.Context, address *types.Address) error { +func (a *eipAssigner) Unassign(ctx context.Context, associationID string) error { // disassociate elastic IP from the instance input := &ec2.DisassociateAddressInput{ - AssociationId: address.AssociationId, + AssociationId: &associationID, } _, err := a.client.DisassociateAddress(ctx, input) diff --git a/mocks/cloud/EipAssigner.go b/mocks/cloud/EipAssigner.go index 5462058..18cda75 100644 --- a/mocks/cloud/EipAssigner.go +++ b/mocks/cloud/EipAssigner.go @@ -5,7 +5,6 @@ package mocks import ( context "context" - types "github.com/aws/aws-sdk-go-v2/service/ec2/types" mock "github.com/stretchr/testify/mock" ) @@ -22,13 +21,13 @@ func (_m *EipAssigner) EXPECT() *EipAssigner_Expecter { return &EipAssigner_Expecter{mock: &_m.Mock} } -// Assign provides a mock function with given fields: ctx, networkInterfaceID, address -func (_m *EipAssigner) Assign(ctx context.Context, networkInterfaceID string, address *types.Address) error { - ret := _m.Called(ctx, networkInterfaceID, address) +// Assign provides a mock function with given fields: ctx, networkInterfaceID, allocationID +func (_m *EipAssigner) Assign(ctx context.Context, networkInterfaceID string, allocationID string) error { + ret := _m.Called(ctx, networkInterfaceID, allocationID) var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string, *types.Address) error); ok { - r0 = rf(ctx, networkInterfaceID, address) + if rf, ok := ret.Get(0).(func(context.Context, string, string) error); ok { + r0 = rf(ctx, networkInterfaceID, allocationID) } else { r0 = ret.Error(0) } @@ -44,14 +43,14 @@ type EipAssigner_Assign_Call struct { // Assign is a helper method to define mock.On call // - ctx context.Context // - networkInterfaceID string -// - address *types.Address -func (_e *EipAssigner_Expecter) Assign(ctx interface{}, networkInterfaceID interface{}, address interface{}) *EipAssigner_Assign_Call { - return &EipAssigner_Assign_Call{Call: _e.mock.On("Assign", ctx, networkInterfaceID, address)} +// - allocationID string +func (_e *EipAssigner_Expecter) Assign(ctx interface{}, networkInterfaceID interface{}, allocationID interface{}) *EipAssigner_Assign_Call { + return &EipAssigner_Assign_Call{Call: _e.mock.On("Assign", ctx, networkInterfaceID, allocationID)} } -func (_c *EipAssigner_Assign_Call) Run(run func(ctx context.Context, networkInterfaceID string, address *types.Address)) *EipAssigner_Assign_Call { +func (_c *EipAssigner_Assign_Call) Run(run func(ctx context.Context, networkInterfaceID string, allocationID string)) *EipAssigner_Assign_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(string), args[2].(*types.Address)) + run(args[0].(context.Context), args[1].(string), args[2].(string)) }) return _c } @@ -61,18 +60,18 @@ func (_c *EipAssigner_Assign_Call) Return(_a0 error) *EipAssigner_Assign_Call { return _c } -func (_c *EipAssigner_Assign_Call) RunAndReturn(run func(context.Context, string, *types.Address) error) *EipAssigner_Assign_Call { +func (_c *EipAssigner_Assign_Call) RunAndReturn(run func(context.Context, string, string) error) *EipAssigner_Assign_Call { _c.Call.Return(run) return _c } -// Unassign provides a mock function with given fields: ctx, address -func (_m *EipAssigner) Unassign(ctx context.Context, address *types.Address) error { - ret := _m.Called(ctx, address) +// Unassign provides a mock function with given fields: ctx, associationID +func (_m *EipAssigner) Unassign(ctx context.Context, associationID string) error { + ret := _m.Called(ctx, associationID) var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *types.Address) error); ok { - r0 = rf(ctx, address) + if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { + r0 = rf(ctx, associationID) } else { r0 = ret.Error(0) } @@ -87,14 +86,14 @@ type EipAssigner_Unassign_Call struct { // Unassign is a helper method to define mock.On call // - ctx context.Context -// - address *types.Address -func (_e *EipAssigner_Expecter) Unassign(ctx interface{}, address interface{}) *EipAssigner_Unassign_Call { - return &EipAssigner_Unassign_Call{Call: _e.mock.On("Unassign", ctx, address)} +// - associationID string +func (_e *EipAssigner_Expecter) Unassign(ctx interface{}, associationID interface{}) *EipAssigner_Unassign_Call { + return &EipAssigner_Unassign_Call{Call: _e.mock.On("Unassign", ctx, associationID)} } -func (_c *EipAssigner_Unassign_Call) Run(run func(ctx context.Context, address *types.Address)) *EipAssigner_Unassign_Call { +func (_c *EipAssigner_Unassign_Call) Run(run func(ctx context.Context, associationID string)) *EipAssigner_Unassign_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(*types.Address)) + run(args[0].(context.Context), args[1].(string)) }) return _c } @@ -104,7 +103,7 @@ func (_c *EipAssigner_Unassign_Call) Return(_a0 error) *EipAssigner_Unassign_Cal return _c } -func (_c *EipAssigner_Unassign_Call) RunAndReturn(run func(context.Context, *types.Address) error) *EipAssigner_Unassign_Call { +func (_c *EipAssigner_Unassign_Call) RunAndReturn(run func(context.Context, string) error) *EipAssigner_Unassign_Call { _c.Call.Return(run) return _c } From 1a18ce37e27e22b5cbccc644bda3759173931e47 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Thu, 19 Oct 2023 12:39:17 +0300 Subject: [PATCH 55/66] prevent reassosiation --- internal/address/aws.go | 1 + internal/cloud/aws_address.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/internal/address/aws.go b/internal/address/aws.go index 8f769b2..fc71522 100644 --- a/internal/address/aws.go +++ b/internal/address/aws.go @@ -165,6 +165,7 @@ func (a *awsAssigner) forceCheckAddressAssigned(ctx context.Context, allocationI return false, nil } +//nolint:funlen func (a *awsAssigner) Assign(ctx context.Context, instanceID, _ string, filter []string, orderBy string) error { // get elastic IP attached to the instance filters := make(map[string][]string) diff --git a/internal/cloud/aws_address.go b/internal/cloud/aws_address.go index 0947edb..de0b2f2 100644 --- a/internal/cloud/aws_address.go +++ b/internal/cloud/aws_address.go @@ -3,6 +3,7 @@ package cloud import ( "context" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/pkg/errors" ) @@ -25,6 +26,7 @@ func (a *eipAssigner) Assign(ctx context.Context, networkInterfaceID, allocation input := &ec2.AssociateAddressInput{ AllocationId: &allocationID, NetworkInterfaceId: &networkInterfaceID, + AllowReassociation: aws.Bool(false), // do not allow reassociation of the elastic IP } _, err := a.client.AssociateAddress(ctx, input) From e91ddd5d411fe3927eba5c7b3eeeb9e302cd13fb Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Thu, 19 Oct 2023 13:06:28 +0300 Subject: [PATCH 56/66] add resources to daemonset and set priority class name --- examples/aws/eks.tf | 6 ++++++ examples/gcp/gke.tf | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/examples/aws/eks.tf b/examples/aws/eks.tf index 9e6aef2..a784f7d 100644 --- a/examples/aws/eks.tf +++ b/examples/aws/eks.tf @@ -219,6 +219,7 @@ resource "kubernetes_daemonset" "kubeip_daemonset" { spec { service_account_name = "kubeip-service-account" termination_grace_period_seconds = 30 + priority_class_name = "system-node-critical" container { name = "kubeip-agent" image = "doitintl/kubeip-agent" @@ -238,6 +239,11 @@ resource "kubernetes_daemonset" "kubeip_daemonset" { name = "LOG_LEVEL" value = "debug" } + resources { + requests = { + cpu = "100m" + } + } } node_selector = { nodegroup = "public" diff --git a/examples/gcp/gke.tf b/examples/gcp/gke.tf index 8b138c5..1cd1d4f 100644 --- a/examples/gcp/gke.tf +++ b/examples/gcp/gke.tf @@ -270,6 +270,7 @@ resource "kubernetes_daemonset" "kubeip_daemonset" { spec { service_account_name = "kubeip-service-account" termination_grace_period_seconds = 30 + priority_class_name = "system-node-critical" container { name = "kubeip-agent" image = "doitintl/kubeip-agent" @@ -289,6 +290,11 @@ resource "kubernetes_daemonset" "kubeip_daemonset" { name = "LOG_LEVEL" value = "debug" } + resources { + requests = { + cpu = "100m" + } + } } node_selector = { nodegroup = "public" From fe49c45c24fe6d54fb693befa333980450be4975 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Thu, 19 Oct 2023 16:57:04 +0300 Subject: [PATCH 57/66] update readme with examples --- README.md | 118 ++++++++++++++++++++++++++++---------------- examples/gcp/gke.tf | 4 ++ 2 files changed, 79 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 711303d..e88abaf 100644 --- a/README.md +++ b/README.md @@ -57,26 +57,11 @@ roleRef: apiGroup: rbac.authorization.k8s.io ``` -### AWS - -Make sure that KubeIP DaemonSet is deployed on nodes that have a public IP (node in public subnet) and uses Kubernetes service -account [bound](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) -to IAM role with the following permissions: +### Kubernetes DaemonSet -```yaml -Version: '2012-10-17' -Statement: - - Effect: Allow - Action: - - ec2:AssociateAddress - - ec2:DisassociateAddress - - ec2:DescribeInstances - - ec2:DescribeAddresses - Resource: '*' -``` - -KubeIP supports filtering of reserved Elastic IPs using tags. To use this feature, add the `filter` flag (or set `FILTER` environment -variable) to the KubeIP DaemonSet: +Deploy KubeIP as a DaemonSet on your desired nodes using standard Kubernetes selectors. Once deployed, KubeIP will assign a static public IP +to the node's primary network interface, selected from a list of reserved static IPs using platform-supported filtering. If no static public +IP is available, KubeIP will wait until one becomes available. When a node is deleted, KubeIP will release the static public IP. ```yaml apiVersion: apps/v1 @@ -92,15 +77,54 @@ spec: labels: app: kubeip spec: - serviceAccountName: kubeip-sa + serviceAccountName: kubeip-service-account + terminationGracePeriodSeconds: 30 + priorityClassName: system-node-critical nodeSelector: kubeip.com/public: "true" containers: - name: kubeip image: doitintl/kubeip-agent + resources: + requests: + cpu: 100m env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName - name: FILTER - value: "Name=tag:env,Values=dev;Name=tag:app,Values=streamer" + value: PUT_PLATFORM_SPECIFIC_FILTER_HERE + - name: LOG_LEVEL + value: debug + - name: LOG_JSON + value: "true" +``` + +### AWS + +Make sure that KubeIP DaemonSet is deployed on nodes that have a public IP (node running in public subnet) and uses a Kubernetes service +account [bound](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) to the IAM role with the following +permissions: + +```yaml +Version: '2012-10-17' +Statement: + - Effect: Allow + Action: + - ec2:AssociateAddress + - ec2:DisassociateAddress + - ec2:DescribeInstances + - ec2:DescribeAddresses + Resource: '*' +``` + +KubeIP supports filtering of reserved Elastic IPs using tags and Elastic IP properties. To use this feature, add the `filter` flag (or +set `FILTER` environment variable) to the KubeIP DaemonSet: + +```yaml +- name: FILTER + value: "Name=tag:env,Values=dev;Name=tag:app,Values=streamer" ``` KubeIP AWS filter supports the same filter syntax as the AWS `describe-addresses` command. For more information, @@ -137,28 +161,8 @@ semicolons (`;`). To use this feature, add the `filter` flag (or set `FILTER` environment variable) to the KubeIP DaemonSet: ```yaml -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kubeip -spec: - selector: - matchLabels: - app: kubeip - template: - metadata: - labels: - app: kubeip - spec: - serviceAccountName: kubeip-sa - nodeSelector: - kubeip.com/public: "true" - containers: - - name: kubeip - image: doitintl/kubeip-agent - env: - - name: FILTER - value: "labels.env=dev;labels.app=streamer" +- name: FILTER + value: "labels.env=dev;labels.app=streamer" ``` ## How to contribute to KubeIP? @@ -221,3 +225,31 @@ public subnet). Configure KubeIP to use the pool of reserved static public IPs, Finally, scale the number of nodes in the cluster and verify that KubeIP assigns a static public IP to each node. Scale down the number of nodes in the cluster and verify that KubeIP releases the static public IP addresses. + +#### AWS EKS Example + +The [examples/aws](examples/aws) folder contains a Terraform configuration that creates an EKS cluster and deploys KubeIP as a DaemonSet on +the cluster nodes in a public subnet. The Terraform configuration also creates a pool of reserved Elastic IPs and configures KubeIP to use +the pool of reserved static public IPs. + +To run the example, follow these steps: + +```shell +cd examples/aws +terraform init +terraform apply +``` + +#### Google Cloud GKE Example + +The [examples/gcp](examples/gcp) folder contains a Terraform configuration that creates a GKE cluster and deploys KubeIP as a DaemonSet on +the cluster nodes in a public subnet. The Terraform configuration also creates a pool of reserved static public IPs and configures KubeIP to +use the pool of reserved static public IPs. + +To run the example, follow these steps: + +```shell +cd examples/gcp +terraform init +terraform apply -var="project_id=" +``` diff --git a/examples/gcp/gke.tf b/examples/gcp/gke.tf index 1cd1d4f..cee99e4 100644 --- a/examples/gcp/gke.tf +++ b/examples/gcp/gke.tf @@ -290,6 +290,10 @@ resource "kubernetes_daemonset" "kubeip_daemonset" { name = "LOG_LEVEL" value = "debug" } + evn { + name = "LOG_JSON" + value = "true" + } resources { requests = { cpu = "100m" From 6df52bb5bbe314dcd427efac33b3ea14d17e1659 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Thu, 19 Oct 2023 17:03:00 +0300 Subject: [PATCH 58/66] update readme with v1 --- README.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e88abaf..c52acf4 100644 --- a/README.md +++ b/README.md @@ -3,17 +3,24 @@ # KubeIP v2 Welcome to KubeIP v2, a complete overhaul of the popular [DoiT](https://www.doit.com/) -KubeIP [v1](https://github.com/doitintl/kubeip/tree/v1-main) open-source project, originally developed +KubeIP [v1-main](https://github.com/doitintl/kubeip/tree/v1-main) open-source project, originally developed by [Aviv Laufer](https://github.com/avivl). KubeIP v2 expands its support beyond Google Cloud (as in v1) to include AWS, and it's designed to be extendable to other cloud providers that allow assigning static public IP to VMs. We've also transitioned from a Kubernetes controller to a standard DaemonSet, enhancing reliability and ease of use. -## What KubeIP does? +## What happens with KubeIP v1 + +KubeIP v1 is still available in the [v1-main](https://github.com/doitintl/kubeip/tree/v1-main) branch. No further development is planned. We +will fix critical bugs and security issues, but we will not add new features. + +## What KubeIP v2 does? KubeIP is a tool that assigns a static public IP to any node it operates on. The IP is assigned to the node's primary network interface, -selected from a list of reserved static IPs using platform-supported filtering. +selected from a list of reserved static IPs using platform-supported filtering and ordering. If no static public IP is available, KubeIP +will wait until one becomes available. When a node is deleted, KubeIP will release the static public IP back to the pool of reserved static +IPs. ## How to use KubeIP? From 1ee40cd8499ae6377da90166d2a8555314978346 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Thu, 19 Oct 2023 17:14:33 +0300 Subject: [PATCH 59/66] enhance readme --- README.md | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c52acf4..498cd39 100644 --- a/README.md +++ b/README.md @@ -17,14 +17,27 @@ will fix critical bugs and security issues, but we will not add new features. ## What KubeIP v2 does? -KubeIP is a tool that assigns a static public IP to any node it operates on. The IP is assigned to the node's primary network interface, -selected from a list of reserved static IPs using platform-supported filtering and ordering. If no static public IP is available, KubeIP -will wait until one becomes available. When a node is deleted, KubeIP will release the static public IP back to the pool of reserved static -IPs. +Kubernetes' nodes don't necessarily need their own public IP addresses to communicate. However, there are certain situations where it's +beneficial for nodes in a node pool to have their own unique public IP addresses. + +For instance, in gaming applications, a console might need to establish a direct connection with a cloud virtual machine to reduce the +number of hops. + +Similarly, if you have multiple agents running on Kubernetes that need a direct server connection, and the server needs to whitelist all +agent IPs, having dedicated public IPs can be useful. These scenarios, among others, can be handled on a cloud-managed Kubernetes cluster +using Node Public IP. + +KubeIP is a utility that assigns a static public IP to each node it manages. The IP is allocated to the node's primary network interface, +chosen from a pool of reserved static IPs using platform-supported filtering and ordering. + +If there are no static public IPs left, KubeIP will hold on until one becomes available. When a node is removed, KubeIP releases the static +public IP back into the pool of reserved static IPs. ## How to use KubeIP? -Deploy KubeIP as a DaemonSet on your desired nodes using standard Kubernetes selectors. Once deployed, KubeIP will assign a static public IP +Deploy KubeIP as a DaemonSet on your desired nodes using standard +Kubernetes [mechanism](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/). Once deployed, KubeIP will assign a static +public IP to each node it operates on. If no static public IP is available, KubeIP will wait until one becomes available. When a node is deleted, KubeIP will release the static public IP. From de8eccecaafe5b5dada801b418bab3f1c298a1a8 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Sun, 22 Oct 2023 10:21:39 +0300 Subject: [PATCH 60/66] spuppress complexity linter --- internal/address/aws.go | 2 +- internal/address/gcp.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/address/aws.go b/internal/address/aws.go index fc71522..6a33b29 100644 --- a/internal/address/aws.go +++ b/internal/address/aws.go @@ -165,7 +165,7 @@ func (a *awsAssigner) forceCheckAddressAssigned(ctx context.Context, allocationI return false, nil } -//nolint:funlen +//nolint:funlen,gocognit,gocyclo func (a *awsAssigner) Assign(ctx context.Context, instanceID, _ string, filter []string, orderBy string) error { // get elastic IP attached to the instance filters := make(map[string][]string) diff --git a/internal/address/gcp.go b/internal/address/gcp.go index 010b8c9..b8443a8 100644 --- a/internal/address/gcp.go +++ b/internal/address/gcp.go @@ -44,7 +44,7 @@ func newOperationError(name string, err *compute.OperationError) *operationError } func isOperationError(err error) bool { - _, ok := err.(*operationError) + _, ok := err.(*operationError) //nolint:errorlint return ok } @@ -216,7 +216,8 @@ func (a *gcpAssigner) forceCheckAddressAssigned(region, addressName string) (boo return address.Status == inUseStatus, nil } -func (a *gcpAssigner) Assign(ctx context.Context, instanceID, zone string, filter []string, orderBy string) error { //nonlint:gocyclo +//nolint:gocyclo +func (a *gcpAssigner) Assign(ctx context.Context, instanceID, zone string, filter []string, orderBy string) error { // check if instance already has a public static IP address assigned instance, err := a.instanceGetter.Get(a.project, zone, instanceID) if err != nil { From e06ccfe518b9990da8df8877e7a245edc8980aab Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Sun, 22 Oct 2023 10:26:44 +0300 Subject: [PATCH 61/66] sonarcloud config added --- sonar-project.properties | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 sonar-project.properties diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 0000000..895ac8d --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,20 @@ +sonar.projectKey=doitintl_kubeip +sonar.organization=doitintl +# This is the name and version displayed in the SonarCloud UI. +sonar.projectName=kubeip +#sonar.projectVersion=1.0 +# Path is relative to the sonar-project.properties file. Replace "\" by "/" on Windows. +# This is where the source code is located. +sonar.sources=. +sonar.exclusions=**/*_test.go,mocks/*,**/mock_*.go +# This is where the tests are located. +sonar.tests=. +sonar.test.inclusions=**/*_test.go +# Test coverage report +sonar.go.coverage.reportPaths=coverage.out +# Test report json file +sonar.go.tests.reportPaths=test-report.out +# golangci-lint report json file +sonar.go.golangci-lint.reportPaths=golangci-lint.out +# Encoding of the source code. Default is default system encoding +sonar.sourceEncoding=UTF-8 \ No newline at end of file From 17e59ddc3f87fb74ca6e667f196deb50a90da663 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Sun, 22 Oct 2023 10:58:35 +0300 Subject: [PATCH 62/66] lint output --- makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/makefile b/makefile index 8a3fb5c..5ea2b88 100644 --- a/makefile +++ b/makefile @@ -54,6 +54,7 @@ lint: setup-lint; $(info $(M) running golangci-lint ...) @ ## run golangci-lint # updating path since golangci-lint is looking for go binary and this may lead to # conflict when multiple go versions are installed $Q $(GOLINT) run -v -c $(LINT_CONFIG) --out-format checkstyle ./... > golangci-lint.out + $Q cat golangci-lint.out mock: setup-mockery ; $(info $(M) running mockery ...) @ ## run mockery to generate mocks $Q $(GOMOCK) --dir internal --all --keeptree --with-expecter From 8587eaad307e8040acac2f79394987b19f93bce9 Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Sun, 22 Oct 2023 11:06:59 +0300 Subject: [PATCH 63/66] keep test artifacts --- .github/workflows/build.yaml | 12 ++++++++++++ makefile | 1 - 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9451032..501587d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -37,6 +37,18 @@ jobs: make lint make test-json + - name: upload test results + uses: actions/upload-artifact@v3 + if: ${{ always() }} + with: + name: test-reports + if-no-files-found: ignore + path: | + test-report.out + coverage.out + golangci-lint.out + + docker-build: runs-on: ubuntu-latest diff --git a/makefile b/makefile index 5ea2b88..14add89 100644 --- a/makefile +++ b/makefile @@ -66,7 +66,6 @@ test: ; $(info $(M) running test ...) @ ## run tests with coverage test-json: ; $(info $(M) running test output JSON ...) @ ## run tests with JSON report and coverage $Q $(GOTEST) -v -cover ./... -coverprofile=coverage.out -json > test-report.out - $Q $(GOTOOL) cover -func=coverage.out precommit: lint test ; $(info $(M) test and lint ...) @ ## release cycle: test > lint From 3bb79ade02b94fc7c6ff1d3688469a51e85abb7d Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Sun, 22 Oct 2023 11:16:40 +0300 Subject: [PATCH 64/66] configure sonarcloud action --- .github/workflows/build.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 501587d..20c5a1a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -47,6 +47,13 @@ jobs: test-report.out coverage.out golangci-lint.out + + - name: SonarCloud scan + uses: SonarSource/sonarcloud-github-action@master + if: ${{ always() }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} docker-build: From b96bb7d12ee04566c28edd634a2438c0efbf7d5e Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Sun, 22 Oct 2023 11:18:56 +0300 Subject: [PATCH 65/66] fix linter error --- internal/address/aws.go | 2 +- internal/cloud/aws_instance.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/address/aws.go b/internal/address/aws.go index 6a33b29..fa82012 100644 --- a/internal/address/aws.go +++ b/internal/address/aws.go @@ -165,7 +165,7 @@ func (a *awsAssigner) forceCheckAddressAssigned(ctx context.Context, allocationI return false, nil } -//nolint:funlen,gocognit,gocyclo +//nolint:funlen,gocyclo func (a *awsAssigner) Assign(ctx context.Context, instanceID, _ string, filter []string, orderBy string) error { // get elastic IP attached to the instance filters := make(map[string][]string) diff --git a/internal/cloud/aws_instance.go b/internal/cloud/aws_instance.go index f665ab2..4f90cb2 100644 --- a/internal/cloud/aws_instance.go +++ b/internal/cloud/aws_instance.go @@ -20,7 +20,7 @@ func NewEc2InstanceGetter(client *ec2.Client) Ec2InstanceGetter { return &ec2InstanceGetter{client: client} } -func (g *ec2InstanceGetter) Get(ctx context.Context, instanceID, region string) (*types.Instance, error) { +func (g *ec2InstanceGetter) Get(ctx context.Context, instanceID, _ string) (*types.Instance, error) { input := &ec2.DescribeInstancesInput{ InstanceIds: []string{ instanceID, From 714375ba349e5431061aead9b5504a0105b7afea Mon Sep 17 00:00:00 2001 From: Alexei Ledenev Date: Sun, 22 Oct 2023 11:40:21 +0300 Subject: [PATCH 66/66] configure sonarcloud action --- .gitignore | 3 +++ sonar-project.properties | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 665715a..4961c4a 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,9 @@ # Local .terraform directories **/.terraform/* +# Sonar Scanner +.scannerwork/ + # .tfstate files *.tfstate *.tfstate.* diff --git a/sonar-project.properties b/sonar-project.properties index 895ac8d..3df65b0 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,12 +1,13 @@ sonar.projectKey=doitintl_kubeip sonar.organization=doitintl +sonar.host.url=https://sonarcloud.io # This is the name and version displayed in the SonarCloud UI. sonar.projectName=kubeip #sonar.projectVersion=1.0 # Path is relative to the sonar-project.properties file. Replace "\" by "/" on Windows. # This is where the source code is located. sonar.sources=. -sonar.exclusions=**/*_test.go,mocks/*,**/mock_*.go +sonar.exclusions=**/*_test.go,mocks/**/*.go,**/mock_*.go # This is where the tests are located. sonar.tests=. sonar.test.inclusions=**/*_test.go