feat(build): use mise to manage third party tools (#5751)

* feat(build): use mise to manage third party tools * feat(ci): add renovate to .tools_versions.yaml
Kong · Mar 28, 2024 · 713765d · 713765d
1 parent d651762
commit 713765d
Show file tree

Hide file tree

Showing 28 changed files with 162 additions and 3,605 deletions.
diff --git a/.github/workflows/_conformance_tests.yaml b/.github/workflows/_conformance_tests.yaml
@@ -48,6 +48,10 @@ jobs:
         with:
           go-version-file: go.mod
 
+      - uses: jdx/mise-action@v2
+        with:
+          install: false
+
       - name: run conformance tests
         run: make test.conformance
         env:

diff --git a/.github/workflows/_e2e_tests.yaml b/.github/workflows/_e2e_tests.yaml
@@ -159,6 +159,10 @@ jobs:
         if: ${{ inputs.kong-image != '' && inputs.load-local-image }}
         run: docker pull ${{ inputs.kong-image }}
 
+      - uses: jdx/mise-action@v2
+        with:
+          install: false
+
       - name: run ${{ matrix.test }}
         run: make test.e2e
         env:
@@ -247,6 +251,10 @@ jobs:
             echo "kic-tag=nightly" >> $GITHUB_OUTPUT
           fi
 
+      - uses: jdx/mise-action@v2
+        with:
+          install: false
+
       - name: run ${{ matrix.test }}
         run: make test.e2e
         env:
@@ -339,6 +347,9 @@ jobs:
             echo "kic-tag=$kic_tag" >> $GITHUB_OUTPUT
           fi
 
+      - uses: jdx/mise-action@v2
+        with:
+          install: false
 
       - name: run Istio tests
         run: make test.istio

diff --git a/.github/workflows/_envtest_tests.yaml b/.github/workflows/_envtest_tests.yaml
@@ -16,6 +16,10 @@ jobs:
         with:
           go-version-file: go.mod
 
+      - uses: jdx/mise-action@v2
+        with:
+          install: false
+
       - name: run envtest tests
         run: make test.envtest.pretty
         env:

diff --git a/.github/workflows/_integration_tests.yaml b/.github/workflows/_integration_tests.yaml
@@ -171,6 +171,10 @@ jobs:
         with:
           go-version-file: go.mod
 
+      - uses: jdx/mise-action@v2
+        with:
+          install: false
+
       - name: run ${{ matrix.name }}
         run: make test.integration.${{ matrix.test }}
         env:

diff --git a/.github/workflows/_kongintegration_tests.yaml b/.github/workflows/_kongintegration_tests.yaml
@@ -21,6 +21,10 @@ jobs:
           echo "TEST_KONG_IMAGE=kong" >> $GITHUB_ENV
           echo "TEST_KONG_TAG=$(yq -ojson -r '.kongintegration.kong-oss' < .github/test_dependencies.yaml )" >> $GITHUB_ENV
 
+      - uses: jdx/mise-action@v2
+        with:
+          install: false
+
       - name: run kong integration tests
         run: make test.kongintegration
         env:

diff --git a/.github/workflows/_linters.yaml b/.github/workflows/_linters.yaml
@@ -16,6 +16,10 @@ jobs:
         with:
           go-version-file: third_party/go.mod
 
+      - uses: jdx/mise-action@v2
+        with:
+          install: false
+
       - name: Run lint
         env:
           # Our .golangci.yaml has fix: true, but we don't want that in CI therefore the below override.

diff --git a/.github/workflows/_performance_tests.yaml b/.github/workflows/_performance_tests.yaml
@@ -78,6 +78,9 @@ jobs:
             echo "kic-tag=$kic_tag" >> $GITHUB_OUTPUT
           fi
 
+      - uses: jdx/mise-action@v2
+        with:
+          install: false
 
       - name: run performance tests
         run: make test.performance

diff --git a/.github/workflows/_unit_tests.yaml b/.github/workflows/_unit_tests.yaml
@@ -16,6 +16,10 @@ jobs:
         with:
           go-version-file: go.mod
 
+      - uses: jdx/mise-action@v2
+        with:
+          install: false
+
       - name: run unit tests
         run: make test.unit
         env:

diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml
@@ -21,6 +21,10 @@ jobs:
       with:
         go-version-file: go.mod
 
+    - uses: jdx/mise-action@v2
+      with:
+        install: false
+
     - name: Run benchmark
       run: make bench | tee bench.out
 

diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml
@@ -69,6 +69,9 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version-file: third_party/go.mod
+      - uses: jdx/mise-action@v2
+        with:
+          install: false
       - run: make tools
 
   linters:

diff --git a/.mise.toml b/.mise.toml
@@ -0,0 +1,3 @@
+[env]
+
+MISE_DATA_DIR='bin/'
diff --git a/.tools_versions.yaml b/.tools_versions.yaml
@@ -0,0 +1,20 @@
+# renovate: datasource=github-releases depName=kubernetes/code-generator
+kube-code-generator: "0.29.1"
+# renovate: datasource=github-releases depName=kubernetes-sigs/controller-tools
+controller-tools: "0.14.0"
+# renovate: datasource=github-releases depName=kubernetes-sigs/kustomize
+kustomize: "5.3.0"
+# renovate: datasource=github-releases depName=golangci/golangci-lint
+golangci-lint: "1.57.1"
+# renovate: datasource=github-releases depName=GoogleContainerTools/skaffold
+skaffold: "2.10.0"
+# renovate: datasource=github-releases depName=kubernetes-sigs/controller-runtime
+setup-envtest: "0.17.2"
+# renovate: datasource=github-releases depName=elastic/crd-ref-docs
+crd-ref-docs: "0.0.12"
+# renovate: datasource=github-releases depName=mikefarah/yq
+yq: "4.43.1"
+# renovate: datasource=github-releases depName=jstemmer/go-junit-report
+gojunit-report: "2.1.0"
+# renovate: datasource=github-releases depName=gotestyourself/gotestsum
+gotestsum: "1.11.0"
diff --git a/Makefile b/Makefile
@@ -44,69 +44,98 @@ _download_tool:
 	(cd third_party && go mod tidy && \
 		GOBIN=$(PROJECT_DIR)/bin go generate -tags=third_party ./$(TOOL).go )
 
-.PHONY: _download_tool_own
-_download_tool_own:
-	(cd third_party/$(TOOL) && \
-		ls ./$(TOOL).go > /dev/null && \
-		GOBIN=$(PROJECT_DIR)/bin go generate -tags=third_party ./$(TOOL).go )
+TOOLS_VERSIONS_FILE = .tools_versions.yaml
+
+MISE := $(shell which mise)
+.PHONY: mise
+mise:
+	@mise -V >/dev/null || (echo "mise - https://github.com/jdx/mise - not found. Please install it." && exit 1)
 
 .PHONY: tools
 tools: controller-gen kustomize client-gen golangci-lint.download gotestsum crd-ref-docs skaffold looppointer.download staticcheck.download
 
-CONTROLLER_GEN = $(PROJECT_DIR)/bin/controller-gen
+export MISE_DATA_DIR = $(PROJECT_DIR)/bin/
+
+CONTROLLER_GEN_VERSION = $(shell yq -ojson -r '.controller-tools' < $(TOOLS_VERSIONS_FILE))
+CONTROLLER_GEN = $(PROJECT_DIR)/bin/installs/kube-controller-tools/$(CONTROLLER_GEN_VERSION)/bin/controller-gen
 .PHONY: controller-gen
-controller-gen: ## Download controller-gen locally if necessary.
-	@$(MAKE) _download_tool TOOL=controller-gen
+controller-gen: mise ## Download controller-gen locally if necessary.
+	@$(MISE) plugin install --yes -q kube-controller-tools
+	@$(MISE) install -q kube-controller-tools@$(CONTROLLER_GEN_VERSION)
 
-KUSTOMIZE = $(PROJECT_DIR)/bin/kustomize
+KUSTOMIZE_VERSION = $(shell yq -ojson -r '.kustomize' < $(TOOLS_VERSIONS_FILE))
+KUSTOMIZE = $(PROJECT_DIR)/bin/installs/kustomize/$(KUSTOMIZE_VERSION)/bin/kustomize
 .PHONY: kustomize
-kustomize: ## Download kustomize locally if necessary.
-	@$(MAKE) _download_tool TOOL=kustomize
+kustomize: mise ## Download kustomize locally if necessary.
+	@$(MISE) plugin install --yes -q kustomize
+	@$(MISE) install -q kustomize@$(KUSTOMIZE_VERSION)
 
-CLIENT_GEN = $(PROJECT_DIR)/bin/client-gen
+CLIENT_GEN_VERSION = $(shell yq -ojson -r '.kube-code-generator' < $(TOOLS_VERSIONS_FILE))
+CLIENT_GEN = $(PROJECT_DIR)/bin/installs/kube-code-generator/$(CLIENT_GEN_VERSION)/bin/client-gen
 .PHONY: client-gen
-client-gen: ## Download client-gen locally if necessary.
-	@$(MAKE) _download_tool TOOL=client-gen
+client-gen: mise ## Download client-gen locally if necessary.
+	@$(MISE) plugin install --yes -q kube-code-generator
+	@$(MISE) install -q kube-code-generator@$(CLIENT_GEN_VERSION)
 
-GOLANGCI_LINT = $(PROJECT_DIR)/bin/golangci-lint
+GOLANGCI_LINT_VERSION = $(shell yq -ojson -r '.golangci-lint' < $(TOOLS_VERSIONS_FILE))
+GOLANGCI_LINT = $(PROJECT_DIR)/bin/installs/golangci-lint/$(GOLANGCI_LINT_VERSION)/bin/golangci-lint
 .PHONY: golangci-lint.download
-golangci-lint.download: ## Download golangci-lint locally if necessary.
-	@$(MAKE) _download_tool TOOL=golangci-lint
+golangci-lint.download: mise ## Download golangci-lint locally if necessary.
+	@$(MISE) plugin install --yes -q golangci-lint
+	@$(MISE) install -q golangci-lint@$(GOLANGCI_LINT_VERSION)
 
-GOTESTSUM = $(PROJECT_DIR)/bin/gotestsum
+GOTESTSUM_VERSION = $(shell yq -ojson -r '.gotestsum' < $(TOOLS_VERSIONS_FILE))
+GOTESTSUM = $(PROJECT_DIR)/bin/installs/gotestsum/$(GOTESTSUM_VERSION)/bin/gotestsum
 .PHONY: gotestsum
 gotestsum: ## Download gotestsum locally if necessary.
-	@$(MAKE) _download_tool TOOL=gotestsum
+	@$(MISE) plugin install --yes -q gotestsum https://github.com/pmalek/mise-gotestsum.git
+	@$(MISE) install -q gotestsum
 
+CRD_REF_DOCS_VERSION = $(shell yq -ojson -r '.crd-ref-docs' < $(TOOLS_VERSIONS_FILE))
 CRD_REF_DOCS = $(PROJECT_DIR)/bin/crd-ref-docs
 .PHONY: crd-ref-docs
 crd-ref-docs: ## Download crd-ref-docs locally if necessary.
-	@$(MAKE) _download_tool TOOL=crd-ref-docs
+	GOBIN=$(PROJECT_DIR)/bin go install -v \
+		github.com/elastic/crd-ref-docs@v$(CRD_REF_DOCS_VERSION)
+
+SKAFFOLD_VERSION = $(shell yq -ojson -r '.skaffold' < $(TOOLS_VERSIONS_FILE))
+SKAFFOLD = $(PROJECT_DIR)/bin/installs/skaffold/$(SKAFFOLD_VERSION)/bin/skaffold
+.PHONY: skaffold
+skaffold: mise ## Download skaffold locally if necessary.
+	@$(MISE) plugin install --yes -q skaffold
+	@$(MISE) install -q skaffold@$(SKAFFOLD_VERSION)
+
+YQ_VERSION = $(shell yq -ojson -r '.yq' < $(TOOLS_VERSIONS_FILE))
+YQ = $(PROJECT_DIR)/bin/installs/yq/$(YQ_VERSION)/bin/yq
+.PHONY: yq
+yq: mise # Download yq locally if necessary.
+	@$(MISE) plugin install --yes -q yq
+	@$(MISE) install -q yq@$(YQ_VERSION)
 
 DLV = $(PROJECT_DIR)/bin/dlv
 .PHONY: dlv
 dlv: ## Download dlv locally if necessary.
 	@$(MAKE) _download_tool TOOL=dlv
 
-SETUP_ENVTEST = $(PROJECT_DIR)/bin/setup-envtest
+SETUP_ENVTEST_VERSION = $(shell yq -ojson -r '.setup-envtest' < $(TOOLS_VERSIONS_FILE))
+SETUP_ENVTEST = $(PROJECT_DIR)/bin/installs/setup-envtest/$(SETUP_ENVTEST_VERSION)/bin/setup-envtest
 .PHONY: setup-envtest
-setup-envtest: ## Download setup-envtest locally if necessary.
-	@$(MAKE) _download_tool TOOL=setup-envtest
-
-SKAFFOLD = $(PROJECT_DIR)/bin/skaffold
-.PHONY: skaffold
-skaffold: ## Download skaffold locally if necessary.
-	@$(MAKE) _download_tool_own TOOL=skaffold
+setup-envtest: mise ## Download setup-envtest locally if necessary.
+	@$(MISE) plugin install --yes -q setup-envtest https://github.com/pmalek/mise-setup-envtest.git
+	@$(MISE) install setup-envtest@v$(SETUP_ENVTEST_VERSION)
 
 STATICCHECK = $(PROJECT_DIR)/bin/staticcheck
 .PHONY: staticcheck.download
 staticcheck.download: ## Download staticcheck locally if necessary.
 	@$(MAKE) _download_tool TOOL=staticcheck
 
-GOJUNIT= $(PROJECT_DIR)/bin/go-junit-report
+GOJUNIT_REPORT_VERSION = $(shell yq -ojson -r '.gojunit-report' < $(TOOLS_VERSIONS_FILE))
+GOJUNIT_REPORT = $(PROJECT_DIR)/bin/installs/go-junit-report/$(GOJUNIT_REPORT_VERSION)/bin/go-junit-report
 .PHONY: go-junit-report
 go-junit-report: ## Download go-junit-report locally if necessary.
-	@$(MAKE) _download_tool TOOL=go-junit-report
+# TODO: Go back to using https://github.com/jwillker/asdf-go-junit-report when https://github.com/jwillker/asdf-go-junit-report/pull/4 merges.
+	@$(MISE) plugin install --yes -q go-junit-report https://github.com/pmalek/asdf-go-junit-report.git
+	@$(MISE) install go-junit-report@v$(GOJUNIT_REPORT_VERSION)
 
 LOOPPOINTER= $(PROJECT_DIR)/bin/looppointer
 .PHONY: looppointer.download
@@ -229,7 +258,7 @@ manifests.webhook: controller-gen ## Generate ValidatingWebhookConfiguration.
 
 .PHONY: manifests.single
 manifests.single: kustomize ## Compose single-file deployment manifests from building blocks
-	./scripts/build-single-manifests.sh
+	./scripts/build-single-manifests.sh $(KUSTOMIZE)
 
 # ------------------------------------------------------------------------------
 # Build - Generators
@@ -339,7 +368,7 @@ test.conformance: _check.container.environment go-junit-report
 		-timeout $(INTEGRATION_TEST_TIMEOUT) \
 		-parallel $(NCPU) \
 		./test/conformance | \
-	$(GOJUNIT) -iocopy -out $(JUNIT_REPORT) -parser gotest
+	$(GOJUNIT_REPORT) -iocopy -out $(JUNIT_REPORT) -parser gotest
 
 .PHONY: test.integration
 test.integration: test.integration.dbless test.integration.postgres
@@ -432,7 +461,7 @@ _test.integration: _check.container.environment go-junit-report
 		-coverpkg=$(PKG_LIST) \
 		-coverprofile=$(COVERAGE_OUT) \
 		./test/integration | \
-	$(GOJUNIT) -iocopy -out $(JUNIT_REPORT) -parser gotest
+	$(GOJUNIT_REPORT) -iocopy -out $(JUNIT_REPORT) -parser gotest
 
 .PHONY: _test.integration.isolated
 _test.integration.isolated: _check.container.environment go-junit-report
@@ -450,7 +479,7 @@ _test.integration.isolated: _check.container.environment go-junit-report
 		-coverpkg=$(PKG_LIST) \
 		-coverprofile=$(COVERAGE_OUT) \
 		./test/integration/isolated -args --parallel $(E2E_FRAMEWORK_FLAGS) | \
-	$(GOJUNIT) -iocopy -out $(JUNIT_REPORT) -parser gotest
+	$(GOJUNIT_REPORT) -iocopy -out $(JUNIT_REPORT) -parser gotest
 
 .PHONY: test.integration.isolated.dbless
 test.integration.isolated.dbless:
@@ -548,7 +577,7 @@ _test.kongintegration: gotestsum go-junit-report
 		-coverpkg=$(PKG_LIST) \
 		-coverprofile=coverage.kongintegration.out \
 		./test/kongintegration | \
-	$(GOJUNIT) -iocopy -out $(JUNIT_REPORT) -parser gotest
+	$(GOJUNIT_REPORT) -iocopy -out $(JUNIT_REPORT) -parser gotest
 
 # ------------------------------------------------------------------------------
 # Operations - Local Deployment
@@ -745,3 +774,12 @@ deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in
 
 undeploy: ## Undeploy controller from the K8s cluster specified in $KUBECONFIG.
 	$(KUSTOMIZE) build config/default | kubectl delete -f -
+
+renovate:
+	docker run --rm -ti -e LOG_LEVEL=debug \
+		-e GITHUB_COM_TOKEN="$(shell gh auth token)" \
+		-e DOCKER_HUB_PASSWORD="" \
+		-v /tmp:/tmp \
+		-v $(shell pwd):/usr/src/app \
+		docker.io/renovate/renovate:full \
+		renovate --platform=local
diff --git a/renovate.json b/renovate.json
@@ -24,6 +24,16 @@
       "matchStrings": [
         "#\\s+renovate:\\s+datasource=(?<datasource>.*?)\\s+depName=(?<depName>.*?)\\s+(packageName=(?<packageName>.*)\\s+)?(registryUrl=(?<registryUrl>.*)\\s+)?versioning=(?<versioning>.*?)\\n.+'(?<currentValue>.*?)'"
       ]
+    },
+    {
+      "description": "Match dependencies in .tools_verisons.yaml that are properly annotated with `# renovate: datasource={} depName={}.`",
+      "customType": "regex",
+      "fileMatch": [
+        "\\.tools_versions\\.yaml$"
+      ],
+      "matchStrings": [
+        "# renovate: datasource=(?<datasource>.*?) depName=(?<depName>.*?)\\n.+\"(?<currentValue>.*?)\""
+      ]
     }
   ],
   "customDatasources": {

diff --git a/scripts/build-single-manifests.sh b/scripts/build-single-manifests.sh
@@ -4,6 +4,11 @@ set -o errexit
 set -o nounset
 set -o pipefail
 REPO_ROOT=$(dirname ${BASH_SOURCE})/..
+KUSTOMIZE="${1}"
+if [ -z "${KUSTOMIZE}" ]; then
+    echo "Usage: $0 <path-to-kustomize-binary>"
+    exit 1
+fi
 
 cd "${REPO_ROOT}"
 
@@ -15,7 +20,7 @@ function generate_all_in_one_manifest()
 {
     echo "Generating kustomize manifest for ${1} in ${2}"
     echo -e '# Generated by build-single-manifest.sh. NOT FOR PRODUCTION USE (only used internally for testing). DO NOT EDIT.\n' > "${2}"
-    "${REPO_ROOT}/bin/kustomize" build "${1}" >> "${2}"
+    "${KUSTOMIZE}" build "${1}" >> "${2}"
 }
 
 function generate_deprecation_message()

diff --git a/third_party/client-gen.go b/third_party/client-gen.go