Merge pull request #5777 from the-wondersmith/dev/switch-to-uv-and-ruff

chore(build/ci): switch to uv and ruff

.github/actions/collect-logs/action.yml

@@ -48,8 +48,12 @@ runs:
 
           tools/bin/kubectl cp xfpredirect:/tmp/ambassador/snapshots /tmp/test-logs/cluster/xfpredirect.snapshots || true
         fi
-        cp /tmp/*.yaml /tmp/test-logs || true
-        cp /tmp/kat-client-*.log /tmp/test-logs || true
+        if compgen -G '/tmp/*.yaml' > /dev/null; then
+          cp /tmp/*.yaml /tmp/test-logs/ || true
+        fi
+        if compgen -G '/tmp/kat-client-*.log' > /dev/null; then
+          cp /tmp/kat-client-*.log /tmp/test-logs/ || true
+        fi
     - name: Upload Logs
       uses: actions/upload-artifact@v4
       with:

.github/actions/setup-deps/action.yml

@@ -2,6 +2,18 @@
 name: setup-deps
 description: Install Go and Python
 
+inputs:
+  install-golang:
+    description: |
+      Install Go
+    required: false
+    default: "true"
+  install-python:
+    description: |
+      Install Python
+    required: false
+    default: "true"
+
 runs:
   using: composite
   steps:
@@ -51,30 +63,24 @@ runs:
             helm plugin install https://github.com/chartmuseum/helm-push
         fi
     - name: Install Go
+      if: ${{ inputs.install-golang == 'true' }}
       uses: actions/setup-go@v5
       env:
         DEBIAN_FRONTEND: noninteractive
       with:
         go-version-file: go.mod
-    - name: Get Required Python Version
-      id: get-python-version
-      uses: mikefarah/yq@master
-      env:
-        DEBIAN_FRONTEND: noninteractive
-      with:
-        cmd: >-
-          yq
-          '.project.requires-python | capture("(?<version>3.[0-9]+(\.[0-9]+)?)").version'
-          pyproject.toml
-    - name: Install Py (${{ steps.get-python-version.outputs.result }})
-      uses: actions/setup-python@v5
+    - name: Install 'uv'
+      shell: bash
+      if: ${{ inputs.install-python == 'true' }}
       env:
         DEBIAN_FRONTEND: noninteractive
-      with:
-        python-version: ${{ steps.get-python-version.outputs.result }}
+      run: |
+        curl -LsSf https://astral.sh/uv/install.sh | bash
     - name: Install Python Requirements
       shell: bash
+      if: ${{ inputs.install-python == 'true' }}
       env:
         DEBIAN_FRONTEND: noninteractive
       run: |
-        python -m pip install -U awscli packaging
+        uv venv
+        uv --no-config pip install -U awscli

.github/workflows/ci.yaml

@@ -30,7 +30,7 @@ jobs:
         run: |
           make lint
       - name: Collect Job Logs
-        uses: ./.github/actions/after-job
+        uses: ./.github/actions/collect-logs
         if: always()
 
   gotest:  ######################################################################
@@ -45,12 +45,14 @@ jobs:
           fetch-depth: 0
       - name: Install Deps
         uses: ./.github/actions/setup-deps
+        with:
+          install-python: "true"
       - name: Run Golang Unit Tests
         shell: bash
         run: |
           make gotest
       - name: Collect Job Logs
-        uses: ./.github/actions/after-job
+        uses: ./.github/actions/collect-logs
         if: always()
 
   pytest:  ######################################################################
@@ -65,6 +67,8 @@ jobs:
           fetch-depth: 0
       - name: Install Deps
         uses: ./.github/actions/setup-deps
+        with:
+          install-golang: "false"
       - name: Create Python Virtual Env
         shell: bash
         run: >-
@@ -73,10 +77,9 @@ jobs:
         env:
           PYTEST_ARGS: --cov-branch --cov=ambassador --cov-report=html:/tmp/cov_html
         run: |
-          source venv/bin/activate
           make pytest-unit-tests
       - name: Collect Job Logs
-        uses: ./.github/actions/after-job
+        uses: ./.github/actions/collect-logs
         if: always()
 
   build:  #######################################################################
@@ -109,7 +112,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           goreleaser build --snapshot --clean
-      - name: Collect Job Logs
+      - name: Post-Build Cleanup
         uses: ./.github/actions/after-job
         if: always()
 

.gitignore

@@ -1,7 +1,11 @@
 **/venv
+**/.venv
 tools/bin
 /build-output/
 /vendor/
+**/uv.lock
+**/.*cache
+**/coverage.xml
 
 *.tgz
 *.img.tar

build-aux/builder.mk

@@ -127,7 +127,7 @@ test-ready: push preflight-cluster
 PYTEST_ARGS ?=
 export PYTEST_ARGS
 
-python-virtual-environment: $(OSS_HOME)/venv
+python-virtual-environment: $(OSS_HOME)/.venv
 .PHONY: python-virtual-environment
 
 python-integration-test-environment: push-pytest-images
@@ -143,7 +143,7 @@ pytest-run-tests:
 	@echo "DEV_KUBECONFIG=$$DEV_KUBECONFIG"
 	@echo "PYTEST_ARGS=$$PYTEST_ARGS"
 	set -e; { \
-	  . $(OSS_HOME)/venv/bin/activate; \
+	  . $(OSS_HOME)/.venv/bin/activate; \
 	  export SOURCE_ROOT=$(CURDIR); \
 	  export ENVOY_DOCKER_TAG=ghcr.io/emissary-ingress/emissary:latest-$(ARCH)
 	  export KUBESTATUS_PATH=$(CURDIR)/tools/bin/kubestatus; \
@@ -152,7 +152,7 @@ pytest-run-tests:
 .PHONY: pytest-run-tests
 
 pytest: python-integration-test-environment
-	$(MAKE) pytest-run-tests PYTEST_ARGS="$$PYTEST_ARGS python/tests"
+	$(MAKE) pytest-run-tests PYTEST_ARGS="$$PYTEST_ARGS python/tests/src/tests"
 .PHONY: pytest
 
 # XXX We use python -m pytest to force the current directory to be included in
@@ -161,9 +161,8 @@ pytest: python-integration-test-environment
 pytest-unit-tests: python-virtual-environment
 	@printf "$(CYN)==> $(GRN)Running $(BLU)py$(GRN) unit tests$(END)\n"
 	set -e; { \
-		. $(OSS_HOME)/venv/bin/activate; \
 		export SOURCE_ROOT=$(CURDIR); \
-		cd python && python -m pytest --tb=short $(PYTEST_ARGS) tests/unit; \
+		uv run pytest --tb=short $(PYTEST_ARGS) python/tests/src/tests/unit; \
 	}
 .PHONY: pytest-unit-tests
 # Make sure to clobber python/.coverage.
@@ -173,41 +172,37 @@ pytest-unit: python-virtual-environment pytest-unit-tests
 .PHONY: pytest-unit
 
 pytest-integration-tests:
-	$(MAKE) pytest-run-tests PYTEST_ARGS="$$PYTEST_ARGS python/tests/integration"
+	$(MAKE) pytest-run-tests PYTEST_ARGS="$$PYTEST_ARGS python/tests/src/tests/integration"
 .PHONY: pytest-integration-tests
 
 pytest-integration: python-integration-test-environment pytest-integration-tests
 .PHONY: pytest-integration
 
 pytest-kat-envoy3-tests: # doing this all at once is too much for CI...
-	$(MAKE) pytest-run-tests PYTEST_ARGS="$$PYTEST_ARGS python/tests/kat"
+	$(MAKE) pytest-run-tests PYTEST_ARGS="$$PYTEST_ARGS python/tests/src/tests/kat"
 pytest-kat-envoy3: python-integration-test-environment pytest-kat-envoy3-tests
 # ... so we have a separate rule to run things split up
-build-aux/.pytest-kat.txt.stamp: $(OSS_HOME)/venv push-pytest-images $(tools/kubectl) FORCE
-	. venv/bin/activate && set -o pipefail && pytest --collect-only python/tests/kat 2>&1 | sed -En 's/.*<Function (.*)>/\1/p' | cut -d. -f1 | sort -u > $@
+build-aux/.pytest-kat.txt.stamp: $(OSS_HOME)/.venv push-pytest-images $(tools/kubectl) FORCE
+	set -o pipefail && uv run pytest --collect-only python/tests/src/tests/kat 2>&1 | sed -En 's/.*<Function (.*)>/\1/p' | cut -d. -f1 | sort -u > $@
 build-aux/pytest-kat.txt: build-aux/%: build-aux/.%.stamp $(tools/copy-ifchanged)
 	$(tools/copy-ifchanged) $< $@
 clean: build-aux/.pytest-kat.txt.stamp.rm build-aux/pytest-kat.txt.rm
 pytest-kat-envoy3-tests-%: build-aux/pytest-kat.txt $(tools/py-split-tests)
-	$(MAKE) pytest-run-tests PYTEST_ARGS="$$PYTEST_ARGS -k '$$($(tools/py-split-tests) $(subst -of-, ,$*) <build-aux/pytest-kat.txt)' python/tests/kat"
+	$(MAKE) pytest-run-tests PYTEST_ARGS="$$PYTEST_ARGS -k '$$($(tools/py-split-tests) $(subst -of-, ,$*) <build-aux/pytest-kat.txt)' python/tests/src/tests/kat"
 pytest-kat-envoy3-%: python-integration-test-environment pytest-kat-envoy3-tests-%
 
-$(OSS_HOME)/venv: python/requirements.txt python/requirements-dev.txt
+$(OSS_HOME)/.venv: pyproject.toml
 	rm -rf $@
-	python3 -m venv $@
-	$@/bin/pip3 install -r python/requirements.txt
-	$@/bin/pip3 install -r python/requirements-dev.txt
-	$@/bin/pip3 install -e $(OSS_HOME)/python
-clobber: venv.rm-r
+	uv sync
+clobber: .venv.rm-r
 
 GOTEST_ARGS ?= -race -count=1 -timeout=30m -ldflags=-linkmode=internal
 GOTEST_ARGS += -parallel=150 # The ./pkg/envoy-control-plane/cache/v{2,3}/ tests require high parallelism to reliably work
 GOTEST_PKGS ?= ./...
-gotest: $(OSS_HOME)/venv $(tools/kubectl)
+gotest: $(OSS_HOME)/.venv $(tools/kubectl)
 	@printf "$(CYN)==> $(GRN)Running $(BLU)go$(GRN) tests$(END)\n"
-	{ . $(OSS_HOME)/venv/bin/activate && \
-	  export PATH=$(tools.bindir):$${PATH} && \
-	  go test $(GOTEST_ARGS) $(GOTEST_PKGS); }
+	{ export PATH=$(tools.bindir):$${PATH} && \
+	  uv run go test $(GOTEST_ARGS) $(GOTEST_PKGS); }
 .PHONY: gotest
 
 # Ingress v1 conformance tests, using KIND and the Ingress Conformance Tests suite.

build-aux/check.mk

@@ -3,7 +3,7 @@ include build-aux/tools.mk
 #
 # Auxiliary Docker images needed for the tests
 
-# Keep this list in-sync with python/tests/integration/manifests.py
+# Keep this list in-sync with python/tests/src/tests/integration/manifests.py
 push-pytest-images: images
 	k3d image load -c $(TEST_CLUSTER) ghcr.io/emissary-ingress/test-auth:latest-$(ARCH)
 	k3d image load -c $(TEST_CLUSTER) ghcr.io/emissary-ingress/test-shadow:latest-$(ARCH)
@@ -29,9 +29,9 @@ build-output/chart-%/ci: build-output/chart-% test-chart-values.yaml
 	cp -a $@.in $@
 	for file in $@/*-values.yaml; do cat test-chart-values.yaml >> "$$file"; done
 
-test-chart: $(tools/ct) $(tools/kubectl) $(chart_dir)/ci build-output/yaml-$(patsubst v%,%,$(VERSION)) $(if $(DEV_USE_IMAGEPULLSECRET),push-pytest-images $(OSS_HOME)/venv)
+test-chart: $(tools/ct) $(tools/kubectl) $(chart_dir)/ci build-output/yaml-$(patsubst v%,%,$(VERSION)) $(if $(DEV_USE_IMAGEPULLSECRET),push-pytest-images $(OSS_HOME)/.venv)
 ifneq ($(DEV_USE_IMAGEPULLSECRET),)
-	. venv/bin/activate && KUBECONFIG=$(DEV_KUBECONFIG) python3 -c 'from tests.integration.utils import install_crds; install_crds()'
+	KUBECONFIG=$(DEV_KUBECONFIG) uv run python3 -c 'from tests.integration.utils import install_crds; install_crds()'
 else
 	$(tools/kubectl) --kubeconfig=$(DEV_KUBECONFIG) apply -f build-output/yaml-$(patsubst v%,%,$(VERSION))/emissary-crds.yaml
 endif

build-aux/generate.mk

@@ -42,10 +42,10 @@ generate-fast/files += $(OSS_HOME)/manifests/emissary/emissary-crds.yaml.in
 generate-fast/files += $(OSS_HOME)/manifests/emissary/emissary-emissaryns.yaml.in
 generate-fast/files += $(OSS_HOME)/manifests/emissary/emissary-defaultns.yaml.in
 generate-fast/files += $(OSS_HOME)/pkg/api/getambassador.io/crds.yaml
-generate-fast/files += $(OSS_HOME)/python/tests/integration/manifests/ambassador.yaml
-generate-fast/files += $(OSS_HOME)/python/tests/integration/manifests/crds.yaml
-generate-fast/files += $(OSS_HOME)/python/tests/integration/manifests/rbac_cluster_scope.yaml
-generate-fast/files += $(OSS_HOME)/python/tests/integration/manifests/rbac_namespace_scope.yaml
+generate-fast/files += $(OSS_HOME)/python/tests/src/tests/integration/manifests/ambassador.yaml
+generate-fast/files += $(OSS_HOME)/python/tests/src/tests/integration/manifests/crds.yaml
+generate-fast/files += $(OSS_HOME)/python/tests/src/tests/integration/manifests/rbac_cluster_scope.yaml
+generate-fast/files += $(OSS_HOME)/python/tests/src/tests/integration/manifests/rbac_namespace_scope.yaml
 generate-fast/files += $(OSS_HOME)/test/apiext/testdata/deployment.yaml
 generate-fast/files += $(OSS_HOME)/test/apiext/testdata/crds.yaml
 generate-fast/files += $(OSS_HOME)/test/apiext/testdata/rbac.yaml
@@ -54,7 +54,7 @@ generate-fast/files += $(OSS_HOME)/docker/test-auth/authsvc.crt
 generate-fast/files += $(OSS_HOME)/docker/test-auth/authsvc.key
 generate-fast/files += $(OSS_HOME)/docker/test-shadow/shadowsvc.crt
 generate-fast/files += $(OSS_HOME)/docker/test-shadow/shadowsvc.key
-generate-fast/files += $(OSS_HOME)/python/tests/selfsigned.py
+generate-fast/files += $(OSS_HOME)/python/tests/src/tests/selfsigned.py
 
 generate: ## Update generated sources that get committed to Git
 generate:
@@ -107,7 +107,7 @@ $(OSS_HOME)/docker/test-shadow/shadowsvc.crt: $(tools/testcert-gen)
 $(OSS_HOME)/docker/test-shadow/shadowsvc.key: $(tools/testcert-gen)
 	$(tools/testcert-gen) --out-cert=/dev/null --out-key=$@ --hosts=demosvc.datawire.io
 
-$(OSS_HOME)/python/tests/selfsigned.py: %: %.gen $(tools/testcert-gen)
+$(OSS_HOME)/python/tests/src/tests/selfsigned.py: %: %.gen $(tools/testcert-gen)
 	$@.gen $(tools/testcert-gen) >$@
 
 #
@@ -186,7 +186,7 @@ $(OSS_HOME)/%/zz_generated.conversion-spoke.go: build-aux/conversion-spoke.go.aw
 $(OSS_HOME)/manifests/emissary/emissary-crds.yaml.in: $(OSS_HOME)/_generate.tmp/crds $(tools/fix-crds)
 	$(tools/fix-crds) --target=apiserver-kubectl $(sort $(wildcard $</*.yaml)) >$@
 
-$(OSS_HOME)/python/tests/integration/manifests/crds.yaml: $(OSS_HOME)/_generate.tmp/crds $(tools/fix-crds)
+$(OSS_HOME)/python/tests/src/tests/integration/manifests/crds.yaml: $(OSS_HOME)/_generate.tmp/crds $(tools/fix-crds)
 	$(tools/fix-crds) --target=apiserver-kat $(sort $(wildcard $</*.yaml)) >$@
 
 $(OSS_HOME)/pkg/api/getambassador.io/crds.yaml: $(OSS_HOME)/_generate.tmp/crds $(tools/fix-crds)
@@ -248,13 +248,13 @@ clean: k8s-config.clean
 $(OSS_HOME)/manifests/emissary/%.yaml.in: $(OSS_HOME)/k8s-config/%/output.yaml
 	cp $< $@
 
-$(OSS_HOME)/python/tests/integration/manifests/%.yaml: $(OSS_HOME)/k8s-config/kat-%/output.yaml
+$(OSS_HOME)/python/tests/src/tests/integration/manifests/%.yaml: $(OSS_HOME)/k8s-config/kat-%/output.yaml
 	sed -e 's/«/{/g' -e 's/»/}/g' -e 's/♯.*//g' -e 's/- ←//g' <$< >$@
 
-$(OSS_HOME)/python/tests/integration/manifests/rbac_cluster_scope.yaml: $(OSS_HOME)/k8s-config/kat-rbac-multinamespace/output.yaml
+$(OSS_HOME)/python/tests/src/tests/integration/manifests/rbac_cluster_scope.yaml: $(OSS_HOME)/k8s-config/kat-rbac-multinamespace/output.yaml
 	sed -e 's/«/{/g' -e 's/»/}/g' -e 's/♯.*//g' -e 's/- ←//g' <$< >$@
 
-$(OSS_HOME)/python/tests/integration/manifests/rbac_namespace_scope.yaml: $(OSS_HOME)/k8s-config/kat-rbac-singlenamespace/output.yaml
+$(OSS_HOME)/python/tests/src/tests/integration/manifests/rbac_namespace_scope.yaml: $(OSS_HOME)/k8s-config/kat-rbac-singlenamespace/output.yaml
 	sed -e 's/«/{/g' -e 's/»/}/g' -e 's/♯.*//g' -e 's/- ←//g' <$< >$@
 
 #

build-aux/lint.mk

@@ -17,13 +17,13 @@ format/go: $(tools/golangci-lint)
 #
 # Python
 
-lint-deps += $(OSS_HOME)/venv
+lint-deps += $(OSS_HOME)/.venv
 
 lint-goals += lint/mypy
-lint/mypy: $(OSS_HOME)/venv
+lint/mypy: $(OSS_HOME)/.venv
 	set -e; { \
-	  . $(OSS_HOME)/venv/bin/activate; \
-	  time mypy \
+	  uv run -- time mypy \
+	    --config-file='pyproject.toml' \
 	    --cache-fine-grained \
 	    --ignore-missing-imports \
 	    --check-untyped-defs \
@@ -33,25 +33,15 @@ lint/mypy: $(OSS_HOME)/venv
 clean: .dmypy.json.rm .mypy_cache.rm-r
 
 lint-goals += lint/black
-lint/black: $(OSS_HOME)/venv
-	. $(OSS_HOME)/venv/bin/activate && black --check ./python/
+lint/black: $(OSS_HOME)/.venv
+	uv run -- ruff check --config='pyproject.toml' python/
 .PHONY: lint/black
 
 format-goals += format/black
-format/black: $(OSS_HOME)/venv
-	. $(OSS_HOME)/venv/bin/activate && black ./python/
+format/black: $(OSS_HOME)/.venv
+	uv run -- ruff format --config='pyproject.toml' python/
 .PHONY: format/black
 
-lint-goals += lint/isort
-lint/isort: $(OSS_HOME)/venv
-	. $(OSS_HOME)/venv/bin/activate && isort --check --diff ./python/
-.PHONY: lint/isort
-
-format-goals += format/isort
-format/isort: $(OSS_HOME)/venv
-	. $(OSS_HOME)/venv/bin/activate && isort ./python/
-.PHONY: format/isort
-
 #
 # Helm
 

build-aux/tools.mk

@@ -19,7 +19,8 @@ clobber: clobber-tools
 
 .PHONY: clobber-tools
 clobber-tools:
-	rm -rf $(tools.bindir) $(tools.dir)/include $(tools.dir)/downloads
+	rm -rf uv.lock coverage.xml $(tools.bindir) $(tools.dir)/include $(tools.dir)/downloads
+	find . -type d -name '.*cache' -exec rm -rf {} \; || true
 
 go-mod-tidy: $(patsubst $(tools.srcdir)/%/go.mod,go-mod-tidy/tools/%,$(wildcard $(tools.srcdir)/*/go.mod))