Add test utilities for installing NGF (#1256)

Problem: 1. In order to start running system tests, we need a framework in place. 2. Sometimes when installed, NGF would start before the control plane CRD existed. Solution: 1. Implement a basic framework that installs and uninstalls NGF. 2. Add a polling loop to find the control plane CRD before exiting. --------- Co-authored-by: bjee19 <[email protected]>
nginx · Nov 20, 2023 · 789b801 · 789b801
1 parent 7de105c
commit 789b801
Show file tree

Hide file tree

Showing 11 changed files with 360 additions and 30 deletions.
diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml
@@ -74,7 +74,7 @@ jobs:
         run: |
           ngf_prefix=$(echo ${{ steps.ngf-meta.outputs.tags }} | cut -d ":" -f 1)
           ngf_tag=$(echo ${{ steps.ngf-meta.outputs.tags }} | cut -d ":" -f 2)
-          make update-ngf-manifest NGF_PREFIX=${ngf_prefix} NGF_TAG=${ngf_tag}
+          make update-ngf-manifest PREFIX=${ngf_prefix} TAG=${ngf_tag}
         working-directory: ./conformance
 
       - name: Build binary
@@ -146,12 +146,12 @@ jobs:
           ngf_tag=$(echo ${{ steps.ngf-meta.outputs.tags }} | cut -d ":" -f 2)
           if [ ${{ github.event_name }} == "schedule" ]; then export GW_API_VERSION=main; fi
           if [ ${{ startsWith(matrix.k8s-version, '1.23') ||  startsWith(matrix.k8s-version, '1.24') }} == "true" ]; then export INSTALL_WEBHOOK=true; fi
-          make install-ngf-local-no-build NGF_PREFIX=${ngf_prefix} NGF_TAG=${ngf_tag}
+          make install-ngf-local-no-build PREFIX=${ngf_prefix} TAG=${ngf_tag}
         working-directory: ./conformance
 
       - name: Run conformance tests
         run: |
-          make run-conformance-tests TAG=${{ github.sha }} VERSION=${{ github.ref_name }}
+          make run-conformance-tests CONFORMANCE_TAG=${{ github.sha }} VERSION=${{ github.ref_name }}
           core_result=$(cat conformance-profile.yaml | yq '.profiles[0].core.result')
           extended_result=$(cat conformance-profile.yaml | yq '.profiles[0].extended.result')
           if [ "${core_result}" == "failure" ] || [ "${extended_result}" == "failure" ]; then echo "Conformance test failed, see above for details." && exit 2; fi

diff --git a/Makefile b/Makefile
@@ -123,7 +123,7 @@ lint: ## Run golangci-lint against code
 
 .PHONY: unit-test
 unit-test: ## Run unit tests for the go code
-	go test ./... -race -coverprofile cover.out
+	go test ./... -tags unit -race -coverprofile cover.out
 	go tool cover -html=cover.out -o cover.html
 
 .PHONY: njs-unit-test

diff --git a/conformance/Makefile b/conformance/Makefile
@@ -1,13 +1,13 @@
-NGF_TAG = edge
-NGF_PREFIX = nginx-gateway-fabric
-NGINX_IMAGE_NAME = $(NGF_PREFIX)/nginx
+TAG = edge
+PREFIX = nginx-gateway-fabric
+NGINX_PREFIX = $(PREFIX)/nginx
 GW_API_VERSION ?= 1.0.0
 GATEWAY_CLASS = nginx
 SUPPORTED_FEATURES = HTTPRoute,HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteSchemeRedirect,GatewayClassObservedGenerationBump
 KIND_IMAGE ?= $(shell grep -m1 'FROM kindest/node' <tests/Dockerfile | awk -F'[ ]' '{print $$2}')
 KIND_KUBE_CONFIG=$${HOME}/.kube/kind/config
-TAG = latest
-PREFIX = conformance-test-runner
+CONFORMANCE_TAG = latest
+CONFORMANCE_PREFIX = conformance-test-runner
 NGF_MANIFEST=../deploy/manifests/nginx-gateway.yaml
 CRDS=../deploy/manifests/crds/
 STATIC_MANIFEST=provisioner/static-deployment.yaml
@@ -27,7 +27,7 @@ update-go-modules: ## Update the gateway-api go modules to latest main version
 
 .PHONY: build-test-runner-image
 build-test-runner-image: ## Build conformance test runner image
-	docker build -t $(PREFIX):$(TAG) -f tests/Dockerfile ..
+	docker build -t $(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) -f tests/Dockerfile ..
 
 .PHONY: create-kind-cluster
 create-kind-cluster: ## Create a kind cluster
@@ -36,15 +36,15 @@ create-kind-cluster: ## Create a kind cluster
 
 .PHONY: update-ngf-manifest
 update-ngf-manifest: ## Update the NGF deployment manifest image names and imagePullPolicies
-	cd .. && make generate-manifests HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE="--set nginxGateway.kind=skip" HELM_TEMPLATE_COMMON_ARGS="--set nginxGateway.image.repository=$(NGF_PREFIX) --set nginxGateway.image.tag=$(NGF_TAG) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=$(NGINX_IMAGE_NAME) --set nginx.image.tag=$(NGF_TAG) --set nginx.image.pullPolicy=Never" && cd -
+	cd .. && make generate-manifests HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE="--set nginxGateway.kind=skip" HELM_TEMPLATE_COMMON_ARGS="--set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=$(NGINX_PREFIX) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never" && cd -
 
 .PHONY: build-images
 build-images: ## Build NGF and nginx images
-	cd .. && make PREFIX=$(NGF_PREFIX) TAG=$(NGF_TAG) build-images
+	cd .. && make PREFIX=$(PREFIX) TAG=$(TAG) build-images
 
 .PHONY: load-images
 load-images: ## Load NGF and NGINX images on configured kind cluster
-	kind load docker-image $(NGF_PREFIX):$(NGF_TAG) $(NGINX_IMAGE_NAME):$(NGF_TAG)
+	kind load docker-image $(PREFIX):$(TAG) $(NGINX_PREFIX):$(TAG)
 
 .PHONY: prepare-ngf-dependencies
 prepare-ngf-dependencies: update-ngf-manifest ## Install NGF dependencies on configured kind cluster
@@ -55,7 +55,7 @@ prepare-ngf-dependencies: update-ngf-manifest ## Install NGF dependencies on con
 .PHONY: deploy-updated-provisioner
 deploy-updated-provisioner: ## Update provisioner manifest and deploy to the configured kind cluster
 	yq '(select(di != 3))' $(PROVISIONER_MANIFEST) | kubectl apply -f -
-	yq '(select(.spec.template.spec.containers[].image) | .spec.template.spec.containers[].image="$(NGF_PREFIX):$(NGF_TAG)" | .spec.template.spec.containers[].imagePullPolicy = "Never")' $(PROVISIONER_MANIFEST) | kubectl apply -f -
+	yq '(select(.spec.template.spec.containers[].image) | .spec.template.spec.containers[].image="$(PREFIX):$(TAG)" | .spec.template.spec.containers[].imagePullPolicy = "Never")' $(PROVISIONER_MANIFEST) | kubectl apply -f -
 
 .PHONY: install-ngf-local-build
 install-ngf-local-build: prepare-ngf-dependencies build-images load-images deploy-updated-provisioner ## Install NGF from local build with provisioner on configured kind cluster
@@ -69,10 +69,10 @@ install-ngf-edge: prepare-ngf-dependencies ## Install NGF with provisioner from
 
 .PHONY: run-conformance-tests
 run-conformance-tests: ## Run conformance tests
-	kind load docker-image $(PREFIX):$(TAG)
+	kind load docker-image $(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG)
 	kubectl apply -f tests/conformance-rbac.yaml
 	kubectl run -i conformance \
-		--image=$(PREFIX):$(TAG) --image-pull-policy=Never \
+		--image=$(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) --image-pull-policy=Never \
 		--overrides='{ "spec": { "serviceAccountName": "conformance" }  }' \
 		--restart=Never -- sh -c "go test -v . -tags conformance,experimental -args --gateway-class=$(GATEWAY_CLASS) \
 						        --supported-features=$(SUPPORTED_FEATURES) --version=$(VERSION) \

diff --git a/conformance/README.md b/conformance/README.md
@@ -40,10 +40,10 @@ update-ngf-manifest            Update the NGF deployment manifest image names an
 
 | Variable             | Default                                                                                                       | Description                                                                                                               |
 |----------------------|---------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------|
-| TAG                  | latest                                                                                                        | The tag for the conformance test image                                                                                    |
-| PREFIX               | conformance-test-runner                                                                                       | The prefix for the conformance test image                                                                                 |
-| NGF_TAG              | edge                                                                                                          | The tag for the locally built NGF image                                                                                   |
-| NGF_PREFIX           | nginx-gateway-fabric                                                                                          | The prefix for the locally built NGF image                                                                                |
+| CONFORMANCE_TAG                  | latest                                                                                                        | The tag for the conformance test image                                                                                    |
+| CONFORMANCE_PREFIX               | conformance-test-runner                                                                                       | The prefix for the conformance test image                                                                                 |
+| TAG              | edge                                                                                                          | The tag for the locally built NGF image                                                                                   |
+| PREFIX           | nginx-gateway-fabric                                                                                          | The prefix for the locally built NGF image                                                                                |
 | GW_API_VERSION       | 1.0.0                                                                                                         | Tag for the Gateway API version to check out. Set to `main` to get the latest version                                     |
 | KIND_IMAGE           | Latest kind image, as defined in the tests/Dockerfile                                                         | The kind image to use                                                                                                     |
 | KIND_KUBE_CONFIG     | ~/.kube/kind/config                                                                                           | The location of the kubeconfig                                                                                            |
@@ -97,7 +97,7 @@ make install-ngf-local-no-build
 > Note:  If choosing this option, the following step *must* be completed manually *before* you build the image:
 
 ```makefile
-make update-ngf-manifest NGF_PREFIX=<ngf_repo_name> NGF_TAG=<ngf_image_tag>
+make update-ngf-manifest PREFIX=<ngf_repo_name> TAG=<ngf_image_tag>
 ```
 
 #### *Option 3* Install NGINX Gateway Fabric from edge to configured kind cluster

diff --git a/docs/developer/release-process.md b/docs/developer/release-process.md
@@ -34,7 +34,7 @@ To create a new release, follow these steps:
 6. Prepare and merge a PR into the release branch to update the repo files for the release:
     1. Update the Helm [Chart.yaml](/deploy/helm-chart/Chart.yaml): the `appVersion` to `X.Y.Z`, the icon and source
        URLs to point at `vX.Y.Z`, and bump the `version`.
-    2. Adjust the `VERSION` variable in the [Makefile](/Makefile) and the `NGF_TAG` in the
+    2. Adjust the `VERSION` variable in the [Makefile](/Makefile) and the `TAG` in the
        [conformance tests Makefile](/conformance/Makefile) to `X.Y.Z`.
     3. Update the tag of NGF container images used in the Helm [values.yaml](/deploy/helm-chart/values.yaml) file, the
        [provisioner manifest](/conformance/provisioner/provisioner.yaml), and all docs to `X.Y.Z`.

diff --git a/go.mod b/go.mod
@@ -16,6 +16,7 @@ require (
 	github.com/spf13/cobra v1.8.0
 	go.uber.org/zap v1.26.0
 	k8s.io/api v0.28.4
+	k8s.io/apiextensions-apiserver v0.28.4
 	k8s.io/apimachinery v0.28.4
 	k8s.io/client-go v0.28.4
 	sigs.k8s.io/controller-runtime v0.16.3
@@ -82,8 +83,7 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apiextensions-apiserver v0.28.3 // indirect
-	k8s.io/component-base v0.28.3 // indirect
+	k8s.io/component-base v0.28.4 // indirect
 	k8s.io/klog/v2 v2.100.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect

diff --git a/go.sum b/go.sum
@@ -255,14 +255,14 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 k8s.io/api v0.28.4 h1:8ZBrLjwosLl/NYgv1P7EQLqoO8MGQApnbgH8tu3BMzY=
 k8s.io/api v0.28.4/go.mod h1:axWTGrY88s/5YE+JSt4uUi6NMM+gur1en2REMR7IRj0=
-k8s.io/apiextensions-apiserver v0.28.3 h1:Od7DEnhXHnHPZG+W9I97/fSQkVpVPQx2diy+2EtmY08=
-k8s.io/apiextensions-apiserver v0.28.3/go.mod h1:NE1XJZ4On0hS11aWWJUTNkmVB03j9LM7gJSisbRt8Lc=
+k8s.io/apiextensions-apiserver v0.28.4 h1:AZpKY/7wQ8n+ZYDtNHbAJBb+N4AXXJvyZx6ww6yAJvU=
+k8s.io/apiextensions-apiserver v0.28.4/go.mod h1:pgQIZ1U8eJSMQcENew/0ShUTlePcSGFq6dxSxf2mwPM=
 k8s.io/apimachinery v0.28.4 h1:zOSJe1mc+GxuMnFzD4Z/U1wst50X28ZNsn5bhgIIao8=
 k8s.io/apimachinery v0.28.4/go.mod h1:wI37ncBvfAoswfq626yPTe6Bz1c22L7uaJ8dho83mgg=
 k8s.io/client-go v0.28.4 h1:Np5ocjlZcTrkyRJ3+T3PkXDpe4UpatQxj85+xjaD2wY=
 k8s.io/client-go v0.28.4/go.mod h1:0VDZFpgoZfelyP5Wqu0/r/TRYcLYuJ2U1KEeoaPa1N4=
-k8s.io/component-base v0.28.3 h1:rDy68eHKxq/80RiMb2Ld/tbH8uAE75JdCqJyi6lXMzI=
-k8s.io/component-base v0.28.3/go.mod h1:fDJ6vpVNSk6cRo5wmDa6eKIG7UlIQkaFmZN2fYgIUD8=
+k8s.io/component-base v0.28.4 h1:c/iQLWPdUgI90O+T9TeECg8o7N3YJTiuz2sKxILYcYo=
+k8s.io/component-base v0.28.4/go.mod h1:m9hR0uvqXDybiGL2nf/3Lf0MerAfQXzkfWhUY58JUbU=
 k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
 k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780=

diff --git a/internal/mode/static/manager.go b/internal/mode/static/manager.go
@@ -9,10 +9,12 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	apiv1 "k8s.io/api/core/v1"
 	discoveryV1 "k8s.io/api/discovery/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/tools/record"
 	ctlr "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -376,8 +378,22 @@ func setInitialConfig(
 	defer cancel()
 
 	var config ngfAPI.NginxGateway
-	if err := reader.Get(ctx, configName, &config); err != nil {
-		return err
+	// Polling to wait for CRD to exist if the Deployment is created first.
+	if err := wait.PollUntilContextCancel(
+		ctx,
+		500*time.Millisecond,
+		true, /* poll immediately */
+		func(ctx context.Context) (bool, error) {
+			if err := reader.Get(ctx, configName, &config); err != nil {
+				if !apierrors.IsNotFound(err) {
+					return false, err
+				}
+				return false, nil
+			}
+			return true, nil
+		},
+	); err != nil {
+		return fmt.Errorf("NginxGateway %s not found: %w", configName, err)
 	}
 
 	// status is not updated until the status updater's cache is started and the

diff --git a/tests/Makefile b/tests/Makefile
@@ -0,0 +1,31 @@
+TAG = edge
+PREFIX = nginx-gateway-fabric
+NGINX_PREFIX = $(PREFIX)/nginx
+PULL_POLICY=Never
+GW_API_VERSION ?= 1.0.0
+K8S_VERSION ?= latest ## Expected format: 1.24 (major.minor) or latest
+
+.PHONY: help
+help: Makefile ## Display this help
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "; printf "Usage:\n\n    make \033[36m<target>\033[0m\n\nTargets:\n\n"}; {printf "    \033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+.PHONY: create-kind-cluster
+create-kind-cluster: ## Create a kind cluster
+	cd .. && make create-kind-cluster
+
+.PHONY: build-images
+build-images: ## Build NGF and NGINX images
+	cd .. && make PREFIX=$(PREFIX) TAG=$(TAG) build-images
+
+.PHONY: load-images
+load-images: ## Load NGF and NGINX images on configured kind cluster
+	kind load docker-image $(PREFIX):$(TAG) $(NGINX_PREFIX):$(TAG)
+
+test: ## Run the system tests against your default k8s cluster
+	go test -v . -args --gateway-api-version=$(GW_API_VERSION) --image-tag=$(TAG) \
+		--ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --pull-policy=$(PULL_POLICY) \
+		--k8s-version=$(K8S_VERSION)
+
+.PHONY: delete-kind-cluster
+delete-kind-cluster: ## Delete kind cluster
+	kind delete cluster