test: implement test runtime extension

.dockerignore

@@ -17,6 +17,9 @@ _artifacts
 Makefile
 **/Makefile
 
+# add yaml files from internal/kubevip which are required for embedding.
+!internal/kubevip/*.yaml
+
 # ignores changes to test-only code to avoid extra rebuilds
 test/e2e/**
 

Makefile

@@ -60,6 +60,7 @@ BUILD_DIR := .build
 TEST_DIR := test
 VCSIM_DIR := test/infrastructure/vcsim
 NETOP_DIR := test/infrastructure/net-operator
+TEST_EXTENSION_DIR := test/infrastructure/test-extension
 TOOLS_DIR := hack/tools
 TOOLS_BIN_DIR := $(abspath $(TOOLS_DIR)/$(BIN_DIR))
 FLAVOR_DIR := $(ROOT_DIR)/templates
@@ -233,6 +234,10 @@ VM_OPERATOR_ALL_ARCH = amd64 arm64
 NET_OPERATOR_IMAGE_NAME ?= cluster-api-net-operator
 NET_OPERATOR_IMG ?= $(STAGING_REGISTRY)/$(NET_OPERATOR_IMAGE_NAME)
 
+# test-extension
+TEST_EXTENSION_IMAGE_NAME ?= cluster-api-test-extension
+TEST_EXTENSION_IMG ?= $(STAGING_REGISTRY)/$(TEST_EXTENSION_IMAGE_NAME)
+
 # boskosctl
 BOSKOSCTL_IMG ?= gcr.io/k8s-staging-capi-vsphere/extra/boskosctl
 BOSKOSCTL_IMG_TAG ?= $(shell git describe --always --dirty)
@@ -271,6 +276,7 @@ SUPERVISOR_WEBHOOK_ROOT ?= $(MANIFEST_ROOT)/supervisor/webhook
 RBAC_ROOT ?= $(MANIFEST_ROOT)/rbac
 VCSIM_RBAC_ROOT ?= $(VCSIM_DIR)/config/rbac
 NETOP_RBAC_ROOT ?= $(NETOP_DIR)/config/rbac
+TEST_EXTENSION_RBAC_ROOT ?= $(TEST_EXTENSION_DIR)/config/rbac
 
 JANITOR_DIR ?= ./$(TOOLS_DIR)/janitor
 
@@ -318,6 +324,11 @@ generate-manifests: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc.
 		paths=./$(NETOP_DIR)/controllers/... \
         output:rbac:dir=$(NETOP_RBAC_ROOT) \
         rbac:roleName=manager-role
+	# test-extension is used for Runtime SDK tests
+	$(CONTROLLER_GEN) \
+		paths=./$(TEST_EXTENSION_DIR)/... \
+        output:rbac:dir=$(TEST_EXTENSION_RBAC_ROOT) \
+        rbac:roleName=manager-role
 	# vcsim crds are used for tests.
 	$(CONTROLLER_GEN) \
 		paths=./$(VCSIM_DIR)/api/v1alpha1 \
@@ -569,6 +580,15 @@ docker-build-net-operator: docker-pull-prerequisites ## Build the docker image f
 		$(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./$(NETOP_DIR)/config/default/manager_pull_policy.yaml"; \
 	fi
 
+.PHONY: docker-build-test-extension
+docker-build-test-extension: docker-pull-prerequisites ## Build the docker image for test-extension controller manager
+## reads Dockerfile from stdin to avoid an incorrectly cached Dockerfile (https://github.com/moby/buildkit/issues/1368)
+	cat $(TEST_EXTENSION_DIR)/Dockerfile | DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg ldflags="$(LDFLAGS)" . -t $(TEST_EXTENSION_IMG)-$(ARCH):$(TAG) --file -
+	@if [ "${DOCKER_BUILD_MODIFY_MANIFESTS}" = "true" ]; then \
+		$(MAKE) set-manifest-image MANIFEST_IMG=$(TEST_EXTENSION_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./$(TEST_EXTENSION_DIR)/config/default/manager_image_patch.yaml"; \
+		$(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./$(TEST_EXTENSION_DIR)/config/default/manager_pull_policy.yaml"; \
+	fi
+
 .PHONY: docker-build-boskosctl
 docker-build-boskosctl:
 	cat hack/tools/boskosctl/Dockerfile | DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) . -t $(BOSKOSCTL_IMG):$(BOSKOSCTL_IMG_TAG) --file -
@@ -636,6 +656,7 @@ e2e-images: ## Build the e2e manager image
 	$(MAKE) REGISTRY=gcr.io/k8s-staging-capi-vsphere PULL_POLICY=IfNotPresent TAG=dev docker-build
 	$(MAKE) REGISTRY=gcr.io/k8s-staging-capi-vsphere PULL_POLICY=IfNotPresent TAG=dev docker-build-vcsim
 	$(MAKE) REGISTRY=gcr.io/k8s-staging-capi-vsphere PULL_POLICY=IfNotPresent TAG=dev docker-build-net-operator
+	$(MAKE) REGISTRY=gcr.io/k8s-staging-capi-vsphere PULL_POLICY=IfNotPresent TAG=dev docker-build-test-extension
 
 .PHONY: e2e
 e2e: e2e-images generate-e2e-templates

test/infrastructure/test-extension/Dockerfile

@@ -0,0 +1,83 @@
+# syntax=docker/dockerfile:1.4
+
+# Copyright 2024 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Build the manager binary
+# Run this with docker build --build-arg builder_image=<golang:x.y.z>
+ARG builder_image
+
+# Build architecture
+ARG ARCH
+
+# Ignore Hadolint rule "Always tag the version of an image explicitly."
+# It's an invalid finding since the image is explicitly set in the Makefile.
+# https://github.com/hadolint/hadolint/wiki/DL3006
+# hadolint ignore=DL3006
+FROM ${builder_image} as builder
+WORKDIR /workspace
+
+# Run this with docker build --build-arg goproxy=$(go env GOPROXY) to override the goproxy
+ARG goproxy=https://proxy.golang.org
+ENV GOPROXY=$goproxy
+
+# Copy the Go Modules manifests
+COPY go.mod go.mod
+COPY go.sum go.sum
+
+# Change directories into the test go module
+WORKDIR /workspace/test
+
+# Copy the Go Modules manifests
+COPY test/go.mod go.mod
+COPY test/go.sum go.sum
+
+# Cache deps before building and copying source so that we don't need to re-download as much
+# and so that source changes don't invalidate our downloaded layer
+RUN --mount=type=cache,target=/go/pkg/mod \
+    go mod download
+
+# This needs to build with the entire CAPV context
+WORKDIR /workspace
+
+# Copy the sources (which includes the test/infrastructure/vcsim subdirectory)
+COPY ./ ./
+
+# Change directories into test-extension
+WORKDIR /workspace/test/infrastructure/test-extension
+
+# Cache the go build into the Go’s compiler cache folder so we take benefits of compiler caching across docker build calls
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    go build .
+
+# Build
+ARG package=.
+ARG ARCH
+ARG ldflags
+
+# Do not force rebuild of up-to-date packages (do not use -a) and use the compiler cache folder
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \
+    go build -trimpath -ldflags "${ldflags} -extldflags '-static'" \
+    -o manager ${package}
+
+
+FROM gcr.io/distroless/static:nonroot-${ARCH}
+WORKDIR /
+COPY --from=builder /workspace/test/infrastructure/test-extension/manager .
+# Use uid of nonroot user (65532) because kubernetes expects numeric user when applying pod security policies
+USER 65532
+ENTRYPOINT ["/manager"]

test/infrastructure/test-extension/README.md

@@ -0,0 +1,3 @@
+# CAPV test-extension
+
+Provide a minimal implementation of a Runtime SDK test-extension.

test/infrastructure/test-extension/config/certmanager/certificate.yaml

@@ -0,0 +1,24 @@
+# The following manifests contain a self-signed issuer CR and a certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: selfsigned-issuer
+  namespace: system
+spec:
+  selfSigned: {}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: serving-cert  # this name should match the one appeared in kustomizeconfig.yaml
+  namespace: system
+spec:
+  # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize
+  dnsNames:
+    - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc
+    - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local
+  issuerRef:
+    kind: Issuer
+    name: selfsigned-issuer
+  secretName: $(SERVICE_NAME)-cert # this secret will not be prefixed, since it's not managed by kustomize

test/infrastructure/test-extension/config/certmanager/kustomization.yaml

@@ -0,0 +1,5 @@
+resources:
+  - certificate.yaml
+
+configurations:
+  - kustomizeconfig.yaml

test/infrastructure/test-extension/config/certmanager/kustomizeconfig.yaml

@@ -0,0 +1,19 @@
+# This configuration is for teaching kustomize how to update name ref and var substitution
+nameReference:
+  - kind: Issuer
+    group: cert-manager.io
+    fieldSpecs:
+      - kind: Certificate
+        group: cert-manager.io
+        path: spec/issuerRef/name
+
+varReference:
+  - kind: Certificate
+    group: cert-manager.io
+    path: spec/commonName
+  - kind: Certificate
+    group: cert-manager.io
+    path: spec/dnsNames
+  - kind: Certificate
+    group: cert-manager.io
+    path: spec/secretName

test/infrastructure/test-extension/config/default/kustomization.yaml

@@ -0,0 +1,54 @@
+namespace: capv-test-extension
+
+namePrefix: capv-test-extension-
+
+commonLabels:
+  # capvsim is not a provider, but by adding this label
+  # we can get this installed by Cluster APIs Tiltfile and by the clusterctl machinery we use in E2E tests.
+  cluster.x-k8s.io/provider: "runtime-extension-capv-test"
+
+resources:
+  - namespace.yaml
+
+bases:
+  - ../rbac
+  - ../manager
+  - ../webhook
+  - ../certmanager
+
+patchesStrategicMerge:
+  # Provide customizable hook for make targets.
+  - manager_image_patch.yaml
+  - manager_pull_policy.yaml
+  - manager_webhook_patch.yaml
+
+vars:
+  - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
+    objref:
+      kind: Certificate
+      group: cert-manager.io
+      version: v1
+      name: serving-cert # this name should match the one in certificate.yaml
+    fieldref:
+      fieldpath: metadata.namespace
+  - name: CERTIFICATE_NAME
+    objref:
+      kind: Certificate
+      group: cert-manager.io
+      version: v1
+      name: serving-cert # this name should match the one in certificate.yaml
+  - name: SERVICE_NAMESPACE # namespace of the service
+    objref:
+      kind: Service
+      version: v1
+      name: webhook-service
+    fieldref:
+      fieldpath: metadata.namespace
+  - name: SERVICE_NAME
+    objref:
+      kind: Service
+      version: v1
+      name: webhook-service
+
+configurations:
+  - kustomizeconfig.yaml

test/infrastructure/test-extension/config/default/kustomizeconfig.yaml

@@ -0,0 +1,4 @@
+# This configuration is for teaching kustomize how to update name ref and var substitution
+varReference:
+- kind: Deployment
+  path: spec/template/spec/volumes/secret/secretName

test/infrastructure/test-extension/config/default/manager_image_patch.yaml

@@ -0,0 +1,11 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - image: gcr.io/k8s-staging-capi-vsphere/cluster-api-test-extension:dev
+        name: manager

test/infrastructure/test-extension/config/default/manager_pull_policy.yaml

@@ -0,0 +1,11 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: manager
+        imagePullPolicy: Always

test/infrastructure/test-extension/config/default/manager_webhook_patch.yaml

@@ -0,0 +1,23 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: manager
+        ports:
+        - containerPort: 9443
+          name: webhook-server
+          protocol: TCP
+        volumeMounts:
+        - mountPath: /tmp/k8s-webhook-server/serving-certs
+          name: cert
+          readOnly: true
+      volumes:
+      - name: cert
+        secret:
+          secretName: $(SERVICE_NAME)-cert # this secret will not be prefixed, since it's not managed by kustomize
+

test/infrastructure/test-extension/config/default/namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    control-plane: controller-manager
+  name: system

test/infrastructure/test-extension/config/manager/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+- manager.yaml

test/infrastructure/test-extension/config/manager/manager.yaml

@@ -0,0 +1,61 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+  labels:
+    control-plane: controller-manager
+spec:
+  selector:
+    matchLabels:
+      control-plane: controller-manager
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        control-plane: controller-manager
+    spec:
+      containers:
+      - command:
+        - /manager
+        args:
+        - "--leader-elect"
+        - "--diagnostics-address=${CAPI_DIAGNOSTICS_ADDRESS:=:8443}"
+        - "--insecure-diagnostics=${CAPI_INSECURE_DIAGNOSTICS:=false}"
+        image: controller:latest
+        name: manager
+        ports:
+        - containerPort: 9440
+          name: healthz
+          protocol: TCP
+        - containerPort: 8443
+          name: metrics
+          protocol: TCP
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: healthz
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: healthz
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+              - ALL
+          privileged: false
+          runAsUser: 65532
+          runAsGroup: 65532
+        terminationMessagePolicy: FallbackToLogsOnError
+      terminationGracePeriodSeconds: 10
+      serviceAccountName: manager
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
+      securityContext:
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault