Add fake apiserver

Signed-off-by: Huy Mai <[email protected]>

Makefile

@@ -36,6 +36,7 @@ export GO111MODULE=on
 # Full directory of where the Makefile resides
 ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
 TOOLS_DIR := hack/tools
+FAKE_APISERVER_DIR := hack/fake-apiserver
 APIS_DIR := api
 TEST_DIR := test
 BIN_DIR := bin
@@ -319,6 +320,7 @@ lint: $(GOLANGCI_LINT) ## Lint codebase
 	$(GOLANGCI_LINT) run -v $(GOLANGCI_LINT_EXTRA_ARGS) --timeout=10m
 	cd $(APIS_DIR) && $(GOLANGCI_LINT) run -v $(GOLANGCI_LINT_EXTRA_ARGS) --timeout=10m
 	cd $(TEST_DIR) && $(GOLANGCI_LINT) run -v $(GOLANGCI_LINT_EXTRA_ARGS) --timeout=10m
+	cd $(FAKE_APISERVER_DIR) && $(GOLANGCI_LINT) run -v $(GOLANGCI_LINT_EXTRA_ARGS) --timeout=10m
 
 .PHONY: lint-fix
 lint-fix: $(GOLANGCI_LINT) ## Lint the codebase and run auto-fixers if supported by the linter
@@ -328,6 +330,7 @@ lint-full: $(GOLANGCI_LINT) ## Run slower linters to detect possible issues
 	$(GOLANGCI_LINT) run -v --fast=false --timeout=30m
 	cd $(APIS_DIR) && $(GOLANGCI_LINT) run -v --fast=false --timeout=30m
 	cd $(TEST_DIR) && $(GOLANGCI_LINT) run -v --fast=false --timeout=30m
+	cd $(FAKE_APISERVER_DIR) && $(GOLANGCI_LINT) run -v --fast=false --timeout=30m
 
 # Run manifest validation
 .PHONY: manifest-lint
@@ -349,6 +352,8 @@ modules: ## Runs go mod to ensure proper vendoring.
 	cd $(APIS_DIR) && $(GO) mod verify
 	cd $(TEST_DIR) && $(GO) mod tidy
 	cd $(TEST_DIR) && $(GO) mod verify
+	cd $(FAKE_APISERVER_DIR) && $(GO) mod tidy
+	cd $(FAKE_APISERVER_DIR) && $(GO) mod verify
 
 .PHONY: generate
 generate: ## Generate code
@@ -449,6 +454,10 @@ docker-build: ## Build the docker image for controller-manager
 docker-push: ## Push the docker image
 	docker push $(CONTROLLER_IMG)-$(ARCH):$(TAG)
 
+.PHONY: build-fake-api-server
+build-fake-api-server: ## Build the fake api server
+	cd $(FAKE_APISERVER_DIR) && docker build --build-arg ARCH=$(ARCH) -t "quay.io/metal3-io/api-server:$(ARCH)" .
+
 ## --------------------------------------
 ## Docker — All ARCH
 ## --------------------------------------

hack/fake-apiserver/.gitignore

@@ -0,0 +1 @@
+fake-apiserver

hack/fake-apiserver/Dockerfile

@@ -0,0 +1,49 @@
+# Copyright 2021 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Support FROM override
+ARG BUILD_IMAGE=docker.io/golang:1.21.9@sha256:7d0dcbe5807b1ad7272a598fbf9d7af15b5e2bed4fd6c4c2b5b3684df0b317dd
+ARG BASE_IMAGE=gcr.io/distroless/static:nonroot@sha256:9ecc53c269509f63c69a266168e4a687c7eb8c0cfd753bd8bfcaa4f58a90876f
+
+# Build the manager binary on golang image
+FROM $BUILD_IMAGE as builder
+WORKDIR /workspace
+
+# Run this with docker build --build_arg $(go env GOPROXY) to override the goproxy
+ARG goproxy=https://proxy.golang.org
+ENV GOPROXY=$goproxy
+
+# Copy the Go Modules manifests
+COPY go.mod go.sum ./
+
+# Cache deps before building and copying source so that we don't need to re-download as much
+# and so that source changes don't invalidate our downloaded layer
+RUN go mod download
+
+# Copy the sources
+COPY main.go .
+
+# Build
+ARG ARCH=amd64
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \
+    go build -a -ldflags '-extldflags "-static"' \
+    -o manager .
+
+# Copy the controller-manager into a thin image
+FROM $BASE_IMAGE
+WORKDIR /
+COPY --from=builder /workspace/manager .
+# Use uid of nonroot user (65532) because kubernetes expects numeric user when applying pod security policies
+USER 65532
+ENTRYPOINT ["/manager"]

hack/fake-apiserver/README.md

@@ -0,0 +1,173 @@
+# Fake API server
+
+Fake API server is a tool running inside a kubernetes cluster,
+and generates "fake" k8s api server endpoints on demand.
+
+These endpoints respond to any request normally responded
+by the apiserver of target clusters created in a normal CAPI
+workflow. They can, hence, be used, for e.g., to represent the clusters
+created by CAPM3 using fake hardware so that CAPI confirms that
+the cluster was provisioned successfully.
+
+## How to use
+
+You can build the `fake-api-server` image that is suitable for
+your local environment with
+
+```shell
+make build-fake-api-server
+```
+
+The result is an image with label `quay.io/metal3-io/fake-apiserver:<your-arch-name>`
+
+Alternatively, you can also build a custom image with
+
+```shell
+cd hack/fake-apiserver
+docker build -t <custom tag> .
+```
+
+For local tests, it's normally needed to load the image into the cluster.
+For e.g. with `minikube`
+
+```shell
+docker image save -o /tmp/api-server.tar <image-name>
+minikube image load /tmp/api-server.tar
+```
+
+Now you can deploy this container to the cluster, for e.g. with a deployment
+
+```api-server-deployment.yaml
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: metal3-fake-api-server
+  namespace: default
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: capim
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: capim
+    spec:
+      containers:
+        - image: quay.io/metal3-io/api-server:amd64
+          imagePullPolicy: IfNotPresent
+          name: capim
+          env:
+          - name: POD_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.podIP
+          name: apiserver
+```
+
+```shell
+kubectl apply -f api-server-deployment.yaml
+```
+
+After building the container image and deploy it to a kubernetes cluster,
+you need to create a tunnel to send request to it and get response, by using
+a LoadBalancer, or a simple port-forward
+
+```shell
+api_server_name=$(kubectl get pods -l app=capim -o jsonpath="{.items[0].metadata.name}")
+kubectl port-forward pod/${api_server_name} ${api_server_port}:3333 2>/dev/null&
+```
+
+Now, you can generate a fake API server endpoint by sending
+a GET request to the fake API server. But first, let's generate some needed certificates
+
+```shell
+openssl req -x509 -subj "/CN=Kubernetes API" -new -newkey rsa:2048 \
+  -nodes -keyout "/tmp/ca.key" -sha256 -days 3650 -out "/tmp/ca.crt"
+openssl req -x509 -subj "/CN=ETCD CA" -new -newkey rsa:2048 \
+  -nodes -keyout "/tmp/etcd.key" -sha256 -days 3650 -out "/tmp/etcd.crt"
+```
+
+```shell
+caKeyEncoded=$(cat /tmp/ca.key | base64 -w 0)
+caCertEncoded=$(cat /tmp/ca.crt | base64 -w 0)
+etcdKeyEncoded=$(cat /tmp/etcd.key | base64 -w 0)
+etcdCertEncoded=$(cat /tmp/etcd.crt | base64 -w 0)
+namespace="metal3"
+cluster_name="test_cluster"
+
+cluster_endpoint=$(curl localhost:${api_server_port}/register?resource=$namespace/$cluster_name&caKey=$caKeyEncoded&caCert=$caCertEncoded&etcdKey=$etcdKeyEncoded&etcdCert=$etcdCertEncoded")
+```
+
+The fake API server will return a response with the ip and port of the newly
+generated api server. These information can be fed to a CAPI infrastructure provider
+(for e.g. CAPM3) to create a cluster. Notice that you need to manually create
+the `ca-secret` and `etcd-secret` of the new cluster, so that it matches the certs
+used by the api server.
+
+```shell
+host=$(echo ${cluster_endpoints} | jq -r ".Host")
+port=$(echo ${cluster_endpoints} | jq -r ".Port")
+
+  cat <<EOF > "/tmp/${cluster}-ca-secrets.yaml"
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ${cluster}
+  name: ${cluster}-ca
+  namespace: ${namespace}
+type: kubernetes.io/tls
+data:
+  tls.crt: ${caCertEncoded}
+  tls.key: ${caKeyEncoded}
+EOF
+
+  kubectl -n ${namespace} apply -f /tmp/${cluster}-ca-secrets.yaml
+
+  cat <<EOF > "/tmp/${cluster}-etcd-secrets.yaml"
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ${cluster}
+  name: ${cluster}-etcd
+  namespace: ${namespace}
+type: kubernetes.io/tls
+data:
+  tls.crt: ${etcdCertEncoded}
+  tls.key: ${etcdKeyEncoded}
+EOF
+
+  kubectl -n ${namespace} apply -f /tmp/${cluster}-etcd-secrets.yaml
+
+  # Generate metal3 cluster
+  export CLUSTER_APIENDPOINT_HOST="${host}"
+  export CLUSTER_APIENDPOINT_PORT="${port}"
+  echo "Generating cluster ${cluster} with clusterctl"
+  clusterctl generate cluster "${cluster}" \
+    --from "${CLUSTER_TEMPLATE}" \
+    --target-namespace "${namespace}" > /tmp/${cluster}-cluster.yaml
+  kubectl apply -f /tmp/${cluster}-cluster.yaml
+```
+
+After the cluster is created, CAPI will expect that information like node name
+and provider ID is registered in the API server. Since our API server doesn't
+live inside the node, we will need to feed the info to it, by sending a
+GET request to `/updateNode` endpoint:
+
+```shell
+retry_curl "localhost:${api_server_port}/updateNode?resource=${namespace}/${cluster}&nodeName=${machine}&providerID=${providerID}"
+```
+
+## Acknowledgements
+
+This was developed thanks to the implementation of
+[Cluster API Provider In Memory (CAPIM)](https://github.com/kubernetes-sigs/cluster-api/tree/main/test/infrastructure/inmemory).
+
+**NOTE:**:
+This is intended for development environments only.
+Do **NOT** use it in production.

hack/fake-apiserver/go.mod

@@ -0,0 +1,107 @@
+module github.com/metal3-io/cluster-api-provider-metal3/hack/fake-apiserver
+
+go 1.21
+
+require (
+	k8s.io/api v0.29.3
+	k8s.io/apimachinery v0.29.3
+	k8s.io/client-go v0.29.3
+	k8s.io/klog/v2 v2.120.1
+	sigs.k8s.io/cluster-api v1.7.1
+	sigs.k8s.io/cluster-api/test v1.7.1
+	sigs.k8s.io/controller-runtime v0.17.3
+)
+
+require (
+	github.com/NYTimes/gziphandler v1.1.1 // indirect
+	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect
+	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/blang/semver/v4 v4.0.0 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/coreos/go-semver v0.3.1 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/emicklei/go-restful/v3 v3.12.0 // indirect
+	github.com/evanphx/json-patch v5.7.0+incompatible // indirect
+	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/cel-go v0.17.7 // indirect
+	github.com/google/gnostic-models v0.6.8 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/uuid v1.4.0 // indirect
+	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
+	github.com/imdario/mergo v0.3.13 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
+	github.com/moby/spdystream v0.2.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/prometheus/client_golang v1.18.0 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.45.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/spf13/cobra v1.8.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stoewer/go-strcase v1.2.0 // indirect
+	go.etcd.io/etcd/api/v3 v3.5.13 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.13 // indirect
+	go.etcd.io/etcd/client/v3 v3.5.13 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.0 // indirect
+	go.opentelemetry.io/otel v1.22.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 // indirect
+	go.opentelemetry.io/otel/metric v1.22.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.22.0 // indirect
+	go.opentelemetry.io/otel/trace v1.22.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.26.0 // indirect
+	golang.org/x/crypto v0.21.0 // indirect
+	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
+	golang.org/x/net v0.23.0 // indirect
+	golang.org/x/oauth2 v0.18.0 // indirect
+	golang.org/x/sync v0.6.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/term v0.18.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
+	google.golang.org/appengine v1.6.8 // indirect
+	google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect
+	google.golang.org/grpc v1.60.1 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/apiextensions-apiserver v0.29.3 // indirect
+	k8s.io/apiserver v0.29.3 // indirect
+	k8s.io/component-base v0.29.3 // indirect
+	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
+	k8s.io/utils v0.0.0-20231127182322-b307cd553661 // indirect
+	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
+)