node selector template (#95)

* refactor e2e struct

* Revert "refactor e2e struct"

This reverts commit 9271224.

* simplify lifecycle test wrapper

* add expected status over lifetime

* add new taint and label for test minikube

* add nodeselector and toleration to status

* use expected status from lifetime status input

* test with different taints and labels

* write a little loop to check tolerations

* update helper script and makefile with new tolerations

* use nodeselector template

* merge template and given

* add gvisor test

* run copied test

* rename tests

* naming

* propogate nodeselector and tolerations to the helm chart

* update status update for nodeselector and tolerations

* ensure test namespaces are deleted

.github/kubectl-patch/nodeselector-pod-toleration.yaml

@@ -1,8 +1,8 @@
 spec:
   tolerations:
-    - key: "testkey"
+    - key: "sandbox.gke.io/runtime"
       operator: "Equal"
-      value: "testvalue"
+      value: "gvisor"
       effect: "NoSchedule"
   nodeSelector:
-    testkey: "testvalue"
+    sandbox.gke.io/runtime: "gvisor"

.github/kubectl-patch/nodeselector-toleration.yaml

@@ -1,10 +1,10 @@
 spec:
   template:
     spec:
-      nodeSelector:
-        testkey: "testvalue"
       tolerations:
-        - key: "testkey"
+        - key: "sandbox.gke.io/runtime"
           operator: "Equal"
-          value: "testvalue"
+          value: "gvisor"
           effect: "NoSchedule"
+      nodeSelector:
+        sandbox.gke.io/runtime: "gvisor"

Makefile

@@ -117,7 +117,7 @@ install-fluxcd-controllers: install-flux-prereq ## Install the fluxcd controller
 
 .PHONE: install-fluxcd-controllers-with-toleration
 install-fluxcd-controllers-with-toleration: install-flux-prereq ## Install the fluxcd controllers with toleration.
-	flux install --namespace=flux-system --components="source-controller,helm-controller" --toleration-keys="testkey" --network-policy=false --insecure-skip-tls-verify
+	flux install --namespace=flux-system --components="source-controller,helm-controller" --toleration-keys="sandbox.gke.io/runtime" --network-policy=false --insecure-skip-tls-verify
 
 .PHONY: start-test-k3d
 start-test-k3d: ## Start a k3d cluster for testing.
@@ -137,10 +137,10 @@ stop-test-minikube: ## Stop the minikube cluster for testing.
 
 .PHONY: start-test-minikube-tainted
 start-test-minikube-tainted: ## Start a minikube cluster with a tainted node for testing.
-	minikube start --addons default-storageclass,storage-provisioner,hostpat --driver=docker
+	minikube start --addons default-storageclass,storage-provisioner --driver=docker
 	sh ./hack/minikube-patch-pod-tolerations.sh
-	kubectl taint nodes minikube testkey=testvalue:NoSchedule || true
-	kubectl label nodes minikube testkey=testvalue || true
+	kubectl taint nodes minikube sandbox.gke.io/runtime=gvisor:NoSchedule || true
+	kubectl label nodes minikube sandbox.gke.io/runtime=gvisor || true
 	$(MAKE) install-fluxcd-controllers-with-toleration
 	sh ./hack/minikube-patch-workload-tolerations.sh
 

chart/templates/uffizziclusters.uffizzi.com_customresourcedefinition.yaml

@@ -152,6 +152,8 @@ spec:
                 additionalProperties:
                   type: string
                 type: object
+              nodeSelectorTemplate:
+                type: string
               resourceQuota:
                 description: UffizziClusterResourceQuota defines the resource quota which defines the quota of resources a namespace has access to
                 properties:
@@ -325,6 +327,32 @@ spec:
               lastAwakeTime:
                 format: date-time
                 type: string
+              nodeSelector:
+                additionalProperties:
+                  type: string
+                type: object
+              tolerations:
+                items:
+                  description: The pod this Toleration is attached to tolerates any taint that matches the triple <key,value,effect> using the matching operator <operator>.
+                  properties:
+                    effect:
+                      description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
+                      type: string
+                    key:
+                      description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys.
+                      type: string
+                    operator:
+                      description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.
+                      type: string
+                    tolerationSeconds:
+                      description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system.
+                      format: int64
+                      type: integer
+                    value:
+                      description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.
+                      type: string
+                  type: object
+                type: array
             type: object
         type: object
     served: true

hack/minikube-patch-pod-tolerations.sh

@@ -1,5 +1,5 @@
 kubectl get pods --all-namespaces -o jsonpath="{range .items[*]}{.metadata.namespace}{' '}{.metadata.name}{'\n'}{end}" | while read -r line; do
   namespace=$(echo "$line" | cut -d' ' -f1)
   pod=$(echo "$line" | cut -d' ' -f2)
-  kubectl patch pod "$pod" -n "$namespace" --type='json' -p='[{"op": "add", "path": "/spec/tolerations", "value": [{"key": "testkey", "operator": "Equal", "value": "testvalue", "effect": "NoSchedule"}]}]'
+  kubectl patch pod "$pod" -n "$namespace" --type='json' -p='[{"op": "add", "path": "/spec/tolerations", "value": [{"key": "sandbox.gke.io/runtime", "operator": "Equal", "value": "gvisor", "effect": "NoSchedule"}]}]' || true
 done

src/api/v1alpha1/uffizzicluster_types.go

@@ -151,17 +151,18 @@ type UffizziClusterStorage struct {
 type UffizziClusterSpec struct {
 	//+kubebuilder:default:="k3s"
 	//+kubebuilder:validation:Enum=k3s;k8s
-	Distro        string                       `json:"distro,omitempty"`
-	NodeSelector  map[string]string            `json:"nodeSelector,omitempty"`
-	Toleration    []v1.Toleration              `json:"tolerations,omitempty"`
-	APIServer     UffizziClusterAPIServer      `json:"apiServer,omitempty"`
-	Ingress       UffizziClusterIngress        `json:"ingress,omitempty"`
-	Helm          []HelmChart                  `json:"helm,omitempty"`
-	Manifests     *string                      `json:"manifests,omitempty"`
-	ResourceQuota *UffizziClusterResourceQuota `json:"resourceQuota,omitempty"`
-	LimitRange    *UffizziClusterLimitRange    `json:"limitRange,omitempty"`
-	Sleep         bool                         `json:"sleep,omitempty"`
-	Storage       *UffizziClusterStorage       `json:"storage,omitempty"`
+	Distro               string                       `json:"distro,omitempty"`
+	NodeSelectorTemplate string                       `json:"nodeSelectorTemplate,omitempty"`
+	NodeSelector         map[string]string            `json:"nodeSelector,omitempty"`
+	Toleration           []v1.Toleration              `json:"tolerations,omitempty"`
+	APIServer            UffizziClusterAPIServer      `json:"apiServer,omitempty"`
+	Ingress              UffizziClusterIngress        `json:"ingress,omitempty"`
+	Helm                 []HelmChart                  `json:"helm,omitempty"`
+	Manifests            *string                      `json:"manifests,omitempty"`
+	ResourceQuota        *UffizziClusterResourceQuota `json:"resourceQuota,omitempty"`
+	LimitRange           *UffizziClusterLimitRange    `json:"limitRange,omitempty"`
+	Sleep                bool                         `json:"sleep,omitempty"`
+	Storage              *UffizziClusterStorage       `json:"storage,omitempty"`
 	//+kubebuilder:default:="sqlite"
 	//+kubebuilder:validation:Enum=etcd;sqlite
 	ExternalDatastore string `json:"externalDatastore,omitempty"`
@@ -176,6 +177,8 @@ type UffizziClusterStatus struct {
 	LastAppliedConfiguration   *string            `json:"lastAppliedConfiguration,omitempty"`
 	LastAppliedHelmReleaseSpec *string            `json:"lastAppliedHelmReleaseSpec,omitempty"`
 	LastAwakeTime              metav1.Time        `json:"lastAwakeTime,omitempty"`
+	Tolerations                []v1.Toleration    `json:"tolerations,omitempty"`
+	NodeSelector               map[string]string  `json:"nodeSelector,omitempty"`
 }
 
 // VClusterKubeConfig is the KubeConfig SecretReference of the related VCluster

src/config/crd/bases/uffizzi.com_uffizziclusters.yaml

@@ -158,6 +158,8 @@ spec:
                 additionalProperties:
                   type: string
                 type: object
+              nodeSelectorTemplate:
+                type: string
               resourceQuota:
                 description: UffizziClusterResourceQuota defines the resource quota
                   which defines the quota of resources a namespace has access to
@@ -380,6 +382,49 @@ spec:
               lastAwakeTime:
                 format: date-time
                 type: string
+              nodeSelector:
+                additionalProperties:
+                  type: string
+                type: object
+              tolerations:
+                items:
+                  description: The pod this Toleration is attached to tolerates any
+                    taint that matches the triple <key,value,effect> using the matching
+                    operator <operator>.
+                  properties:
+                    effect:
+                      description: Effect indicates the taint effect to match. Empty
+                        means match all taint effects. When specified, allowed values
+                        are NoSchedule, PreferNoSchedule and NoExecute.
+                      type: string
+                    key:
+                      description: Key is the taint key that the toleration applies
+                        to. Empty means match all taint keys. If the key is empty,
+                        operator must be Exists; this combination means to match all
+                        values and all keys.
+                      type: string
+                    operator:
+                      description: Operator represents a key's relationship to the
+                        value. Valid operators are Exists and Equal. Defaults to Equal.
+                        Exists is equivalent to wildcard for value, so that a pod
+                        can tolerate all taints of a particular category.
+                      type: string
+                    tolerationSeconds:
+                      description: TolerationSeconds represents the period of time
+                        the toleration (which must be of effect NoExecute, otherwise
+                        this field is ignored) tolerates the taint. By default, it
+                        is not set, which means tolerate the taint forever (do not
+                        evict). Zero and negative values will be treated as 0 (evict
+                        immediately) by the system.
+                      format: int64
+                      type: integer
+                    value:
+                      description: Value is the taint value the toleration matches
+                        to. If the operator is Exists, the value should be empty,
+                        otherwise just a regular string.
+                      type: string
+                  type: object
+                type: array
             type: object
         type: object
     served: true

src/controllers/uffizzicluster/helm.go

@@ -27,6 +27,8 @@ func (r *UffizziClusterReconciler) deleteLoftHelmRepo(ctx context.Context, req c
 }
 
 func (r *UffizziClusterReconciler) upsertVClusterK3SHelmRelease(update bool, ctx context.Context, uCluster *uclusteruffizzicomv1alpha1.UffizziCluster) (*fluxhelmv2beta1.HelmRelease, error) {
+	patch := client.MergeFrom(uCluster.DeepCopy())
+
 	vclusterK3sHelmValues, helmReleaseName := vcluster.BuildK3SHelmValues(uCluster)
 	helmValuesJSONObj, err := build.HelmValuesToJSON(vclusterK3sHelmValues)
 	if err != nil {
@@ -77,7 +79,6 @@ func (r *UffizziClusterReconciler) upsertVClusterK3SHelmRelease(update bool, ctx
 			return nil, errors.Wrap(err, "failed to create HelmRelease")
 		}
 		uCluster.Status.LastAppliedHelmReleaseSpec = &newHelmReleaseSpec
-		patch := client.MergeFrom(uCluster.DeepCopy())
 		if err := r.Status().Patch(ctx, uCluster, patch); err != nil {
 			return nil, errors.Wrap(err, "Failed to update the default UffizziCluster lastAppliedHelmReleaseSpec")
 		}

src/pkg/helm/build/vcluster/build.go

@@ -10,11 +10,32 @@ import (
 )
 
 func BuildK3SHelmValues(uCluster *v1alpha1.UffizziCluster) (vcluster.K3S, string) {
-	helmReleaseName, vclusterIngressHostname, outKubeConfigServerArgValue := configStrings(uCluster)
+	var (
+		helmReleaseName, vclusterIngressHostname, outKubeConfigServerArgValue = configStrings(uCluster)
+		nodeSelector                                                          = vcluster.NodeSelector{}
+		tolerations                                                           = []v1.Toleration{}
+	)
+
+	if uCluster.Spec.NodeSelectorTemplate == constants.GVISOR {
+		nodeSelector = vcluster.GvisorNodeSelector
+		tolerations = []v1.Toleration{vcluster.GvisorToleration.ToV1()}
+	}
+
+	if len(uCluster.Spec.NodeSelector) > 0 {
+		// merge nodeSelector and uCluster.Spec.NodeSelector
+		for k, v := range uCluster.Spec.NodeSelector {
+			map[string]string(nodeSelector)[k] = v
+		}
+	}
+
+	if len(uCluster.Spec.Toleration) > 0 {
+		// merge tolerations and uCluster.Spec.Toleration
+		tolerations = append(tolerations, uCluster.Spec.Toleration...)
+	}
 
 	vclusterK3sHelmValues := vcluster.K3S{
 		VCluster: k3SAPIServer(uCluster),
-		Common:   common(helmReleaseName, vclusterIngressHostname, uCluster.Spec.NodeSelector, uCluster.Spec.Toleration),
+		Common:   common(helmReleaseName, vclusterIngressHostname, nodeSelector, tolerations),
 	}
 
 	if uCluster.Spec.ExternalDatastore == constants.ETCD {
@@ -94,13 +115,21 @@ func BuildK3SHelmValues(uCluster *v1alpha1.UffizziCluster) (vcluster.K3S, string
 		)
 	}
 
-	for _, t := range uCluster.Spec.Toleration {
+	for _, t := range tolerations {
 		vclusterK3sHelmValues.Syncer.ExtraArgs = append(vclusterK3sHelmValues.Syncer.ExtraArgs, "--enforce-toleration="+vcluster.Toleration(t).Notation())
+		if uCluster.Status.Tolerations == nil {
+			uCluster.Status.Tolerations = []v1.Toleration{}
+		}
+		uCluster.Status.Tolerations = append(uCluster.Status.Tolerations, t)
 	}
 
-	if len(uCluster.Spec.NodeSelector) > 0 {
-		for k, v := range uCluster.Spec.NodeSelector {
+	if len(nodeSelector) > 0 {
+		for k, v := range nodeSelector {
 			vclusterK3sHelmValues.Syncer.ExtraArgs = append(vclusterK3sHelmValues.Syncer.ExtraArgs, "--node-selector="+k+"="+v)
+			if uCluster.Status.NodeSelector == nil {
+				uCluster.Status.NodeSelector = make(map[string]string)
+			}
+			uCluster.Status.NodeSelector[k] = v
 		}
 		vclusterK3sHelmValues.Syncer.ExtraArgs = append(vclusterK3sHelmValues.Syncer.ExtraArgs, "--enforce-node-selector")
 	}
@@ -131,11 +160,31 @@ func BuildK3SHelmValues(uCluster *v1alpha1.UffizziCluster) (vcluster.K3S, string
 }
 
 func BuildK8SHelmValues(uCluster *v1alpha1.UffizziCluster) (vcluster.K8S, string) {
-	helmReleaseName, vclusterIngressHostname, outKubeConfigServerArgValue := configStrings(uCluster)
+	var (
+		helmReleaseName, vclusterIngressHostname, outKubeConfigServerArgValue = configStrings(uCluster)
+		nodeSelector                                                          = vcluster.NodeSelector{}
+		tolerations                                                           = []v1.Toleration{}
+	)
+	if uCluster.Spec.NodeSelectorTemplate == constants.GVISOR {
+		nodeSelector = vcluster.GvisorNodeSelector
+		tolerations = []v1.Toleration{vcluster.GvisorToleration.ToV1()}
+	}
+
+	if len(uCluster.Spec.NodeSelector) > 0 {
+		// merge nodeSelector and uCluster.Spec.NodeSelector
+		for k, v := range uCluster.Spec.NodeSelector {
+			map[string]string(nodeSelector)[k] = v
+		}
+	}
+
+	if len(uCluster.Spec.Toleration) > 0 {
+		// merge tolerations and uCluster.Spec.Toleration
+		tolerations = append(tolerations, uCluster.Spec.Toleration...)
+	}
 
 	vclusterHelmValues := vcluster.K8S{
 		APIServer: k8SAPIServer(),
-		Common:    common(helmReleaseName, vclusterIngressHostname, uCluster.Spec.NodeSelector, uCluster.Spec.Toleration),
+		Common:    common(helmReleaseName, vclusterIngressHostname, nodeSelector, tolerations),
 	}
 
 	if uCluster.Spec.APIServer.Image != "" {
@@ -210,6 +259,25 @@ func BuildK8SHelmValues(uCluster *v1alpha1.UffizziCluster) (vcluster.K8S, string
 		)
 	}
 
+	for _, t := range tolerations {
+		vclusterHelmValues.Syncer.ExtraArgs = append(vclusterHelmValues.Syncer.ExtraArgs, "--enforce-toleration="+vcluster.Toleration(t).Notation())
+		if uCluster.Status.Tolerations == nil {
+			uCluster.Status.Tolerations = []v1.Toleration{}
+		}
+		uCluster.Status.Tolerations = append(uCluster.Status.Tolerations, t)
+	}
+
+	if len(nodeSelector) > 0 {
+		for k, v := range nodeSelector {
+			vclusterHelmValues.Syncer.ExtraArgs = append(vclusterHelmValues.Syncer.ExtraArgs, "--node-selector="+k+"="+v)
+			if uCluster.Status.NodeSelector == nil {
+				uCluster.Status.NodeSelector = make(map[string]string)
+			}
+			uCluster.Status.NodeSelector[k] = v
+		}
+		vclusterHelmValues.Syncer.ExtraArgs = append(vclusterHelmValues.Syncer.ExtraArgs, "--enforce-node-selector")
+	}
+
 	if len(uCluster.Spec.Helm) > 0 {
 		vclusterHelmValues.Init.Helm = uCluster.Spec.Helm
 	}

src/pkg/helm/types/vcluster/tolerations.go

@@ -0,0 +1,36 @@
+package vcluster
+
+import v1 "k8s.io/api/core/v1"
+
+type Toleration v1.Toleration
+type NodeSelector map[string]string
+
+func (t Toleration) Notation() string {
+	if t.Value == "" {
+		return t.Key + ":" + string(t.Effect)
+	}
+	return t.Key + "=" + t.Value + ":" + string(t.Effect)
+}
+
+func (t Toleration) ToV1() v1.Toleration {
+	return v1.Toleration(t)
+}
+
+func (n NodeSelector) Notation() string {
+	for k, v := range n {
+		return k + "=" + v
+	}
+	return ""
+}
+
+var (
+	GvisorToleration = Toleration{
+		Key:      "sandbox.gke.io/runtime",
+		Operator: v1.TolerationOpEqual,
+		Value:    "gvisor",
+		Effect:   v1.TaintEffectNoSchedule,
+	}
+	GvisorNodeSelector = NodeSelector{
+		"sandbox.gke.io/runtime": "gvisor",
+	}
+)