Merge pull request #413 from buildkite/default-job-metadata

Configurable default job metadata

README.md

@@ -381,6 +381,40 @@ default-checkout-params:
 ...
 ```
 
+### Default job metadata
+agent-stack-k8s can automatically add labels and annotations to the Kubernetes jobs it creates.
+Default labels and annotations can be set in `values.yaml` with `default-metadata`, e.g.:
+
+```yaml
+# config.yaml / values.yaml
+...
+default-metadata:
+  annotations:
+    imageregistry: "https://hub.docker.com/"
+    mycoolannotation: llamas
+  labels:
+    argocd.argoproj.io/tracking-id: example-id-here
+    mycoollabel: alpacas
+```
+
+Similarly they can be set for each step in a pipeline individually using the kubernetes plugin,
+e.g.:
+
+```yaml
+# pipeline.yaml
+...
+  plugins:
+    - kubernetes:
+        metadata:
+          annotations:
+            imageregistry: "https://hub.docker.com/"
+            mycoolannotation: llamas
+          labels:
+            argocd.argoproj.io/tracking-id: example-id-here
+            mycoollabel: alpacas
+```
+
+
 ### Pod Spec Patch
 Rather than defining the entire Pod Spec in a step, there is the option to define a [strategic merge patch](https://kubernetes.io/docs/tasks/manage-kubernetes-objects/update-api-object-kubectl-patch/) in the controller.
 Agent Stack K8s will first generate a K8s Job with a PodSpec from a Buildkite Job and then apply the patch in the controller.

charts/agent-stack-k8s/values.schema.json

@@ -455,6 +455,27 @@
             }
           }
         },
+        "default-metadata": {
+          "type": "object",
+          "default": null,
+          "title": "Default metadata (labels, annotations) for jobs",
+          "properties": {
+            "labels": {
+              "type": "object",
+              "additionalProperties": {
+                "type": "string"
+              },
+              "title": "Default labels to apply to jobs. Note the constraints on label names: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set"
+            },
+            "annotations": {
+              "type": "object",
+              "additionalProperties": {
+                "type": "string"
+              },
+              "title": "Default annotations to apply to jobs. Note the constraints on annotation names: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set"
+            }
+          }
+        },
         "pod-spec-patch": {
           "$ref": "https://kubernetesjsonschema.dev/master/_definitions.json#/definitions/io.k8s.api.core.v1.PodSpec"
         }

cmd/controller/controller_test.go

@@ -72,6 +72,14 @@ func TestReadAndParseConfig(t *testing.T) {
 				},
 			}},
 		},
+		DefaultMetadata: config.Metadata{
+			Annotations: map[string]string{
+				"imageregistry": "https://hub.docker.com/",
+			},
+			Labels: map[string]string{
+				"argocd.argoproj.io/tracking-id": "example-id-here",
+			},
+		},
 		PodSpecPatch: &corev1.PodSpec{
 			ServiceAccountName:           "buildkite-agent-sa",
 			AutomountServiceAccountToken: ptr(true),

examples/config.yaml

@@ -56,6 +56,17 @@ default-sidecar-params:
       configMapRef:
         name: logging-config
 
+# Applied to all spawned jobs.
+default-metadata:
+  annotations:
+    # Note the constraints on annotation names:
+    # https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set
+    imageregistry: "https://hub.docker.com/"
+  labels:
+    # Note the constraints on label names:
+    # https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
+    argocd.argoproj.io/tracking-id: example-id-here
+
 # This will be applied to the job's podSpec as a strategic merge patch
 # See https://kubernetes.io/docs/tasks/manage-kubernetes-objects/update-api-object-kubectl-patch
 pod-spec-patch:

internal/controller/config/config.go

@@ -47,10 +47,11 @@ type Config struct {
 	ImagePullBackOffGracePeriod  time.Duration   `json:"image-pull-backoff-grace-period"  validate:"omitempty"`
 	JobCancelCheckerPollInterval time.Duration   `json:"job-cancel-checker-poll-interval" validate:"omitempty"`
 
-	AgentConfig           *AgentConfig    `json:"agent-config" validate:"omitempty"`
+	AgentConfig           *AgentConfig    `json:"agent-config"            validate:"omitempty"`
 	DefaultCheckoutParams *CheckoutParams `json:"default-checkout-params" validate:"omitempty"`
 	DefaultCommandParams  *CommandParams  `json:"default-command-params"  validate:"omitempty"`
 	DefaultSidecarParams  *SidecarParams  `json:"default-sidecar-params"  validate:"omitempty"`
+	DefaultMetadata       Metadata        `json:"default-metadata"        validate:"omitempty"`
 
 	// ProhibitKubernetesPlugin can be used to prevent alterations to the pod
 	// from the job (the kubernetes "plugin" in pipeline.yml). If enabled,
@@ -76,6 +77,9 @@ func (c Config) MarshalLogObject(enc zapcore.ObjectEncoder) error {
 	enc.AddInt("max-in-flight", c.MaxInFlight)
 	enc.AddString("namespace", c.Namespace)
 	enc.AddString("org", c.Org)
+	if err := enc.AddArray("tags", c.Tags); err != nil {
+		return err
+	}
 	enc.AddString("profiler-address", c.ProfilerAddress)
 	enc.AddString("cluster-uuid", c.ClusterUUID)
 	enc.AddBool("prohibit-kubernetes-plugin", c.ProhibitKubernetesPlugin)
@@ -99,7 +103,10 @@ func (c Config) MarshalLogObject(enc zapcore.ObjectEncoder) error {
 	if err := enc.AddReflected("default-sidecar-params", c.DefaultSidecarParams); err != nil {
 		return err
 	}
-	return enc.AddArray("tags", c.Tags)
+	if err := enc.AddReflected("default-metadata", c.DefaultMetadata); err != nil {
+		return err
+	}
+	return nil
 }
 
 // Helpers for applying configs / params to container env.

internal/controller/config/metadata.go

@@ -0,0 +1,8 @@
+package config
+
+// Metadata contains k8s job metadata to apply when creating pods. It can be
+// set as a default within the config, or per step using the kubernetes plugin.
+type Metadata struct {
+	Annotations map[string]string
+	Labels      map[string]string
+}

internal/controller/controller.go

@@ -61,6 +61,7 @@ func Run(
 		DefaultCheckoutParams:  cfg.DefaultCheckoutParams,
 		DefaultCommandParams:   cfg.DefaultCommandParams,
 		DefaultSidecarParams:   cfg.DefaultSidecarParams,
+		DefaultMetadata:        cfg.DefaultMetadata,
 		PodSpecPatch:           cfg.PodSpecPatch,
 		ProhibitK8sPlugin:      cfg.ProhibitKubernetesPlugin,
 	})

internal/controller/scheduler/scheduler.go

@@ -50,6 +50,7 @@ type Config struct {
 	DefaultCheckoutParams  *config.CheckoutParams
 	DefaultCommandParams   *config.CommandParams
 	DefaultSidecarParams   *config.SidecarParams
+	DefaultMetadata        config.Metadata
 	PodSpecPatch           *corev1.PodSpec
 	ProhibitK8sPlugin      bool
 }
@@ -67,18 +68,13 @@ type KubernetesPlugin struct {
 	PodSpecPatch      *corev1.PodSpec        `json:"podSpecPatch,omitempty"`
 	GitEnvFrom        []corev1.EnvFromSource `json:"gitEnvFrom,omitempty"`
 	Sidecars          []corev1.Container     `json:"sidecars,omitempty"`
-	Metadata          Metadata               `json:"metadata,omitempty"`
+	Metadata          config.Metadata        `json:"metadata,omitempty"`
 	ExtraVolumeMounts []corev1.VolumeMount   `json:"extraVolumeMounts,omitempty"`
 	CheckoutParams    *config.CheckoutParams `json:"checkout,omitempty"`
 	CommandParams     *config.CommandParams  `json:"commandParams,omitempty"`
 	SidecarParams     *config.SidecarParams  `json:"sidecarParams,omitempty"`
 }
 
-type Metadata struct {
-	Annotations map[string]string
-	Labels      map[string]string
-}
-
 type worker struct {
 	cfg    Config
 	client kubernetes.Interface
@@ -212,6 +208,8 @@ func (w *worker) Build(podSpec *corev1.PodSpec, skipCheckout bool, inputs buildI
 		},
 	}
 
+	maps.Copy(kjob.Labels, w.cfg.DefaultMetadata.Labels)
+	maps.Copy(kjob.Annotations, w.cfg.DefaultMetadata.Annotations)
 	if inputs.k8sPlugin != nil {
 		maps.Copy(kjob.Labels, inputs.k8sPlugin.Metadata.Labels)
 		maps.Copy(kjob.Annotations, inputs.k8sPlugin.Metadata.Annotations)