From b9ede4a01821ff2cf26e8fa4271877a34e866b08 Mon Sep 17 00:00:00 2001 From: 42Atomys Date: Wed, 7 Feb 2024 22:21:28 +0100 Subject: [PATCH 1/6] feat: allow git-credentials to be set in agent and plugin --- examples/config.yaml | 8 +++- internal/controller/config/config.go | 24 +++++----- internal/controller/controller.go | 9 ++-- internal/controller/scheduler/scheduler.go | 54 +++++++++++++++------- 4 files changed, 62 insertions(+), 33 deletions(-) diff --git a/examples/config.yaml b/examples/config.yaml index aeb12c11..1c52b565 100644 --- a/examples/config.yaml +++ b/examples/config.yaml @@ -10,5 +10,9 @@ org: my-buildkite-org # the UUID may be found in the cluster settings cluster-uuid: beefcafe-abbe-baba-abba-deedcedecade tags: -- queue=my-queue -- priority=high + - queue=my-queue + - priority=high +# git-credentials-secret are the secret with the git credentials +# configured on the agent with docker-ssh-config +# @see https://github.com/buildkite/docker-ssh-env-config +git-credentials-secret: buildkite-ssh-github-creds diff --git a/internal/controller/config/config.go b/internal/controller/config/config.go index 081acb03..8691ed14 100644 --- a/internal/controller/config/config.go +++ b/internal/controller/config/config.go @@ -15,17 +15,18 @@ const ( ) type Config struct { - Debug bool `mapstructure:"debug"` - AgentTokenSecret string `mapstructure:"agent-token-secret" validate:"required"` - BuildkiteToken string `mapstructure:"buildkite-token" validate:"required"` - Image string `mapstructure:"image" validate:"required"` - JobTTL time.Duration `mapstructure:"job-ttl"` - MaxInFlight int `mapstructure:"max-in-flight" validate:"min=0"` - Namespace string `mapstructure:"namespace" validate:"required"` - Org string `mapstructure:"org" validate:"required"` - Tags stringSlice `mapstructure:"tags" validate:"min=1"` - ProfilerAddress string `mapstructure:"profiler-address" validate:"omitempty,hostname_port"` - ClusterUUID string `mapstructure:"cluster-uuid" validate:"omitempty"` + Debug bool `mapstructure:"debug"` + AgentTokenSecret string `mapstructure:"agent-token-secret" validate:"required"` + BuildkiteToken string `mapstructure:"buildkite-token" validate:"required"` + Image string `mapstructure:"image" validate:"required"` + JobTTL time.Duration `mapstructure:"job-ttl"` + MaxInFlight int `mapstructure:"max-in-flight" validate:"min=0"` + Namespace string `mapstructure:"namespace" validate:"required"` + Org string `mapstructure:"org" validate:"required"` + Tags stringSlice `mapstructure:"tags" validate:"min=1"` + ProfilerAddress string `mapstructure:"profiler-address" validate:"omitempty,hostname_port"` + ClusterUUID string `mapstructure:"cluster-uuid" validate:"omitempty"` + GitCredentialsSecret string `mapstructure:"git-credentials-secret" validate:"omitempty"` } type stringSlice []string @@ -39,6 +40,7 @@ func (s stringSlice) MarshalLogArray(enc zapcore.ArrayEncoder) error { func (c Config) MarshalLogObject(enc zapcore.ObjectEncoder) error { enc.AddString("agent-token-secret", c.AgentTokenSecret) + enc.AddString("git-credentials-secret", c.GitCredentialsSecret) enc.AddBool("debug", c.Debug) enc.AddString("image", c.Image) enc.AddDuration("job-ttl", c.JobTTL) diff --git a/internal/controller/controller.go b/internal/controller/controller.go index e5a14def..9f60f305 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -49,10 +49,11 @@ func Run( } sched := scheduler.New(logger.Named("scheduler"), k8sClient, scheduler.Config{ - Namespace: cfg.Namespace, - Image: cfg.Image, - AgentToken: cfg.AgentTokenSecret, - JobTTL: cfg.JobTTL, + Namespace: cfg.Namespace, + Image: cfg.Image, + AgentToken: cfg.AgentTokenSecret, + JobTTL: cfg.JobTTL, + GitCredentialsSecret: cfg.GitCredentialsSecret, }) limiter := scheduler.NewLimiter(logger.Named("limiter"), sched, cfg.MaxInFlight) diff --git a/internal/controller/scheduler/scheduler.go b/internal/controller/scheduler/scheduler.go index 0198edc7..03194029 100644 --- a/internal/controller/scheduler/scheduler.go +++ b/internal/controller/scheduler/scheduler.go @@ -29,10 +29,11 @@ const ( ) type Config struct { - Namespace string - Image string - AgentToken string - JobTTL time.Duration + Namespace string + Image string + AgentToken string + JobTTL time.Duration + GitCredentialsSecret string } func New(logger *zap.Logger, client kubernetes.Interface, cfg Config) *worker { @@ -44,11 +45,12 @@ func New(logger *zap.Logger, client kubernetes.Interface, cfg Config) *worker { } type KubernetesPlugin struct { - PodSpec *corev1.PodSpec - GitEnvFrom []corev1.EnvFromSource - Sidecars []corev1.Container `json:"sidecars,omitempty"` - Metadata Metadata - ExtraVolumeMounts []corev1.VolumeMount + PodSpec *corev1.PodSpec + GitCredentialsSecret string + GitEnvFrom []corev1.EnvFromSource + Sidecars []corev1.Container `json:"sidecars,omitempty"` + Metadata Metadata + ExtraVolumeMounts []corev1.VolumeMount } type Metadata struct { @@ -98,6 +100,7 @@ type jobWrapper struct { logger *zap.Logger job *api.CommandJob envMap map[string]string + envFrom []corev1.EnvFromSource err error k8sPlugin KubernetesPlugin otherPlugins []map[string]json.RawMessage @@ -106,10 +109,11 @@ type jobWrapper struct { func NewJobWrapper(logger *zap.Logger, job *api.CommandJob, config Config) *jobWrapper { return &jobWrapper{ - logger: logger, - job: job, - cfg: config, - envMap: make(map[string]string), + logger: logger, + job: job, + cfg: config, + envMap: make(map[string]string), + envFrom: make([]corev1.EnvFromSource, 0), } } @@ -207,6 +211,24 @@ func (w *jobWrapper) Build() (*batchv1.Job, error) { Value: w.job.Uuid, }, } + + // Generate env from configuration for git credentials + secretName := w.cfg.GitCredentialsSecret + if w.k8sPlugin.GitCredentialsSecret != "" { + secretName = w.cfg.GitCredentialsSecret + } + + if secretName != "" && len(w.k8sPlugin.GitEnvFrom) == 0 { + w.envFrom = append(w.envFrom, corev1.EnvFromSource{ + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: w.cfg.GitCredentialsSecret}, + }, + }) + } else if len(w.k8sPlugin.GitEnvFrom) > 0 { + w.logger.Warn("git-env-from is deprecated, please use git-credentials-secret instead") + w.envFrom = append(w.envFrom, w.k8sPlugin.GitEnvFrom...) + } + if w.otherPlugins != nil { otherPluginsJson, err := json.Marshal(w.otherPlugins) if err != nil { @@ -271,7 +293,7 @@ func (w *jobWrapper) Build() (*batchv1.Job, error) { c.WorkingDir = "/workspace" } c.VolumeMounts = append(c.VolumeMounts, volumeMounts...) - c.EnvFrom = append(c.EnvFrom, w.k8sPlugin.GitEnvFrom...) + c.EnvFrom = w.envFrom podSpec.Containers[i] = c } @@ -282,7 +304,7 @@ func (w *jobWrapper) Build() (*batchv1.Job, error) { c.Name = fmt.Sprintf("%s-%d", "sidecar", i) } c.VolumeMounts = append(c.VolumeMounts, volumeMounts...) - c.EnvFrom = append(c.EnvFrom, w.k8sPlugin.GitEnvFrom...) + c.EnvFrom = w.envFrom podSpec.Containers = append(podSpec.Containers, c) } @@ -440,7 +462,7 @@ func (w *jobWrapper) createCheckoutContainer( Value: "0", }, }, - EnvFrom: w.k8sPlugin.GitEnvFrom, + EnvFrom: w.envFrom, } checkoutContainer.Env = append(checkoutContainer.Env, env...) From a20da48ffea1fd78133890768768cf4f85ed77e9 Mon Sep 17 00:00:00 2001 From: 42Atomys Date: Thu, 8 Feb 2024 10:39:29 +0100 Subject: [PATCH 2/6] chore: rename git credentials into ssh to be more readable --- examples/config.yaml | 2 +- internal/controller/config/config.go | 4 ++-- internal/controller/controller.go | 2 +- internal/controller/scheduler/scheduler.go | 12 ++++++------ 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/examples/config.yaml b/examples/config.yaml index 1c52b565..9668845c 100644 --- a/examples/config.yaml +++ b/examples/config.yaml @@ -15,4 +15,4 @@ tags: # git-credentials-secret are the secret with the git credentials # configured on the agent with docker-ssh-config # @see https://github.com/buildkite/docker-ssh-env-config -git-credentials-secret: buildkite-ssh-github-creds +ssh-credentials-secret: buildkite-ssh-github-creds diff --git a/internal/controller/config/config.go b/internal/controller/config/config.go index 8691ed14..81ca0988 100644 --- a/internal/controller/config/config.go +++ b/internal/controller/config/config.go @@ -26,7 +26,7 @@ type Config struct { Tags stringSlice `mapstructure:"tags" validate:"min=1"` ProfilerAddress string `mapstructure:"profiler-address" validate:"omitempty,hostname_port"` ClusterUUID string `mapstructure:"cluster-uuid" validate:"omitempty"` - GitCredentialsSecret string `mapstructure:"git-credentials-secret" validate:"omitempty"` + SSHCredentialsSecret string `mapstructure:"git-credentials-secret" validate:"omitempty"` } type stringSlice []string @@ -40,7 +40,7 @@ func (s stringSlice) MarshalLogArray(enc zapcore.ArrayEncoder) error { func (c Config) MarshalLogObject(enc zapcore.ObjectEncoder) error { enc.AddString("agent-token-secret", c.AgentTokenSecret) - enc.AddString("git-credentials-secret", c.GitCredentialsSecret) + enc.AddString("ssh-credentials-secret", c.SSHCredentialsSecret) enc.AddBool("debug", c.Debug) enc.AddString("image", c.Image) enc.AddDuration("job-ttl", c.JobTTL) diff --git a/internal/controller/controller.go b/internal/controller/controller.go index 9f60f305..0b6f30e7 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -53,7 +53,7 @@ func Run( Image: cfg.Image, AgentToken: cfg.AgentTokenSecret, JobTTL: cfg.JobTTL, - GitCredentialsSecret: cfg.GitCredentialsSecret, + SSHCredentialsSecret: cfg.SSHCredentialsSecret, }) limiter := scheduler.NewLimiter(logger.Named("limiter"), sched, cfg.MaxInFlight) diff --git a/internal/controller/scheduler/scheduler.go b/internal/controller/scheduler/scheduler.go index 03194029..31fab3ee 100644 --- a/internal/controller/scheduler/scheduler.go +++ b/internal/controller/scheduler/scheduler.go @@ -33,7 +33,7 @@ type Config struct { Image string AgentToken string JobTTL time.Duration - GitCredentialsSecret string + SSHCredentialsSecret string } func New(logger *zap.Logger, client kubernetes.Interface, cfg Config) *worker { @@ -46,7 +46,7 @@ func New(logger *zap.Logger, client kubernetes.Interface, cfg Config) *worker { type KubernetesPlugin struct { PodSpec *corev1.PodSpec - GitCredentialsSecret string + SSHCredentialsSecret string GitEnvFrom []corev1.EnvFromSource Sidecars []corev1.Container `json:"sidecars,omitempty"` Metadata Metadata @@ -213,15 +213,15 @@ func (w *jobWrapper) Build() (*batchv1.Job, error) { } // Generate env from configuration for git credentials - secretName := w.cfg.GitCredentialsSecret - if w.k8sPlugin.GitCredentialsSecret != "" { - secretName = w.cfg.GitCredentialsSecret + secretName := w.cfg.SSHCredentialsSecret + if w.k8sPlugin.SSHCredentialsSecret != "" { + secretName = w.cfg.SSHCredentialsSecret } if secretName != "" && len(w.k8sPlugin.GitEnvFrom) == 0 { w.envFrom = append(w.envFrom, corev1.EnvFromSource{ SecretRef: &corev1.SecretEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{Name: w.cfg.GitCredentialsSecret}, + LocalObjectReference: corev1.LocalObjectReference{Name: w.cfg.SSHCredentialsSecret}, }, }) } else if len(w.k8sPlugin.GitEnvFrom) > 0 { From 550248f899856809ccba0f33dcd93f7fe2ec7d4c Mon Sep 17 00:00:00 2001 From: Atomys Date: Thu, 8 Feb 2024 10:55:16 +0100 Subject: [PATCH 3/6] chore: do a self review --- examples/config.yaml | 2 +- internal/controller/config/config.go | 2 +- internal/controller/scheduler/scheduler.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/config.yaml b/examples/config.yaml index 9668845c..86c978a6 100644 --- a/examples/config.yaml +++ b/examples/config.yaml @@ -12,7 +12,7 @@ cluster-uuid: beefcafe-abbe-baba-abba-deedcedecade tags: - queue=my-queue - priority=high -# git-credentials-secret are the secret with the git credentials +# ssh-credentials-secret are the secret with the ssh credentials # configured on the agent with docker-ssh-config # @see https://github.com/buildkite/docker-ssh-env-config ssh-credentials-secret: buildkite-ssh-github-creds diff --git a/internal/controller/config/config.go b/internal/controller/config/config.go index 81ca0988..38f59a86 100644 --- a/internal/controller/config/config.go +++ b/internal/controller/config/config.go @@ -26,7 +26,7 @@ type Config struct { Tags stringSlice `mapstructure:"tags" validate:"min=1"` ProfilerAddress string `mapstructure:"profiler-address" validate:"omitempty,hostname_port"` ClusterUUID string `mapstructure:"cluster-uuid" validate:"omitempty"` - SSHCredentialsSecret string `mapstructure:"git-credentials-secret" validate:"omitempty"` + SSHCredentialsSecret string `mapstructure:"ssh-credentials-secret" validate:"omitempty"` } type stringSlice []string diff --git a/internal/controller/scheduler/scheduler.go b/internal/controller/scheduler/scheduler.go index 31fab3ee..627716d0 100644 --- a/internal/controller/scheduler/scheduler.go +++ b/internal/controller/scheduler/scheduler.go @@ -293,7 +293,7 @@ func (w *jobWrapper) Build() (*batchv1.Job, error) { c.WorkingDir = "/workspace" } c.VolumeMounts = append(c.VolumeMounts, volumeMounts...) - c.EnvFrom = w.envFrom + c.EnvFrom = append(c.envFrom, w.envFrom...) podSpec.Containers[i] = c } @@ -304,7 +304,7 @@ func (w *jobWrapper) Build() (*batchv1.Job, error) { c.Name = fmt.Sprintf("%s-%d", "sidecar", i) } c.VolumeMounts = append(c.VolumeMounts, volumeMounts...) - c.EnvFrom = w.envFrom + c.EnvFrom = append(c.envFrom, w.envFrom...) podSpec.Containers = append(podSpec.Containers, c) } From 603cafcbf26eb094db81fcc2f9d6543a339c2262 Mon Sep 17 00:00:00 2001 From: 42Atomys Date: Mon, 12 Feb 2024 20:00:29 +0100 Subject: [PATCH 4/6] fix: env form on container are public --- internal/controller/scheduler/scheduler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/controller/scheduler/scheduler.go b/internal/controller/scheduler/scheduler.go index 627716d0..7afac0af 100644 --- a/internal/controller/scheduler/scheduler.go +++ b/internal/controller/scheduler/scheduler.go @@ -293,7 +293,7 @@ func (w *jobWrapper) Build() (*batchv1.Job, error) { c.WorkingDir = "/workspace" } c.VolumeMounts = append(c.VolumeMounts, volumeMounts...) - c.EnvFrom = append(c.envFrom, w.envFrom...) + c.EnvFrom = append(c.EnvFrom, w.envFrom...) podSpec.Containers[i] = c } @@ -304,7 +304,7 @@ func (w *jobWrapper) Build() (*batchv1.Job, error) { c.Name = fmt.Sprintf("%s-%d", "sidecar", i) } c.VolumeMounts = append(c.VolumeMounts, volumeMounts...) - c.EnvFrom = append(c.envFrom, w.envFrom...) + c.EnvFrom = append(c.EnvFrom, w.envFrom...) podSpec.Containers = append(podSpec.Containers, c) } From 5806b5e258b9f0dffbed15bf07120232b7ec9f53 Mon Sep 17 00:00:00 2001 From: 42Atomys Date: Mon, 12 Feb 2024 20:14:07 +0100 Subject: [PATCH 5/6] test: add integration tests for ssh credentials --- internal/integration/fixtures/secretref.yaml | 30 ++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/internal/integration/fixtures/secretref.yaml b/internal/integration/fixtures/secretref.yaml index b1aa12d9..73669207 100644 --- a/internal/integration/fixtures/secretref.yaml +++ b/internal/integration/fixtures/secretref.yaml @@ -14,6 +14,19 @@ steps: - echo - hello world +- label: ":git::console::superhero: checkout private git with ssh credentials secret as root" + agents: + queue: {{.queue}} + plugins: + - kubernetes: + ssh-credentials-secret: agent-stack-k8s + podSpec: + containers: + - image: alpine:latest + command: + - echo + - hello world + - label: ":git::console::student: checkout private git repo as user" agents: queue: {{.queue}} @@ -32,3 +45,20 @@ steps: runAsNonRoot: true runAsUser: 1000 runAsGroup: 1001 + +- label: ":git::console::student: checkout private git with ssh credentials secret repo as user" + agents: + queue: {{.queue}} + plugins: + - kubernetes: + ssh-credentials-secret: agent-stack-k8s + podSpec: + containers: + - image: alpine:latest + command: + - echo + - hello world + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1001 \ No newline at end of file From 0257e6f2804dcf58076dde9bc8fe28b75f1911ee Mon Sep 17 00:00:00 2001 From: 42Atomys Date: Tue, 13 Feb 2024 11:04:21 +0100 Subject: [PATCH 6/6] fix: incorrect secret naming --- internal/controller/scheduler/scheduler.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/controller/scheduler/scheduler.go b/internal/controller/scheduler/scheduler.go index 7afac0af..4bfb760c 100644 --- a/internal/controller/scheduler/scheduler.go +++ b/internal/controller/scheduler/scheduler.go @@ -215,17 +215,17 @@ func (w *jobWrapper) Build() (*batchv1.Job, error) { // Generate env from configuration for git credentials secretName := w.cfg.SSHCredentialsSecret if w.k8sPlugin.SSHCredentialsSecret != "" { - secretName = w.cfg.SSHCredentialsSecret + secretName = w.k8sPlugin.SSHCredentialsSecret } if secretName != "" && len(w.k8sPlugin.GitEnvFrom) == 0 { w.envFrom = append(w.envFrom, corev1.EnvFromSource{ SecretRef: &corev1.SecretEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{Name: w.cfg.SSHCredentialsSecret}, + LocalObjectReference: corev1.LocalObjectReference{Name: secretName}, }, }) } else if len(w.k8sPlugin.GitEnvFrom) > 0 { - w.logger.Warn("git-env-from is deprecated, please use git-credentials-secret instead") + w.logger.Warn("git-env-from is deprecated, please use ssh-credentials-secret instead") w.envFrom = append(w.envFrom, w.k8sPlugin.GitEnvFrom...) }