diff --git a/config/config.sample.toml b/config/config.sample.toml index b21014d..fd07232 100644 --- a/config/config.sample.toml +++ b/config/config.sample.toml @@ -47,6 +47,11 @@ image_registry = "mattermost" # # The Persistent Volume Claim name to use to store data produced by jobs (e.g. recording files). #persistent_volume_claim_name = "my-pvc" +# +# A comma separated list of Sysctls to apply on the node through priviledged init container before starting jobs. +# For example, enabling the `kernel.unprivileged_userns_clone` at node level was necessary +# on Debian based systems (pre kernel 5.10) in order to run Chromium sandbox. +#node_sysctls = "kernel.unprivileged_userns_clone=1" [logger] # A boolean controlling whether to log to the console. diff --git a/service/kubernetes/service.go b/service/kubernetes/service.go index 8629999..b0c9ed7 100644 --- a/service/kubernetes/service.go +++ b/service/kubernetes/service.go @@ -56,6 +56,7 @@ type JobServiceConfig struct { ImageRegistry string JobsResourceRequirements JobsResourceRequirements `toml:"jobs_resource_requirements"` PersistentVolumeClaimName string `toml:"persistent_volume_claim_name"` + NodeSysctls string `toml:"node_sysctls"` } func (c JobServiceConfig) IsValid() error { @@ -153,30 +154,12 @@ func (s *JobService) CreateJob(cfg job.Config, onStopCb job.StopCb) (job.Job, er var jobID string var jobPrefix string var env []corev1.EnvVar - var initContainers []corev1.Container switch cfg.Type { case job.TypeRecording: cfg.InputData.SetSiteURL(getSiteURLForJob(cfg.InputData.GetSiteURL())) jobPrefix = job.RecordingJobPrefix jobID = jobPrefix + "-job-" + random.NewID() env = append(env, getEnvFromJobInputData(cfg.InputData)...) - initContainers = []corev1.Container{ - { - Name: jobID + "-init", - Image: k8sInitContainerImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{ - // Enabling the `kernel.unprivileged_userns_clone` sysctl at node level is necessary in order to run Chromium sandbox. - // See https://developer.chrome.com/docs/puppeteer/troubleshooting/#recommended-enable-user-namespace-cloning for details. - "sysctl", - "-w", - "kernel.unprivileged_userns_clone=1", - }, - SecurityContext: &corev1.SecurityContext{ - Privileged: newBool(true), - }, - }, - } case job.TypeTranscribing: cfg.InputData.SetSiteURL(getSiteURLForJob(cfg.InputData.GetSiteURL())) jobPrefix = job.TranscribingJobPrefix @@ -184,6 +167,15 @@ func (s *JobService) CreateJob(cfg job.Config, onStopCb job.StopCb) (job.Job, er env = append(env, getEnvFromJobInputData(cfg.InputData)...) } + var initContainers []corev1.Container + if s.cfg.NodeSysctls != "" { + s.log.Info("generating init containers", mlog.String("sysctls", s.cfg.NodeSysctls)) + initContainers, err = genInitContainers(jobID, k8sInitContainerImage, s.cfg.NodeSysctls) + if err != nil { + return job.Job{}, fmt.Errorf("failed to generate init containers: %w", err) + } + } + var hostNetwork bool if devMode { s.log.Info("DEV_MODE enabled, enabling host networking", mlog.String("hostIP", os.Getenv("HOST_IP"))) diff --git a/service/kubernetes/utils.go b/service/kubernetes/utils.go index 2b30c34..de0c23f 100644 --- a/service/kubernetes/utils.go +++ b/service/kubernetes/utils.go @@ -114,3 +114,37 @@ func getSiteURLForJob(siteURL string) string { return siteURL } + +func genInitContainers(jobID, image, sysctls string) ([]corev1.Container, error) { + if jobID == "" { + return nil, fmt.Errorf("invalid empty jobID") + } + + if image == "" { + return nil, fmt.Errorf("invalid empty image") + } + + if sysctls == "" { + return nil, fmt.Errorf("invalid empty sysctls") + } + + ctls := strings.Split(sysctls, ",") + cnts := make([]corev1.Container, len(ctls)) + for i, ctl := range ctls { + cnts[i] = corev1.Container{ + Name: fmt.Sprintf("%s-init-%d", jobID, i), + Image: image, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{ + "sysctl", + "-w", + ctl, + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: newBool(true), + }, + } + } + + return cnts, nil +} diff --git a/service/kubernetes/utils_test.go b/service/kubernetes/utils_test.go index cbdfb16..319e04b 100644 --- a/service/kubernetes/utils_test.go +++ b/service/kubernetes/utils_test.go @@ -171,3 +171,95 @@ func TestGetJobPodTolerations(t *testing.T) { require.EqualError(t, err, "failed to open file invalid: open invalid: no such file or directory") }) } + +func TestGenInitContainers(t *testing.T) { + for _, tc := range []struct { + name string + jobID string + image string + sysctls string + err string + cnts []corev1.Container + }{ + { + name: "empty jobID", + err: "invalid empty jobID", + }, + { + name: "empty image", + jobID: "jobID", + err: "invalid empty image", + }, + { + name: "empty sysctls", + jobID: "jobID", + image: "image", + err: "invalid empty sysctls", + }, + { + name: "single sysctl", + jobID: "jobID", + image: "image", + sysctls: "kernel.unprivileged_userns_clone=1", + cnts: []corev1.Container{ + { + Name: "jobID-init-0", + Image: "image", + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{ + "sysctl", + "-w", + "kernel.unprivileged_userns_clone=1", + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: newBool(true), + }, + }, + }, + }, + { + name: "multiple sysctls", + jobID: "jobID", + image: "image", + sysctls: "kernel.unprivileged_userns_clone=1,user.max_user_namespaces=4545", + cnts: []corev1.Container{ + { + Name: "jobID-init-0", + Image: "image", + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{ + "sysctl", + "-w", + "kernel.unprivileged_userns_clone=1", + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: newBool(true), + }, + }, + { + Name: "jobID-init-1", + Image: "image", + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{ + "sysctl", + "-w", + "user.max_user_namespaces=4545", + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: newBool(true), + }, + }, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + cnts, err := genInitContainers(tc.jobID, tc.image, tc.sysctls) + if tc.err != "" { + require.Empty(t, cnts) + require.EqualError(t, err, tc.err) + } else { + require.Equal(t, tc.cnts, cnts) + } + }) + } +}