diff --git a/.vscode/launch.json b/.vscode/launch.json index 66ec26274..033686877 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -99,7 +99,9 @@ "RADIXOPERATOR_PODSECURITYSTANDARD_WARN_VERSION": "v1.23", "RADIX_ZONE": "dev", "RADIX_DEPLOYMENTS_PER_ENVIRONMENT_HISTORY_LIMIT": "10", - "RADIX_PIPELINE_JOBS_HISTORY_LIMIT": "5" + "RADIX_PIPELINE_JOBS_HISTORY_LIMIT": "5", + "SECCOMP_PROFILE_FILENAME": "allow-buildah.json", + "RADIX_BUILDAH_IMAGE_BUILDER": "quay.io/buildah/stable:v1.31" }, "args": ["--useOutClusterClient=false"] }, diff --git a/charts/radix-operator/Chart.yaml b/charts/radix-operator/Chart.yaml index 3dbc194c3..9b2fe9326 100644 --- a/charts/radix-operator/Chart.yaml +++ b/charts/radix-operator/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: radix-operator -version: 1.21.9 -appVersion: 1.41.9 +version: 1.22.0 +appVersion: 1.42.0 kubeVersion: ">=1.24.0" description: Radix Operator keywords: diff --git a/charts/radix-operator/templates/seccomp-profile-daemonset.yaml b/charts/radix-operator/templates/seccomp-profile-daemonset.yaml index a72867e72..fc41b5428 100644 --- a/charts/radix-operator/templates/seccomp-profile-daemonset.yaml +++ b/charts/radix-operator/templates/seccomp-profile-daemonset.yaml @@ -12,6 +12,14 @@ spec: labels: app: {{ .Values.seccompProfile.daemonSetName }} spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "nodepool-type" + operator: NotIn + values: ["system"] containers: - name: file-copy-container image: alpine @@ -30,3 +38,5 @@ spec: - name: configmap-volume configMap: name: {{ .Values.seccompProfile.configMapName }} + tolerations: + - operator: Exists diff --git a/pipeline-runner/steps/build_acr.go b/pipeline-runner/steps/build_acr.go index 215ec4c46..8a83a1f8d 100644 --- a/pipeline-runner/steps/build_acr.go +++ b/pipeline-runner/steps/build_acr.go @@ -78,6 +78,8 @@ func createACRBuildJob(rr *v1.RadixRegistration, pipelineInfo *model.PipelineInf Containers: buildContainers, SecurityContext: buildPodSecurityContext, Volumes: getACRBuildJobVolumes(&defaultMode, buildSecrets), + Affinity: utils.GetPodSpecAffinity(nil, appName, "", false, true), + Tolerations: utils.GetPodSpecTolerations(nil, false, true), }, }, }, diff --git a/pipeline-runner/utils/tekton.go b/pipeline-runner/utils/tekton.go index 3b9e5ace6..56e71dd76 100644 --- a/pipeline-runner/utils/tekton.go +++ b/pipeline-runner/utils/tekton.go @@ -20,7 +20,7 @@ const ( podLabelsFileName = "labels" ) -//CreateActionPipelineJob Create action pipeline job +// CreateActionPipelineJob Create action pipeline job func CreateActionPipelineJob(containerName string, action string, pipelineInfo *model.PipelineInfo, appName string, initContainers []corev1.Container, envVars *[]corev1.EnvVar) *batchv1.Job { imageTag := pipelineInfo.PipelineArguments.ImageTag jobName := pipelineInfo.PipelineArguments.JobName @@ -56,6 +56,8 @@ func CreateActionPipelineJob(containerName string, action string, pipelineInfo * }, Volumes: getJobVolumes(), RestartPolicy: "Never", + Affinity: utils.GetPodSpecAffinity(nil, appName, "", false, true), + Tolerations: utils.GetPodSpecTolerations(nil, false, true), }, }, }, diff --git a/pkg/apis/batch/kubejob.go b/pkg/apis/batch/kubejob.go index 8fe060adc..b6c85e9c7 100644 --- a/pkg/apis/batch/kubejob.go +++ b/pkg/apis/batch/kubejob.go @@ -177,8 +177,8 @@ func (s *syncer) buildJob(batchJob *radixv1.RadixBatchJob, jobComponent *radixv1 SecurityContext: securitycontext.Pod(securitycontext.WithPodSeccompProfile(corev1.SeccompProfileTypeRuntimeDefault)), RestartPolicy: corev1.RestartPolicyNever, ImagePullSecrets: rd.Spec.ImagePullSecrets, - Affinity: operatorUtils.GetPodSpecAffinity(node, rd.Spec.AppName, jobComponent.GetName()), - Tolerations: operatorUtils.GetPodSpecTolerations(node), + Affinity: operatorUtils.GetPodSpecAffinity(node, rd.Spec.AppName, jobComponent.GetName(), true, false), + Tolerations: operatorUtils.GetPodSpecTolerations(jobComponent.GetNode(), true, false), ActiveDeadlineSeconds: timeLimitSeconds, ServiceAccountName: serviceAccountSpec.ServiceAccountName(), AutomountServiceAccountToken: serviceAccountSpec.AutomountServiceAccountToken(), diff --git a/pkg/apis/batch/syncer_test.go b/pkg/apis/batch/syncer_test.go index d4fb8b62a..1ac0c443c 100644 --- a/pkg/apis/batch/syncer_test.go +++ b/pkg/apis/batch/syncer_test.go @@ -489,8 +489,17 @@ func (s *syncerTestSuite) Test_BatchStaticConfiguration() { s.Equal(corev1.PullAlways, kubejob.Spec.Template.Spec.Containers[0].ImagePullPolicy) s.Equal("default", kubejob.Spec.Template.Spec.ServiceAccountName) s.Equal(utils.BoolPtr(false), kubejob.Spec.Template.Spec.AutomountServiceAccountToken) - s.Nil(kubejob.Spec.Template.Spec.Affinity.NodeAffinity) - s.Len(kubejob.Spec.Template.Spec.Tolerations, 0) + s.Len(kubejob.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, 1) + s.Equal(corev1.NodeSelectorRequirement{ + Key: kube.RadixJobNodeLabel, + Operator: corev1.NodeSelectorOpExists, + }, kubejob.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0]) + s.Len(kubejob.Spec.Template.Spec.Tolerations, 1) + s.Equal(corev1.Toleration{ + Key: kube.NodeTaintJobsKey, + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }, kubejob.Spec.Template.Spec.Tolerations[0]) s.Len(kubejob.Spec.Template.Spec.Volumes, 0) s.Len(kubejob.Spec.Template.Spec.Containers[0].VolumeMounts, 0) services, err := s.kubeClient.CoreV1().Services(namespace).List(context.Background(), metav1.ListOptions{}) @@ -1023,34 +1032,45 @@ func (s *syncerTestSuite) Test_JobWithGpuNode() { s.Require().Len(jobs.Items, 2) job1 := slice.FindAll(jobs.Items, func(job batchv1.Job) bool { return job.GetName() == getKubeJobName(batchName, job1Name) })[0] - s.Len(job1.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, 1) - s.Len(job1.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions, 2) - gpu := s.getNodeSelectorRequirementByKeyForTest(job1.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions, kube.RadixGpuLabel) - s.Equal(corev1.NodeSelectorOpIn, gpu.Operator) - s.ElementsMatch([]string{"gpu1", "gpu2"}, gpu.Values) - gpuCount := s.getNodeSelectorRequirementByKeyForTest(job1.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions, kube.RadixGpuCountLabel) - s.Equal(corev1.NodeSelectorOpGt, gpuCount.Operator) - s.Equal([]string{"3"}, gpuCount.Values) + job1NodeSelectorTerms := job1.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms + s.Len(job1NodeSelectorTerms, 2) + s.Equal(corev1.NodeSelectorRequirement{ + Key: kube.RadixGpuLabel, + Operator: corev1.NodeSelectorOpIn, + Values: []string{"gpu1", "gpu2"}, + }, job1NodeSelectorTerms[0].MatchExpressions[0]) + s.Equal(corev1.NodeSelectorRequirement{ + Key: kube.RadixGpuCountLabel, + Operator: corev1.NodeSelectorOpGt, + Values: []string{"3"}, + }, job1NodeSelectorTerms[0].MatchExpressions[1]) + s.Equal(corev1.NodeSelectorRequirement{ + Key: kube.RadixJobNodeLabel, + Operator: corev1.NodeSelectorOpExists, + }, job1NodeSelectorTerms[1].MatchExpressions[0]) + tolerations := job1.Spec.Template.Spec.Tolerations - s.Len(tolerations, 1) - s.Equal(kube.NodeTaintGpuCountKey, tolerations[0].Key) - s.Equal(corev1.TolerationOpExists, tolerations[0].Operator) - s.Equal(corev1.TaintEffectNoSchedule, tolerations[0].Effect) + s.Len(tolerations, 2) + s.Equal(corev1.Toleration{Key: kube.RadixGpuCountLabel, Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}, tolerations[0]) + s.Equal(corev1.Toleration{Key: kube.NodeTaintJobsKey, Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}, tolerations[1]) job2 := slice.FindAll(jobs.Items, func(job batchv1.Job) bool { return job.GetName() == getKubeJobName(batchName, job2Name) })[0] - s.Len(job2.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, 1) - s.Len(job2.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions, 2) - gpu = s.getNodeSelectorRequirementByKeyForTest(job2.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions, kube.RadixGpuLabel) - s.Equal(corev1.NodeSelectorOpIn, gpu.Operator) - s.ElementsMatch([]string{"gpu3", "gpu4"}, gpu.Values) - gpuCount = s.getNodeSelectorRequirementByKeyForTest(job2.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions, kube.RadixGpuCountLabel) - s.Equal(corev1.NodeSelectorOpGt, gpuCount.Operator) - s.Equal([]string{"7"}, gpuCount.Values) - tolerations = job2.Spec.Template.Spec.Tolerations - s.Len(tolerations, 1) - s.Equal(kube.NodeTaintGpuCountKey, tolerations[0].Key) - s.Equal(corev1.TolerationOpExists, tolerations[0].Operator) - s.Equal(corev1.TaintEffectNoSchedule, tolerations[0].Effect) + job2NodeSelectorTerms := job2.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms + s.Len(job2NodeSelectorTerms, 2) + s.Equal(corev1.NodeSelectorRequirement{ + Key: kube.RadixGpuLabel, + Operator: corev1.NodeSelectorOpIn, + Values: []string{"gpu3", "gpu4"}, + }, job2NodeSelectorTerms[0].MatchExpressions[0]) + s.Equal(corev1.NodeSelectorRequirement{ + Key: kube.RadixGpuCountLabel, + Operator: corev1.NodeSelectorOpGt, + Values: []string{"7"}, + }, job2NodeSelectorTerms[0].MatchExpressions[1]) + s.Equal(corev1.NodeSelectorRequirement{ + Key: kube.RadixJobNodeLabel, + Operator: corev1.NodeSelectorOpExists, + }, job2NodeSelectorTerms[1].MatchExpressions[0]) } func (s *syncerTestSuite) Test_StopJob() { @@ -1503,15 +1523,6 @@ func (s *syncerTestSuite) Test_BatchJobStatusWaitingToStopped() { s.NotNil(batch.Status.JobStatuses[0].EndTime) } -func (s *syncerTestSuite) getNodeSelectorRequirementByKeyForTest(requirements []corev1.NodeSelectorRequirement, key string) *corev1.NodeSelectorRequirement { - for _, requirement := range requirements { - if requirement.Key == key { - return &requirement - } - } - return nil -} - func (s *syncerTestSuite) updateKubeJobStatus(jobName, namespace string) func(updater func(status *batchv1.JobStatus)) { job, err := s.kubeClient.BatchV1().Jobs(namespace).Get(context.Background(), jobName, metav1.GetOptions{}) if err != nil { diff --git a/pkg/apis/deployment/deployment_test.go b/pkg/apis/deployment/deployment_test.go index f559b624b..734214f2f 100644 --- a/pkg/apis/deployment/deployment_test.go +++ b/pkg/apis/deployment/deployment_test.go @@ -2965,19 +2965,20 @@ func TestUseGpuNodeOnDeploy(t *testing.T) { assert.NotNil(t, affinity) assert.NotNil(t, affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution) nodeSelectorTerms := affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms - assert.Equal(t, 1, len(nodeSelectorTerms)) - assert.Equal(t, 1, len(nodeSelectorTerms[0].MatchExpressions)) - expression := nodeSelectorTerms[0].MatchExpressions[0] - assert.Equal(t, kube.RadixGpuLabel, expression.Key) - assert.Equal(t, corev1.NodeSelectorOpIn, expression.Operator) - assert.Equal(t, 1, len(expression.Values)) - assert.Contains(t, expression.Values, gpuNvidiaV100) + assert.Len(t, nodeSelectorTerms, 2) + assert.Equal(t, corev1.NodeSelectorRequirement{ + Key: kube.RadixGpuLabel, + Operator: corev1.NodeSelectorOpIn, + Values: []string{gpuNvidiaV100}, + }, nodeSelectorTerms[0].MatchExpressions[0]) + assert.Equal(t, corev1.NodeSelectorRequirement{ + Key: kube.RadixJobNodeLabel, + Operator: corev1.NodeSelectorOpDoesNotExist, + }, nodeSelectorTerms[1].MatchExpressions[0]) tolerations := deployment.Spec.Template.Spec.Tolerations assert.Len(t, tolerations, 1) - assert.Equal(t, kube.NodeTaintGpuCountKey, tolerations[0].Key) - assert.Equal(t, corev1.TolerationOpExists, tolerations[0].Operator) - assert.Equal(t, corev1.TaintEffectNoSchedule, tolerations[0].Effect) + assert.Equal(t, corev1.Toleration{Key: kube.RadixGpuCountLabel, Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}, tolerations[0]) }) t.Run("has node with nvidia-v100, nvidia-p100", func(t *testing.T) { t.Parallel() @@ -2986,20 +2987,20 @@ func TestUseGpuNodeOnDeploy(t *testing.T) { assert.NotNil(t, affinity) assert.NotNil(t, affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution) nodeSelectorTerms := affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms - assert.Equal(t, 1, len(nodeSelectorTerms)) - assert.Equal(t, 1, len(nodeSelectorTerms[0].MatchExpressions)) - expression := nodeSelectorTerms[0].MatchExpressions[0] - assert.Equal(t, kube.RadixGpuLabel, expression.Key) - assert.Equal(t, corev1.NodeSelectorOpIn, expression.Operator) - assert.Equal(t, 2, len(expression.Values)) - assert.Contains(t, expression.Values, gpuNvidiaV100) - assert.Contains(t, expression.Values, gpuNvidiaP100) + assert.Len(t, nodeSelectorTerms, 2) + assert.Equal(t, corev1.NodeSelectorRequirement{ + Key: kube.RadixGpuLabel, + Operator: corev1.NodeSelectorOpIn, + Values: []string{gpuNvidiaV100, gpuNvidiaP100}, + }, nodeSelectorTerms[0].MatchExpressions[0]) + assert.Equal(t, corev1.NodeSelectorRequirement{ + Key: kube.RadixJobNodeLabel, + Operator: corev1.NodeSelectorOpDoesNotExist, + }, nodeSelectorTerms[1].MatchExpressions[0]) tolerations := deployment.Spec.Template.Spec.Tolerations assert.Len(t, tolerations, 1) - assert.Equal(t, kube.NodeTaintGpuCountKey, tolerations[0].Key) - assert.Equal(t, corev1.TolerationOpExists, tolerations[0].Operator) - assert.Equal(t, corev1.TaintEffectNoSchedule, tolerations[0].Effect) + assert.Equal(t, corev1.Toleration{Key: kube.RadixGpuCountLabel, Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}, tolerations[0]) }) t.Run("has node with nvidia-v100, nvidia-p100, not nvidia-k80", func(t *testing.T) { t.Parallel() @@ -3008,30 +3009,33 @@ func TestUseGpuNodeOnDeploy(t *testing.T) { assert.NotNil(t, affinity) assert.NotNil(t, affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution) nodeSelectorTerms := affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms - assert.Equal(t, 1, len(nodeSelectorTerms)) - assert.Equal(t, 2, len(nodeSelectorTerms[0].MatchExpressions)) - expression0 := nodeSelectorTerms[0].MatchExpressions[0] - assert.Equal(t, kube.RadixGpuLabel, expression0.Key) - assert.Equal(t, corev1.NodeSelectorOpIn, expression0.Operator) - assert.Equal(t, 2, len(expression0.Values)) - assert.Contains(t, expression0.Values, gpuNvidiaV100) - assert.Contains(t, expression0.Values, gpuNvidiaP100) - expression1 := nodeSelectorTerms[0].MatchExpressions[1] - assert.Equal(t, kube.RadixGpuLabel, expression1.Key) - assert.Equal(t, corev1.NodeSelectorOpNotIn, expression1.Operator) - assert.Equal(t, 1, len(expression1.Values)) - assert.Contains(t, expression1.Values, gpuNvidiaK80) + assert.Len(t, nodeSelectorTerms, 2) + assert.Equal(t, corev1.NodeSelectorRequirement{ + Key: kube.RadixGpuLabel, + Operator: corev1.NodeSelectorOpIn, + Values: []string{gpuNvidiaV100, gpuNvidiaP100}, + }, nodeSelectorTerms[0].MatchExpressions[0]) + assert.Equal(t, corev1.NodeSelectorRequirement{ + Key: kube.RadixJobNodeLabel, + Operator: corev1.NodeSelectorOpDoesNotExist, + }, nodeSelectorTerms[1].MatchExpressions[0]) tolerations := deployment.Spec.Template.Spec.Tolerations assert.Len(t, tolerations, 1) - assert.Equal(t, kube.NodeTaintGpuCountKey, tolerations[0].Key) - assert.Equal(t, corev1.TolerationOpExists, tolerations[0].Operator) - assert.Equal(t, corev1.TaintEffectNoSchedule, tolerations[0].Effect) + assert.Equal(t, corev1.Toleration{Key: kube.RadixGpuCountLabel, Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}, tolerations[0]) }) t.Run("has node with no gpu", func(t *testing.T) { t.Parallel() deployment, _ := client.AppsV1().Deployments(envNamespace).Get(context.TODO(), componentName4, metav1.GetOptions{}) - assert.Nil(t, deployment.Spec.Template.Spec.Affinity.NodeAffinity) + affinity := deployment.Spec.Template.Spec.Affinity + assert.NotNil(t, affinity) + assert.NotNil(t, affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution) + nodeSelectorTerms := affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms + assert.Len(t, nodeSelectorTerms, 1) + assert.Equal(t, corev1.NodeSelectorRequirement{ + Key: kube.RadixJobNodeLabel, + Operator: corev1.NodeSelectorOpDoesNotExist, + }, nodeSelectorTerms[0].MatchExpressions[0]) tolerations := deployment.Spec.Template.Spec.Tolerations assert.Len(t, tolerations, 0) @@ -3040,7 +3044,15 @@ func TestUseGpuNodeOnDeploy(t *testing.T) { t.Parallel() deployment, _ := client.AppsV1().Deployments(envNamespace).Get(context.TODO(), jobComponentName, metav1.GetOptions{}) affinity := deployment.Spec.Template.Spec.Affinity - assert.Nil(t, affinity.NodeAffinity) + assert.NotNil(t, affinity) + assert.NotNil(t, affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution) + nodeSelectorTerms := affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms + assert.Len(t, nodeSelectorTerms, 1) + assert.Len(t, nodeSelectorTerms[0].MatchExpressions, 1) + assert.Equal(t, corev1.NodeSelectorRequirement{ + Key: kube.RadixJobNodeLabel, + Operator: corev1.NodeSelectorOpDoesNotExist, + }, nodeSelectorTerms[0].MatchExpressions[0]) tolerations := deployment.Spec.Template.Spec.Tolerations assert.Len(t, tolerations, 0) @@ -3213,13 +3225,20 @@ func TestUseGpuNodeCountOnDeployment(t *testing.T) { assert.NotNil(t, affinity) assert.NotNil(t, affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution) nodeSelectorTerms := affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms - assert.Equal(t, 1, len(nodeSelectorTerms)) - assert.Equal(t, 1, len(nodeSelectorTerms[0].MatchExpressions)) - expression0 := nodeSelectorTerms[0].MatchExpressions[0] - assert.Equal(t, kube.RadixGpuCountLabel, expression0.Key) - assert.Equal(t, corev1.NodeSelectorOpGt, expression0.Operator) - assert.Equal(t, 1, len(expression0.Values)) - assert.Contains(t, expression0.Values, "0") + assert.Len(t, nodeSelectorTerms, 2) + assert.Len(t, nodeSelectorTerms[0].MatchExpressions, 1) + assert.Equal(t, corev1.NodeSelectorRequirement{ + Key: kube.RadixGpuCountLabel, + Operator: corev1.NodeSelectorOpGt, + Values: []string{"0"}, + }, nodeSelectorTerms[0].MatchExpressions[0]) + assert.Equal(t, corev1.NodeSelectorRequirement{ + Key: kube.RadixJobNodeLabel, + Operator: corev1.NodeSelectorOpDoesNotExist, + }, nodeSelectorTerms[1].MatchExpressions[0]) + + tolerations := deployment.Spec.Template.Spec.Tolerations + assert.Len(t, tolerations, 0) // missing node.gpu }) t.Run("has node with gpu-count 10", func(t *testing.T) { t.Parallel() @@ -3228,43 +3247,100 @@ func TestUseGpuNodeCountOnDeployment(t *testing.T) { assert.NotNil(t, affinity) assert.NotNil(t, affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution) nodeSelectorTerms := affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms - assert.Equal(t, 1, len(nodeSelectorTerms)) - assert.Equal(t, 1, len(nodeSelectorTerms[0].MatchExpressions)) - expression0 := nodeSelectorTerms[0].MatchExpressions[0] - assert.Equal(t, kube.RadixGpuCountLabel, expression0.Key) - assert.Equal(t, corev1.NodeSelectorOpGt, expression0.Operator) - assert.Equal(t, 1, len(expression0.Values)) - assert.Contains(t, expression0.Values, "9") + assert.Len(t, nodeSelectorTerms, 2) + assert.Len(t, nodeSelectorTerms[0].MatchExpressions, 1) + assert.Equal(t, corev1.NodeSelectorRequirement{ + Key: kube.RadixGpuCountLabel, + Operator: corev1.NodeSelectorOpGt, + Values: []string{"9"}, + }, nodeSelectorTerms[0].MatchExpressions[0]) + assert.Equal(t, corev1.NodeSelectorRequirement{ + Key: kube.RadixJobNodeLabel, + Operator: corev1.NodeSelectorOpDoesNotExist, + }, nodeSelectorTerms[1].MatchExpressions[0]) + + tolerations := deployment.Spec.Template.Spec.Tolerations + assert.Len(t, tolerations, 0) // missing node.gpu }) t.Run("has node with gpu-count 0", func(t *testing.T) { t.Parallel() deployment, _ := client.AppsV1().Deployments(envNamespace).Get(context.TODO(), componentName3, metav1.GetOptions{}) affinity := deployment.Spec.Template.Spec.Affinity - assert.Nil(t, affinity.NodeAffinity) + assert.NotNil(t, affinity) + assert.NotNil(t, affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution) + nodeSelectorTerms := affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms + assert.Len(t, nodeSelectorTerms, 1) + assert.Equal(t, corev1.NodeSelectorRequirement{ + Key: kube.RadixJobNodeLabel, + Operator: corev1.NodeSelectorOpDoesNotExist, + }, nodeSelectorTerms[0].MatchExpressions[0]) + + tolerations := deployment.Spec.Template.Spec.Tolerations + assert.Len(t, tolerations, 0) }) t.Run("has node with gpu-count -1", func(t *testing.T) { t.Parallel() deployment, _ := client.AppsV1().Deployments(envNamespace).Get(context.TODO(), componentName4, metav1.GetOptions{}) affinity := deployment.Spec.Template.Spec.Affinity - assert.Nil(t, affinity.NodeAffinity) + assert.NotNil(t, affinity) + assert.NotNil(t, affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution) + nodeSelectorTerms := affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms + assert.Len(t, nodeSelectorTerms, 1) + assert.Equal(t, corev1.NodeSelectorRequirement{ + Key: kube.RadixJobNodeLabel, + Operator: corev1.NodeSelectorOpDoesNotExist, + }, nodeSelectorTerms[0].MatchExpressions[0]) + + tolerations := deployment.Spec.Template.Spec.Tolerations + assert.Len(t, tolerations, 0) }) t.Run("has node with invalid value of gpu-count", func(t *testing.T) { t.Parallel() deployment, _ := client.AppsV1().Deployments(envNamespace).Get(context.TODO(), componentName5, metav1.GetOptions{}) affinity := deployment.Spec.Template.Spec.Affinity - assert.Nil(t, affinity.NodeAffinity) + assert.NotNil(t, affinity) + assert.NotNil(t, affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution) + nodeSelectorTerms := affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms + assert.Len(t, nodeSelectorTerms, 1) + assert.Equal(t, corev1.NodeSelectorRequirement{ + Key: kube.RadixJobNodeLabel, + Operator: corev1.NodeSelectorOpDoesNotExist, + }, nodeSelectorTerms[0].MatchExpressions[0]) + + tolerations := deployment.Spec.Template.Spec.Tolerations + assert.Len(t, tolerations, 0) }) t.Run("has node with no gpu-count", func(t *testing.T) { t.Parallel() deployment, _ := client.AppsV1().Deployments(envNamespace).Get(context.TODO(), componentName6, metav1.GetOptions{}) affinity := deployment.Spec.Template.Spec.Affinity - assert.Nil(t, affinity.NodeAffinity) + assert.NotNil(t, affinity) + assert.NotNil(t, affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution) + nodeSelectorTerms := affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms + assert.Len(t, nodeSelectorTerms, 1) + assert.Equal(t, corev1.NodeSelectorRequirement{ + Key: kube.RadixJobNodeLabel, + Operator: corev1.NodeSelectorOpDoesNotExist, + }, nodeSelectorTerms[0].MatchExpressions[0]) + + tolerations := deployment.Spec.Template.Spec.Tolerations + assert.Len(t, tolerations, 0) }) t.Run("job has node, but pod template of Job Scheduler does not have it", func(t *testing.T) { t.Parallel() deployment, _ := client.AppsV1().Deployments(envNamespace).Get(context.TODO(), jobComponentName, metav1.GetOptions{}) affinity := deployment.Spec.Template.Spec.Affinity - assert.Nil(t, affinity.NodeAffinity) + assert.NotNil(t, affinity) + assert.NotNil(t, affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution) + nodeSelectorTerms := affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms + assert.Len(t, nodeSelectorTerms, 1) + assert.Equal(t, corev1.NodeSelectorRequirement{ + Key: kube.RadixJobNodeLabel, + Operator: corev1.NodeSelectorOpDoesNotExist, + }, nodeSelectorTerms[0].MatchExpressions[0]) + + tolerations := deployment.Spec.Template.Spec.Tolerations + assert.Len(t, tolerations, 0) }) } @@ -3308,30 +3384,48 @@ func TestUseGpuNodeWithGpuCountOnDeployment(t *testing.T) { assert.NotNil(t, affinity) assert.NotNil(t, affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution) nodeSelectorTerms := affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms - assert.Equal(t, 1, len(nodeSelectorTerms)) - assert.Equal(t, 3, len(nodeSelectorTerms[0].MatchExpressions)) - expression0 := nodeSelectorTerms[0].MatchExpressions[0] - assert.Equal(t, kube.RadixGpuLabel, expression0.Key) - assert.Equal(t, corev1.NodeSelectorOpIn, expression0.Operator) - assert.Equal(t, 2, len(expression0.Values)) - assert.Contains(t, expression0.Values, gpuNvidiaV100) - assert.Contains(t, expression0.Values, gpuNvidiaP100) - expression1 := nodeSelectorTerms[0].MatchExpressions[1] - assert.Equal(t, kube.RadixGpuLabel, expression1.Key) - assert.Equal(t, corev1.NodeSelectorOpNotIn, expression1.Operator) - assert.Equal(t, 1, len(expression1.Values)) - assert.Contains(t, expression1.Values, gpuNvidiaK80) - expression3 := nodeSelectorTerms[0].MatchExpressions[2] - assert.Equal(t, kube.RadixGpuCountLabel, expression3.Key) - assert.Equal(t, corev1.NodeSelectorOpGt, expression3.Operator) - assert.Equal(t, 1, len(expression3.Values)) - assert.Contains(t, expression3.Values, "9") + assert.Len(t, nodeSelectorTerms, 2) + assert.Len(t, nodeSelectorTerms[0].MatchExpressions, 3) + assert.Equal(t, corev1.NodeSelectorRequirement{ + Key: kube.RadixGpuLabel, + Operator: corev1.NodeSelectorOpIn, + Values: []string{gpuNvidiaV100, gpuNvidiaP100}, + }, nodeSelectorTerms[0].MatchExpressions[0]) + assert.Equal(t, corev1.NodeSelectorRequirement{ + Key: kube.RadixGpuLabel, + Operator: corev1.NodeSelectorOpNotIn, + Values: []string{gpuNvidiaK80}, + }, nodeSelectorTerms[0].MatchExpressions[1]) + assert.Equal(t, corev1.NodeSelectorRequirement{ + Key: kube.RadixGpuCountLabel, + Operator: corev1.NodeSelectorOpGt, + Values: []string{"9"}, + }, nodeSelectorTerms[0].MatchExpressions[2]) + assert.Equal(t, corev1.NodeSelectorRequirement{ + Key: kube.RadixJobNodeLabel, + Operator: corev1.NodeSelectorOpDoesNotExist, + }, nodeSelectorTerms[1].MatchExpressions[0]) + + tolerations := deployment.Spec.Template.Spec.Tolerations + assert.Len(t, tolerations, 1) + assert.Equal(t, corev1.Toleration{Key: kube.RadixGpuCountLabel, Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}, tolerations[0]) }) t.Run("job has node, but pod template of Job Scheduler does not have it", func(t *testing.T) { t.Parallel() deployment, _ := client.AppsV1().Deployments(envNamespace).Get(context.TODO(), jobComponentName, metav1.GetOptions{}) affinity := deployment.Spec.Template.Spec.Affinity - assert.Nil(t, affinity.NodeAffinity) + tolerations := deployment.Spec.Template.Spec.Tolerations + assert.NotNil(t, affinity) + assert.NotNil(t, affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution) + nodeSelectorTerms := affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms + assert.Len(t, nodeSelectorTerms, 1) + assert.Len(t, nodeSelectorTerms[0].MatchExpressions, 1) + assert.Equal(t, corev1.NodeSelectorRequirement{ + Key: kube.RadixJobNodeLabel, + Operator: corev1.NodeSelectorOpDoesNotExist, + }, nodeSelectorTerms[0].MatchExpressions[0]) + + assert.Len(t, tolerations, 0) }) } diff --git a/pkg/apis/deployment/kubedeployment.go b/pkg/apis/deployment/kubedeployment.go index c2eb7d426..c5f86d124 100644 --- a/pkg/apis/deployment/kubedeployment.go +++ b/pkg/apis/deployment/kubedeployment.go @@ -233,7 +233,7 @@ func (deploy *Deployment) getJobAuxDeploymentLabels(deployComponent v1.RadixComm ) } -func (deploy *Deployment) getDeploymentAnnotations(deployComponent v1.RadixCommonDeployComponent) map[string]string { +func (deploy *Deployment) getDeploymentAnnotations() map[string]string { branch, _ := deploy.getRadixBranchAndCommitId() return radixannotations.ForRadixBranch(branch) } @@ -244,7 +244,7 @@ func (deploy *Deployment) setDesiredDeploymentProperties(deployComponent v1.Radi desiredDeployment.ObjectMeta.Name = deployComponent.GetName() desiredDeployment.ObjectMeta.OwnerReferences = []metav1.OwnerReference{getOwnerReferenceOfDeployment(deploy.radixDeployment)} desiredDeployment.ObjectMeta.Labels = deploy.getDeploymentLabels(deployComponent) - desiredDeployment.ObjectMeta.Annotations = deploy.getDeploymentAnnotations(deployComponent) + desiredDeployment.ObjectMeta.Annotations = deploy.getDeploymentAnnotations() desiredDeployment.Spec.Selector.MatchLabels = radixlabels.ForComponentName(componentName) desiredDeployment.Spec.Replicas = getDesiredComponentReplicas(deployComponent) @@ -266,8 +266,8 @@ func (deploy *Deployment) setDesiredDeploymentProperties(deployComponent v1.Radi spec := NewServiceAccountSpec(deploy.radixDeployment, deployComponent) desiredDeployment.Spec.Template.Spec.AutomountServiceAccountToken = spec.AutomountServiceAccountToken() desiredDeployment.Spec.Template.Spec.ServiceAccountName = spec.ServiceAccountName() - desiredDeployment.Spec.Template.Spec.Affinity = utils.GetPodSpecAffinity(deployComponent.GetNode(), appName, componentName) - desiredDeployment.Spec.Template.Spec.Tolerations = utils.GetPodSpecTolerations(deployComponent.GetNode()) + desiredDeployment.Spec.Template.Spec.Affinity = utils.GetPodSpecAffinity(deployComponent.GetNode(), appName, componentName, false, false) + desiredDeployment.Spec.Template.Spec.Tolerations = utils.GetPodSpecTolerations(deployComponent.GetNode(), false, false) volumes, err := deploy.GetVolumesForComponent(deployComponent) if err != nil { diff --git a/pkg/apis/job/kubejob.go b/pkg/apis/job/kubejob.go index 8c5ba16db..b8d2cfee6 100644 --- a/pkg/apis/job/kubejob.go +++ b/pkg/apis/job/kubejob.go @@ -10,6 +10,7 @@ import ( pipelineJob "github.com/equinor/radix-operator/pkg/apis/pipeline" "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/securitycontext" + "github.com/equinor/radix-operator/pkg/apis/utils" "github.com/equinor/radix-operator/pkg/apis/utils/git" log "github.com/sirupsen/logrus" batchv1 "k8s.io/api/batch/v1" @@ -98,6 +99,8 @@ func (job *Job) getPipelineJobConfig() (*batchv1.Job, error) { }, }, RestartPolicy: "Never", + Affinity: utils.GetPodSpecAffinity(nil, appName, "", false, true), + Tolerations: utils.GetPodSpecTolerations(nil, false, true), }, }, }, diff --git a/pkg/apis/kube/kube.go b/pkg/apis/kube/kube.go index ffb30b8ea..1b361a4b0 100644 --- a/pkg/apis/kube/kube.go +++ b/pkg/apis/kube/kube.go @@ -56,6 +56,7 @@ const ( RadixVolumeMountNameLabel = "radix-volume-mount-name" RadixGpuLabel = "radix-node-gpu" RadixGpuCountLabel = "radix-node-gpu-count" + RadixJobNodeLabel = "nodepooltasks" RadixNamespace = "radix-namespace" RadixConfigMapTypeLabel = "radix-config-map-type" RadixSecretTypeLabel = "radix-secret-type" @@ -72,6 +73,7 @@ const ( // NodeTaintGpuCountKey defines the taint key on GPU nodes. // Pods required to run on nodes with this taint must add a toleration with effect NoSchedule NodeTaintGpuCountKey = "radix-node-gpu-count" + NodeTaintJobsKey = "nodepooltasks" // RadixBranchDeprecated Only for backward compatibility RadixBranchDeprecated = "radix-branch" diff --git a/pkg/apis/utils/affinity.go b/pkg/apis/utils/affinity.go index adecf74c0..375c8f4e1 100644 --- a/pkg/apis/utils/affinity.go +++ b/pkg/apis/utils/affinity.go @@ -12,48 +12,53 @@ import ( corev1 "k8s.io/api/core/v1" ) -func GetPodSpecAffinity(node *v1.RadixNode, appName string, componentName string) *corev1.Affinity { - +func GetPodSpecAffinity(node *v1.RadixNode, appName string, componentName string, isScheduledJob bool, isPipelineJob bool) *corev1.Affinity { affinity := &corev1.Affinity{ PodAntiAffinity: &corev1.PodAntiAffinity{ PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{ { - Weight: 1, - PodAffinityTerm: corev1.PodAffinityTerm{ - LabelSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: kube.RadixAppLabel, - Operator: metav1.LabelSelectorOpIn, - Values: []string{appName}, - }, - { - Key: kube.RadixComponentLabel, - Operator: metav1.LabelSelectorOpIn, - Values: []string{componentName}, - }, - }, - }, - TopologyKey: corev1.LabelHostname, - }, + Weight: 1, + PodAffinityTerm: getPodAffinityTerm(appName, componentName), }, }, }, } - if node != nil { - nodeAffinity := &corev1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{}}, - } - addGpuNodeSelectorTerms(node, nodeAffinity) - if len(nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms) > 0 { - affinity.NodeAffinity = nodeAffinity - } + nodeAffinity := &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{}}, + } + addGpuNodeSelectorTerms(node, nodeAffinity) + addJobNodeSelectorTerms(nodeAffinity, isScheduledJob, isPipelineJob) + if len(nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms) > 0 { + affinity.NodeAffinity = nodeAffinity } return affinity } +func getPodAffinityTerm(appName string, componentName string) corev1.PodAffinityTerm { + matchExpressions := []metav1.LabelSelectorRequirement{ + { + Key: kube.RadixAppLabel, + Operator: metav1.LabelSelectorOpIn, + Values: []string{appName}, + }, + } + if len(componentName) > 0 { + matchExpressions = append(matchExpressions, metav1.LabelSelectorRequirement{ + Key: kube.RadixComponentLabel, + Operator: metav1.LabelSelectorOpIn, + Values: []string{componentName}, + }) + } + return corev1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: matchExpressions, + }, + TopologyKey: corev1.LabelHostname, + } +} + func addGpuNodeSelectorTerms(node *v1.RadixNode, nodeAffinity *corev1.NodeAffinity) { nodeSelectorTerm := corev1.NodeSelectorTerm{} @@ -95,6 +100,18 @@ func addNodeSelectorRequirementForGpuCount(gpuCount string, nodeSelectorTerm *co addNodeSelectorRequirement(nodeSelectorTerm, kube.RadixGpuCountLabel, corev1.NodeSelectorOpGt, values) } +func addJobNodeSelectorTerms(nodeAffinity *corev1.NodeAffinity, isScheduledJob bool, isPipelineJob bool) { + requirement := corev1.NodeSelectorRequirement{Key: kube.RadixJobNodeLabel} + if isPipelineJob || isScheduledJob { + requirement.Operator = corev1.NodeSelectorOpExists + } else { + requirement.Operator = corev1.NodeSelectorOpDoesNotExist + } + nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append(nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, corev1.NodeSelectorTerm{ + MatchExpressions: []corev1.NodeSelectorRequirement{requirement}, + }) +} + func getGpuLists(nodeGpuList []string) ([]string, []string) { includingGpus := make([]string, 0) excludingGpus := make([]string, 0) diff --git a/pkg/apis/utils/tolerations.go b/pkg/apis/utils/tolerations.go index 66c709303..de758be63 100644 --- a/pkg/apis/utils/tolerations.go +++ b/pkg/apis/utils/tolerations.go @@ -8,14 +8,13 @@ import ( corev1 "k8s.io/api/core/v1" ) -// GetPodSpecTolerations returns tolerations required to schedule the pod on nodes defined by RadixNode -func GetPodSpecTolerations(node *v1.RadixNode) []corev1.Toleration { - if node == nil { - return nil - } - +// GetPodSpecTolerations returns tolerations required to schedule the pod on nodes +func GetPodSpecTolerations(node *v1.RadixNode, isScheduledJob bool, isPipelineJob bool) []corev1.Toleration { var tolerations []corev1.Toleration tolerations = append(tolerations, getGpuNodeTolerations(node)...) + if isPipelineJob || isScheduledJob { + return append(tolerations, getJobNodeToleration()) + } return tolerations } @@ -29,9 +28,15 @@ func getGpuNodeTolerations(node *v1.RadixNode) []corev1.Toleration { return nil } - tolerations := []corev1.Toleration{ - {Key: kube.NodeTaintGpuCountKey, Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}, + return []corev1.Toleration{ + getNodeTolerationExists(kube.NodeTaintGpuCountKey), } +} - return tolerations +func getJobNodeToleration() corev1.Toleration { + return getNodeTolerationExists(kube.NodeTaintJobsKey) +} + +func getNodeTolerationExists(key string) corev1.Toleration { + return corev1.Toleration{Key: key, Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule} }