feat: Allow to configure resource requirements of individual containe…

…rs (#708) Allow to configure [resource requirements](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) for each of individuals containers deployed by network-operator. Each state might deploy one or more containers, user needs to specify the name of the specific container to set its resource requirements. By default, requirements are not applied or set to previously set defaults.
Mellanox · Dec 20, 2023 · 011c546 · 011c546
2 parents ad48cf1 + bd43ffb
commit 011c546
Show file tree

Hide file tree

Showing 28 changed files with 1,119 additions and 31 deletions.
diff --git a/api/v1alpha1/nicclusterpolicy_types.go b/api/v1alpha1/nicclusterpolicy_types.go
@@ -47,7 +47,8 @@ type ImageSpec struct {
 	Version string `json:"version"`
 	// +optional
 	// +kubebuilder:default:={}
-	ImagePullSecrets []string `json:"imagePullSecrets"`
+	ImagePullSecrets   []string               `json:"imagePullSecrets"`
+	ContainerResources []ResourceRequirements `json:"containerResources,omitempty"`
 }
 
 // ImageSpecWithConfig Contains ImageSpec and optional configuration
@@ -184,6 +185,22 @@ type SecondaryNetworkSpec struct {
 	IpamPlugin *ImageSpec `json:"ipamPlugin,omitempty"`
 }
 
+// ResourceRequirements describes the compute resource requirements.
+type ResourceRequirements struct {
+	// Name of the container the requirements are set for
+	Name string `json:"name"`
+	// Limits describes the maximum amount of compute resources allowed.
+	// More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
+	// +optional
+	Limits v1.ResourceList `json:"limits,omitempty"`
+	// Requests describes the minimum amount of compute resources required.
+	// If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,
+	// otherwise to an implementation-defined value. Requests cannot exceed Limits.
+	// More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
+	// +optional
+	Requests v1.ResourceList `json:"requests,omitempty"`
+}
+
 // IBKubernetesSpec describes configuration options for ib-kubernetes
 type IBKubernetesSpec struct {
 	// Image information for ib-kubernetes

diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/mellanox.com_nicclusterpolicies.yaml b/config/crd/bases/mellanox.com_nicclusterpolicies.yaml
diff --git a/deployment/network-operator/crds/mellanox.com_nicclusterpolicies.yaml b/deployment/network-operator/crds/mellanox.com_nicclusterpolicies.yaml
diff --git a/manifests/state-container-networking-plugins/0010-container-networking-plugins-ds.yml b/manifests/state-container-networking-plugins/0010-container-networking-plugins-ds.yml
@@ -55,13 +55,25 @@ spec:
           imagePullPolicy: IfNotPresent
           securityContext:
             privileged: true
+          {{- with index .RuntimeSpec.ContainerResources "cni-plugins" }}
+          resources:
+            {{- if .Requests }}
+            requests:
+              {{ .Requests | yaml | nindent 14}}
+            {{- end }}
+            {{- if .Limits }}
+            limits:
+              {{ .Limits | yaml | nindent 14}}
+            {{- end }}
+          {{- else }}
           resources:
             requests:
               cpu: "100m"
               memory: "50Mi"
             limits:
               cpu: "100m"
               memory: "50Mi"
+          {{- end }}
           volumeMounts:
             - name: cnibin
               mountPath: /host/opt/cni/bin

diff --git a/manifests/state-ib-kubernetes/0070-deployment.yaml b/manifests/state-ib-kubernetes/0070-deployment.yaml
@@ -71,10 +71,22 @@ spec:
           image: {{ .CrSpec.Repository }}/{{ .CrSpec.Image }}:{{ .CrSpec.Version }}
           imagePullPolicy: IfNotPresent
           command: ["/usr/bin/ib-kubernetes"]
+          {{- with index .RuntimeSpec.ContainerResources "ib-kubernetes" }}
+          resources:
+            {{- if .Requests }}
+            requests:
+              {{ .Requests | yaml | nindent 14}}
+            {{- end }}
+            {{- if .Limits }}
+            limits:
+              {{ .Limits | yaml | nindent 14}}
+            {{- end }}
+          {{- else }}
           resources:
             requests:
               cpu: 100m
               memory: 300Mi
+          {{- end }}
           env:
             - name: DAEMON_SM_PLUGIN
               value: "ufm"

diff --git a/manifests/state-ipoib-cni/0050-ipoib-cni-ds.yml b/manifests/state-ipoib-cni/0050-ipoib-cni-ds.yml
@@ -59,13 +59,25 @@ spec:
       containers:
         - name: ipoib-cni
           image: {{ .CrSpec.Repository }}/{{ .CrSpec.Image }}:{{ .CrSpec.Version }}
+          {{- with index .RuntimeSpec.ContainerResources "ipoib-cni" }}
+          resources:
+            {{- if .Requests }}
+            requests:
+              {{ .Requests | yaml | nindent 14}}
+            {{- end }}
+            {{- if .Limits }}
+            limits:
+              {{ .Limits | yaml | nindent 14}}
+            {{- end }}
+          {{- else }}
           resources:
             requests:
               cpu: "100m"
               memory: "50Mi"
             limits:
               cpu: "100m"
               memory: "50Mi"
+          {{- end }}
           securityContext:
             privileged: true
           volumeMounts:

diff --git a/manifests/state-multus-cni/0050-multus-ds.yml b/manifests/state-multus-cni/0050-multus-ds.yml
@@ -55,13 +55,25 @@ spec:
             preStop:
               exec:
                 command: ["/bin/sh", "-c", "rm -f /host/etc/cni/net.d/00-multus.conf"]
+          {{- with index .RuntimeSpec.ContainerResources "kube-multus" }}
+          resources:
+            {{- if .Requests }}
+            requests:
+              {{ .Requests | yaml | nindent 14}}
+            {{- end }}
+            {{- if .Limits }}
+            limits:
+              {{ .Limits | yaml | nindent 14}}
+            {{- end }}
+          {{- else }}
           resources:
             requests:
               cpu: "100m"
               memory: "50Mi"
             limits:
               cpu: "100m"
               memory: "50Mi"
+          {{- end }}
           securityContext:
             privileged: true
           volumeMounts:

diff --git a/manifests/state-nic-feature-discovery/030-nic-feature-discovery-ds.yaml b/manifests/state-nic-feature-discovery/030-nic-feature-discovery-ds.yaml
@@ -63,13 +63,25 @@ spec:
           args:
             - --v=0
             - --logging-format=json
+          {{- with index .RuntimeSpec.ContainerResources "nic-feature-discovery" }}
+          resources:
+            {{- if .Requests }}
+            requests:
+              {{ .Requests | yaml | nindent 14}}
+            {{- end }}
+            {{- if .Limits }}
+            limits:
+              {{ .Limits | yaml | nindent 14}}
+            {{- end }}
+          {{- else }}
           resources:
             requests:
               cpu: "100m"
               memory: "50Mi"
             limits:
               cpu: "300m"
               memory: "150Mi"
+          {{- end }}
           securityContext:
             privileged: true
           volumeMounts:

diff --git a/manifests/state-nv-ipam-cni/040-nv-ipam-controller.yaml b/manifests/state-nv-ipam-cni/040-nv-ipam-controller.yaml
@@ -119,10 +119,22 @@ spec:
               port: 8081
             initialDelaySeconds: 5
             periodSeconds: 10
+          {{- with index .RuntimeSpec.ContainerResources "nv-ipam-controller" }}
           resources:
+            {{- if .Requests }}
             requests:
-              cpu: 100m
-              memory: 300Mi
+              {{ .Requests | yaml | nindent 14}}
+            {{- end }}
+            {{- if .Limits }}
+            limits:
+              {{ .Limits | yaml | nindent 14}}
+            {{- end }}
+          {{- else }}
+          resources:
+            requests:
+              cpu: "100m"
+              memory: "300Mi"
+          {{- end }}
           volumeMounts:
           {{- if .CrSpec.EnableWebhook }}
             - mountPath: /tmp/k8s-webhook-server/serving-certs

diff --git a/manifests/state-nv-ipam-cni/040-nv-ipam-node.yaml b/manifests/state-nv-ipam-cni/040-nv-ipam-node.yaml
@@ -81,13 +81,25 @@ spec:
           - --cni-log-file=/var/log/nv-ipam-cni.log
           - --cni-log-level=info # log level for shim CNI
           - --ippools-namespace=$(IPPOOLS_NAMESPACE)
+        {{- with index .RuntimeSpec.ContainerResources "nv-ipam-node" }}
+          resources:
+            {{- if .Requests }}
+            requests:
+              {{ .Requests | yaml | nindent 14}}
+            {{- end }}
+            {{- if .Limits }}
+            limits:
+              {{ .Limits | yaml | nindent 14}}
+            {{- end }}
+        {{- else }}
         resources:
           requests:
             cpu: "100m"
             memory: "50Mi"
           limits:
             cpu: "300m"
             memory: "300Mi"
+        {{- end }}
         securityContext:
           privileged: true
         volumeMounts:

diff --git a/manifests/state-ofed-driver/0050_ofed-driver-ds.yaml b/manifests/state-ofed-driver/0050_ofed-driver-ds.yaml
@@ -112,6 +112,17 @@ spec:
               readOnly: {{ .ReadOnly }}
             {{- end }}
             {{- end }}
+          {{- with index .RuntimeSpec.ContainerResources "mofed-container" }}
+          resources:
+            {{- if .Requests }}
+            requests:
+              {{ .Requests | yaml | nindent 14}}
+            {{- end }}
+            {{- if .Limits }}
+            limits:
+              {{ .Limits | yaml | nindent 14}}
+            {{- end }}
+          {{- end }}
           startupProbe:
             exec:
               command:

diff --git a/manifests/state-rdma-device-plugin/0060_rdma-shared-dev-plugin-ds.yaml b/manifests/state-rdma-device-plugin/0060_rdma-shared-dev-plugin-ds.yaml
@@ -78,6 +78,17 @@ spec:
           - name: host-config-volume
             mountPath: /host/etc/pcidp/
           {{- end }}
+        {{- with index .RuntimeSpec.ContainerResources "rdma-shared-dp" }}
+        resources:
+          {{- if .Requests }}
+          requests:
+            {{ .Requests | yaml | nindent 12}}
+          {{- end }}
+          {{- if .Limits }}
+          limits:
+            {{ .Limits | yaml | nindent 12}}
+          {{- end }}
+        {{- end }}
       volumes:
         - name: device-plugin
           hostPath:

diff --git a/manifests/state-sriov-device-plugin/0030-sriov-dp-daemonset.yml b/manifests/state-sriov-device-plugin/0030-sriov-dp-daemonset.yml
@@ -93,6 +93,17 @@ spec:
             - name: host-config-volume
               mountPath: /host/etc/pcidp/
             {{- end}}
+          {{- with index .RuntimeSpec.ContainerResources "kube-sriovdp" }}
+          resources:
+            {{- if .Requests }}
+            requests:
+              {{ .Requests | yaml | nindent 12}}
+            {{- end }}
+            {{- if .Limits }}
+            limits:
+              {{ .Limits | yaml | nindent 12}}
+            {{- end }}
+          {{- end }}
       volumes:
         - name: devicesock
           hostPath:

diff --git a/manifests/state-whereabouts-cni/0050-whereabouts-ds.yaml b/manifests/state-whereabouts-cni/0050-whereabouts-ds.yaml
@@ -60,13 +60,25 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
+        {{- with index .RuntimeSpec.ContainerResources "whereabouts" }}
+        resources:
+          {{- if .Requests }}
+          requests:
+            {{ .Requests | yaml | nindent 12}}
+          {{- end }}
+          {{- if .Limits }}
+          limits:
+            {{ .Limits | yaml | nindent 12}}
+          {{- end }}
+        {{- else }}
         resources:
           requests:
             cpu: "100m"
             memory: "100Mi"
           limits:
             cpu: "100m"
             memory: "200Mi"
+        {{- end }}
         securityContext:
           privileged: true
         volumeMounts:

diff --git a/pkg/state/container_resources.go b/pkg/state/container_resources.go
@@ -0,0 +1,30 @@
+/*
+2023 NVIDIA CORPORATION & AFFILIATES
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package state
+
+import "github.com/Mellanox/network-operator/api/v1alpha1"
+
+type ContainerResourcesMap map[string]v1alpha1.ResourceRequirements
+
+func createContainerResourcesMap(resources []v1alpha1.ResourceRequirements) ContainerResourcesMap {
+	containerResources := ContainerResourcesMap{}
+	for _, val := range resources {
+		containerResources[val.Name] = val
+	}
+
+	return containerResources
+}
diff --git a/pkg/state/state_cni_plugins.go b/pkg/state/state_cni_plugins.go
@@ -131,6 +131,7 @@ func (s *stateCNIPlugins) GetWatchSources() map[string]client.Object {
 	return wr
 }
 
+//nolint:dupl
 func (s *stateCNIPlugins) getManifestObjects(
 	cr *mellanoxv1alpha1.NicClusterPolicy, staticConfig staticconfig.Provider,
 	reqLogger logr.Logger) ([]*unstructured.Unstructured, error) {
@@ -139,8 +140,9 @@ func (s *stateCNIPlugins) getManifestObjects(
 		Tolerations:  cr.Spec.Tolerations,
 		NodeAffinity: cr.Spec.NodeAffinity,
 		RuntimeSpec: &cniRuntimeSpec{
-			runtimeSpec:     runtimeSpec{config.FromEnv().State.NetworkOperatorResourceNamespace},
-			CniBinDirectory: utils.GetCniBinDirectory(staticConfig, nil),
+			runtimeSpec:        runtimeSpec{config.FromEnv().State.NetworkOperatorResourceNamespace},
+			CniBinDirectory:    utils.GetCniBinDirectory(staticConfig, nil),
+			ContainerResources: createContainerResourcesMap(cr.Spec.SecondaryNetwork.CniPlugins.ContainerResources),
 		},
 	}
 	// render objects