feat(api,ui,sdk): Make CPU limits configurable

.github/workflows/codecov-config/codecov.yml

@@ -0,0 +1,5 @@
+coverage:
+  status:
+    patch:
+      default:
+        threshold: 0.03%

.github/workflows/merlin.yml

@@ -141,6 +141,7 @@ jobs:
           name: sdk-test-${{ matrix.python-version }}
           token: ${{ secrets.CODECOV_TOKEN }}
           working-directory: ./python/sdk
+          codecov_yml_path: ../../.github/workflows/codecov-config/codecov.yml
 
   lint-api:
     runs-on: ubuntu-latest
@@ -197,6 +198,7 @@ jobs:
           name: api-test
           token: ${{ secrets.CODECOV_TOKEN }}
           working-directory: ./api
+          codecov_yml_path: ../.github/workflows/codecov-config/codecov.yml
 
   test-observation-publisher:
     runs-on: ubuntu-latest

api/cluster/resource/templater.go

@@ -194,20 +194,29 @@ func (t *InferenceServiceTemplater) CreateInferenceServiceSpec(modelService *mod
 func (t *InferenceServiceTemplater) createPredictorSpec(modelService *models.Service) (kservev1beta1.PredictorSpec, error) {
 	envVars := modelService.EnvVars
 
-	// Set resource limits to request * userContainerCPULimitRequestFactor or UserContainerMemoryLimitRequestFactor
+	// Set resource limits to request * userContainerCPULimitRequestFactor or userContainerMemoryLimitRequestFactor
 	limits := map[corev1.ResourceName]resource.Quantity{}
-	if t.deploymentConfig.UserContainerCPULimitRequestFactor != 0 {
-		limits[corev1.ResourceCPU] = ScaleQuantity(
-			modelService.ResourceRequest.CPURequest, t.deploymentConfig.UserContainerCPULimitRequestFactor,
-		)
-	} else {
-		// TODO: Remove this else-block when KServe finally allows default CPU limits to be removed
-		var err error
-		limits[corev1.ResourceCPU], err = resource.ParseQuantity(t.deploymentConfig.UserContainerCPUDefaultLimit)
-		if err != nil {
-			return kservev1beta1.PredictorSpec{}, err
+
+	// Set cpu resource limits automatically if they have not been set
+	if modelService.ResourceRequest.CPULimit == nil || modelService.ResourceRequest.CPULimit.IsZero() {
+		if t.deploymentConfig.UserContainerCPULimitRequestFactor != 0 {
+			limits[corev1.ResourceCPU] = ScaleQuantity(
+				modelService.ResourceRequest.CPURequest, t.deploymentConfig.UserContainerCPULimitRequestFactor,
+			)
+		} else {
+			// TODO: Remove this else-block when KServe finally allows default CPU limits to be removed
+			var err error
+			limits[corev1.ResourceCPU], err = resource.ParseQuantity(t.deploymentConfig.UserContainerCPUDefaultLimit)
+			if err != nil {
+				return kservev1beta1.PredictorSpec{}, err
+			}
+			// Set additional env vars to manage concurrency so model performance improves when no CPU limits are set
+			envVars = models.MergeEnvVars(ParseEnvVars(t.deploymentConfig.DefaultEnvVarsWithoutCPULimits), envVars)
 		}
+	} else {
+		limits[corev1.ResourceCPU] = *modelService.ResourceRequest.CPULimit
 	}
+
 	if t.deploymentConfig.UserContainerMemoryLimitRequestFactor != 0 {
 		limits[corev1.ResourceMemory] = ScaleQuantity(
 			modelService.ResourceRequest.MemoryRequest, t.deploymentConfig.UserContainerMemoryLimitRequestFactor,
@@ -329,7 +338,7 @@ func (t *InferenceServiceTemplater) createPredictorSpec(modelService *models.Ser
 		// 1. PyFunc default env
 		// 2. User environment variable
 		// 3. Default env variable that can be override by user environment
-		higherPriorityEnvVars := models.MergeEnvVars(modelService.EnvVars, pyfuncDefaultEnv)
+		higherPriorityEnvVars := models.MergeEnvVars(envVars, pyfuncDefaultEnv)
 		lowerPriorityEnvVars := models.EnvVars{}
 		if modelService.Protocol == protocol.UpiV1 {
 			lowerPriorityEnvVars = append(lowerPriorityEnvVars, models.EnvVar{Name: envGRPCOptions, Value: t.deploymentConfig.PyfuncGRPCOptions})
@@ -364,7 +373,7 @@ func (t *InferenceServiceTemplater) createPredictorSpec(modelService *models.Ser
 		}
 
 	case models.ModelTypeCustom:
-		predictorSpec = createCustomPredictorSpec(modelService, resources, nodeSelector, tolerations)
+		predictorSpec = createCustomPredictorSpec(modelService, envVars, resources, nodeSelector, tolerations)
 	}
 
 	if len(nodeSelector) > 0 {
@@ -392,28 +401,36 @@ func (t *InferenceServiceTemplater) createTransformerSpec(
 	modelService *models.Service,
 	transformer *models.Transformer,
 ) (*kservev1beta1.TransformerSpec, error) {
+	envVars := transformer.EnvVars
+
 	// Set resource limits to request * userContainerCPULimitRequestFactor or UserContainerMemoryLimitRequestFactor
 	limits := map[corev1.ResourceName]resource.Quantity{}
-	if t.deploymentConfig.UserContainerCPULimitRequestFactor != 0 {
-		limits[corev1.ResourceCPU] = ScaleQuantity(
-			transformer.ResourceRequest.CPURequest, t.deploymentConfig.UserContainerCPULimitRequestFactor,
-		)
-	} else {
-		// TODO: Remove this else-block when KServe finally allows default CPU limits to be removed
-		var err error
-		limits[corev1.ResourceCPU], err = resource.ParseQuantity(t.deploymentConfig.UserContainerCPUDefaultLimit)
-		if err != nil {
-			return nil, err
+	// Set cpu resource limits automatically if they have not been set
+	if transformer.ResourceRequest.CPULimit == nil || transformer.ResourceRequest.CPULimit.IsZero() {
+		if t.deploymentConfig.UserContainerCPULimitRequestFactor != 0 {
+			limits[corev1.ResourceCPU] = ScaleQuantity(
+				transformer.ResourceRequest.CPURequest, t.deploymentConfig.UserContainerCPULimitRequestFactor,
+			)
+		} else {
+			// TODO: Remove this else-block when KServe finally allows default CPU limits to be removed
+			var err error
+			limits[corev1.ResourceCPU], err = resource.ParseQuantity(t.deploymentConfig.UserContainerCPUDefaultLimit)
+			if err != nil {
+				return nil, err
+			}
+			// Set additional env vars to manage concurrency so model performance improves when no CPU limits are set
+			envVars = models.MergeEnvVars(ParseEnvVars(t.deploymentConfig.DefaultEnvVarsWithoutCPULimits), envVars)
 		}
+	} else {
+		limits[corev1.ResourceCPU] = *transformer.ResourceRequest.CPULimit
 	}
+
 	if t.deploymentConfig.UserContainerMemoryLimitRequestFactor != 0 {
 		limits[corev1.ResourceMemory] = ScaleQuantity(
 			transformer.ResourceRequest.MemoryRequest, t.deploymentConfig.UserContainerMemoryLimitRequestFactor,
 		)
 	}
 
-	envVars := transformer.EnvVars
-
 	// Put in defaults if not provided by users (user's input is used)
 	if transformer.TransformerType == models.StandardTransformerType {
 		transformer.Image = t.deploymentConfig.StandardTransformer.ImageName
@@ -780,9 +797,13 @@ func createDefaultPredictorEnvVars(modelService *models.Service) models.EnvVars
 	return defaultEnvVars
 }
 
-func createCustomPredictorSpec(modelService *models.Service, resources corev1.ResourceRequirements, nodeSelector map[string]string, tolerations []corev1.Toleration) kservev1beta1.PredictorSpec {
-	envVars := modelService.EnvVars
-
+func createCustomPredictorSpec(
+	modelService *models.Service,
+	envVars models.EnvVars,
+	resources corev1.ResourceRequirements,
+	nodeSelector map[string]string,
+	tolerations []corev1.Toleration,
+) kservev1beta1.PredictorSpec {
 	// Add default env var (Overwrite by user not allowed)
 	defaultEnvVar := createDefaultPredictorEnvVars(modelService)
 	envVars = models.MergeEnvVars(envVars, defaultEnvVar)
@@ -910,3 +931,11 @@ func (t *InferenceServiceTemplater) applyDefaults(service *models.Service) {
 		}
 	}
 }
+
+func ParseEnvVars(envVars []corev1.EnvVar) models.EnvVars {
+	var parsedEnvVars models.EnvVars
+	for _, envVar := range envVars {
+		parsedEnvVars = append(parsedEnvVars, models.EnvVar{Name: envVar.Name, Value: envVar.Value})
+	}
+	return parsedEnvVars
+}

api/cluster/resource/templater_gpu_test.go

@@ -58,7 +58,7 @@ var (
 			"nvidia.com/gpu":      resource.MustParse("1"),
 		},
 		Limits: corev1.ResourceList{
-			corev1.ResourceCPU:    resource.MustParse("8"),
+			corev1.ResourceCPU:    resource.MustParse("10"),
 			corev1.ResourceMemory: ScaleQuantity(defaultModelResourceRequests.MemoryRequest, 2),
 			"nvidia.com/gpu":      resource.MustParse("1"),
 		},