From 03baa777decac382ca24639a574c50f59496c829 Mon Sep 17 00:00:00 2001 From: ewezy Date: Fri, 17 May 2024 11:11:16 +0800 Subject: [PATCH 01/26] Add new unit test helper method to sort env vars in isvc specs --- api/cluster/resource/templater_test.go | 36 ++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/api/cluster/resource/templater_test.go b/api/cluster/resource/templater_test.go index ba3a1df90..78806279f 100644 --- a/api/cluster/resource/templater_test.go +++ b/api/cluster/resource/templater_test.go @@ -4482,3 +4482,39 @@ func createPyFuncPublisherEnvVars(svc *models.Service, pyfuncPublisher config.Py } return envVars } + +func sortInferenceServiceSpecEnvVars(isvc kservev1beta1.InferenceServiceSpec) { + // Sort env vars in predictor + if isvc.Predictor.SKLearn != nil { + sort.Slice(isvc.Predictor.SKLearn.Env, func(i, j int) bool { + return isvc.Predictor.SKLearn.Env[i].Name < isvc.Predictor.SKLearn.Env[j].Name + }) + } else if isvc.Predictor.XGBoost != nil { + sort.Slice(isvc.Predictor.XGBoost.Env, func(i, j int) bool { + return isvc.Predictor.XGBoost.Env[i].Name < isvc.Predictor.XGBoost.Env[j].Name + }) + } else if isvc.Predictor.Tensorflow != nil { + sort.Slice(isvc.Predictor.Tensorflow.Env, func(i, j int) bool { + return isvc.Predictor.Tensorflow.Env[i].Name < isvc.Predictor.Tensorflow.Env[j].Name + }) + } else if isvc.Predictor.PyTorch != nil { + sort.Slice(isvc.Predictor.PyTorch.Env, func(i, j int) bool { + return isvc.Predictor.PyTorch.Env[i].Name < isvc.Predictor.PyTorch.Env[j].Name + }) + } else if isvc.Predictor.PodSpec.Containers != nil { + for _, c := range isvc.Predictor.PodSpec.Containers { + sort.Slice(c.Env, func(i, j int) bool { + return c.Env[i].Name < c.Env[j].Name + }) + } + } + + // Sort env vars in transformer + if isvc.Transformer != nil && isvc.Transformer.PodSpec.Containers != nil { + for _, c := range isvc.Transformer.PodSpec.Containers { + sort.Slice(c.Env, func(i, j int) bool { + return c.Env[i].Name < c.Env[j].Name + }) + } + } +} From ffd4e9ab81833e89c39194729af8a1f941c51776 Mon Sep 17 00:00:00 2001 From: ewezy Date: Fri, 17 May 2024 11:11:58 +0800 Subject: [PATCH 02/26] Extend merge env vars helper function to merge more than 2 sets of env vars --- api/models/env_var.go | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/api/models/env_var.go b/api/models/env_var.go index 379e95982..be50579f7 100644 --- a/api/models/env_var.go +++ b/api/models/env_var.go @@ -69,19 +69,21 @@ func (evs EnvVars) ToKubernetesEnvVars() []v1.EnvVar { return kubeEnvVars } -// MergeEnvVars merges two environment variables and return the merging result. -// Both `left` and `right` EnvVars value will be not mutated. -// `right` EnvVars has higher precedence. -func MergeEnvVars(left, right EnvVars) EnvVars { - envIndexMap := make(map[string]int, len(left)+len(right)) - for index, ev := range left { - envIndexMap[ev.Name] = index - } - for _, add := range right { - if index, exist := envIndexMap[add.Name]; exist { - left[index].Value = add.Value - } else { - left = append(left, add) +// MergeEnvVars merges multiple sets of environment variables and return the merging result. +// All the EnvVars passed as arguments will be not mutated. +// EnvVars to the right have higher precedence. +func MergeEnvVars(left EnvVars, rightEnvVars ...EnvVars) EnvVars { + for _, right := range rightEnvVars { + envIndexMap := make(map[string]int, len(left)+len(right)) + for index, ev := range left { + envIndexMap[ev.Name] = index + } + for _, add := range right { + if index, exist := envIndexMap[add.Name]; exist { + left[index].Value = add.Value + } else { + left = append(left, add) + } } } return left From 3c9306b37a0da411d75aff84fcf2473a3d1ba0da Mon Sep 17 00:00:00 2001 From: ewezy Date: Fri, 17 May 2024 11:17:02 +0800 Subject: [PATCH 03/26] Add new configs to set default env vars when cpu limits are not specified --- api/cluster/resource/templater.go | 86 +++++++++++++++++++++---------- api/config/config.go | 8 +-- api/config/deployment.go | 2 + api/config/environment.go | 1 + api/models/resource_request.go | 2 + 5 files changed, 68 insertions(+), 31 deletions(-) diff --git a/api/cluster/resource/templater.go b/api/cluster/resource/templater.go index a44271fbe..d30ed9677 100644 --- a/api/cluster/resource/templater.go +++ b/api/cluster/resource/templater.go @@ -21,6 +21,7 @@ import ( "strconv" "strings" + "github.com/caraml-dev/merlin/client" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" kserveconstant "github.com/kserve/kserve/pkg/constants" "github.com/mitchellh/copystructure" @@ -194,20 +195,29 @@ func (t *InferenceServiceTemplater) CreateInferenceServiceSpec(modelService *mod func (t *InferenceServiceTemplater) createPredictorSpec(modelService *models.Service) (kservev1beta1.PredictorSpec, error) { envVars := modelService.EnvVars - // Set resource limits to request * userContainerCPULimitRequestFactor or UserContainerMemoryLimitRequestFactor + // Set resource limits to request * userContainerCPULimitRequestFactor or userContainerMemoryLimitRequestFactor limits := map[corev1.ResourceName]resource.Quantity{} - if t.deploymentConfig.UserContainerCPULimitRequestFactor != 0 { - limits[corev1.ResourceCPU] = ScaleQuantity( - modelService.ResourceRequest.CPURequest, t.deploymentConfig.UserContainerCPULimitRequestFactor, - ) - } else { - // TODO: Remove this else-block when KServe finally allows default CPU limits to be removed - var err error - limits[corev1.ResourceCPU], err = resource.ParseQuantity(t.deploymentConfig.UserContainerCPUDefaultLimit) - if err != nil { - return kservev1beta1.PredictorSpec{}, err + + // Set cpu resource limits automatically if they have not been set + if modelService.ResourceRequest.CPULimit.IsZero() { + if t.deploymentConfig.UserContainerCPULimitRequestFactor != 0 { + limits[corev1.ResourceCPU] = ScaleQuantity( + modelService.ResourceRequest.CPURequest, t.deploymentConfig.UserContainerCPULimitRequestFactor, + ) + } else { + // TODO: Remove this else-block when KServe finally allows default CPU limits to be removed + var err error + limits[corev1.ResourceCPU], err = resource.ParseQuantity(t.deploymentConfig.UserContainerCPUDefaultLimit) + if err != nil { + return kservev1beta1.PredictorSpec{}, err + } + // Set additional env vars to manage concurrency so model performance improves when no CPU limits are set + envVars = models.MergeEnvVars(ParseEnvVars(t.deploymentConfig.DefaultEnvVarsWithoutCPULimits), envVars) } + } else { + limits[corev1.ResourceCPU] = modelService.ResourceRequest.CPULimit } + if t.deploymentConfig.UserContainerMemoryLimitRequestFactor != 0 { limits[corev1.ResourceMemory] = ScaleQuantity( modelService.ResourceRequest.MemoryRequest, t.deploymentConfig.UserContainerMemoryLimitRequestFactor, @@ -329,7 +339,7 @@ func (t *InferenceServiceTemplater) createPredictorSpec(modelService *models.Ser // 1. PyFunc default env // 2. User environment variable // 3. Default env variable that can be override by user environment - higherPriorityEnvVars := models.MergeEnvVars(modelService.EnvVars, pyfuncDefaultEnv) + higherPriorityEnvVars := models.MergeEnvVars(envVars, pyfuncDefaultEnv) lowerPriorityEnvVars := models.EnvVars{} if modelService.Protocol == protocol.UpiV1 { lowerPriorityEnvVars = append(lowerPriorityEnvVars, models.EnvVar{Name: envGRPCOptions, Value: t.deploymentConfig.PyfuncGRPCOptions}) @@ -364,7 +374,7 @@ func (t *InferenceServiceTemplater) createPredictorSpec(modelService *models.Ser } case models.ModelTypeCustom: - predictorSpec = createCustomPredictorSpec(modelService, resources, nodeSelector, tolerations) + predictorSpec = createCustomPredictorSpec(modelService, envVars, resources, nodeSelector, tolerations) } if len(nodeSelector) > 0 { @@ -392,28 +402,36 @@ func (t *InferenceServiceTemplater) createTransformerSpec( modelService *models.Service, transformer *models.Transformer, ) (*kservev1beta1.TransformerSpec, error) { + envVars := transformer.EnvVars + // Set resource limits to request * userContainerCPULimitRequestFactor or UserContainerMemoryLimitRequestFactor limits := map[corev1.ResourceName]resource.Quantity{} - if t.deploymentConfig.UserContainerCPULimitRequestFactor != 0 { - limits[corev1.ResourceCPU] = ScaleQuantity( - transformer.ResourceRequest.CPURequest, t.deploymentConfig.UserContainerCPULimitRequestFactor, - ) - } else { - // TODO: Remove this else-block when KServe finally allows default CPU limits to be removed - var err error - limits[corev1.ResourceCPU], err = resource.ParseQuantity(t.deploymentConfig.UserContainerCPUDefaultLimit) - if err != nil { - return nil, err + // Set cpu resource limits automatically if they have not been set + if transformer.ResourceRequest.CPULimit.IsZero() { + if t.deploymentConfig.UserContainerCPULimitRequestFactor != 0 { + limits[corev1.ResourceCPU] = ScaleQuantity( + transformer.ResourceRequest.CPURequest, t.deploymentConfig.UserContainerCPULimitRequestFactor, + ) + } else { + // TODO: Remove this else-block when KServe finally allows default CPU limits to be removed + var err error + limits[corev1.ResourceCPU], err = resource.ParseQuantity(t.deploymentConfig.UserContainerCPUDefaultLimit) + if err != nil { + return nil, err + } + // Set additional env vars to manage concurrency so model performance improves when no CPU limits are set + envVars = models.MergeEnvVars(ParseEnvVars(t.deploymentConfig.DefaultEnvVarsWithoutCPULimits), envVars) } + } else { + limits[corev1.ResourceCPU] = transformer.ResourceRequest.CPULimit } + if t.deploymentConfig.UserContainerMemoryLimitRequestFactor != 0 { limits[corev1.ResourceMemory] = ScaleQuantity( transformer.ResourceRequest.MemoryRequest, t.deploymentConfig.UserContainerMemoryLimitRequestFactor, ) } - envVars := transformer.EnvVars - // Put in defaults if not provided by users (user's input is used) if transformer.TransformerType == models.StandardTransformerType { transformer.Image = t.deploymentConfig.StandardTransformer.ImageName @@ -780,9 +798,13 @@ func createDefaultPredictorEnvVars(modelService *models.Service) models.EnvVars return defaultEnvVars } -func createCustomPredictorSpec(modelService *models.Service, resources corev1.ResourceRequirements, nodeSelector map[string]string, tolerations []corev1.Toleration) kservev1beta1.PredictorSpec { - envVars := modelService.EnvVars - +func createCustomPredictorSpec( + modelService *models.Service, + envVars models.EnvVars, + resources corev1.ResourceRequirements, + nodeSelector map[string]string, + tolerations []corev1.Toleration, +) kservev1beta1.PredictorSpec { // Add default env var (Overwrite by user not allowed) defaultEnvVar := createDefaultPredictorEnvVars(modelService) envVars = models.MergeEnvVars(envVars, defaultEnvVar) @@ -910,3 +932,11 @@ func (t *InferenceServiceTemplater) applyDefaults(service *models.Service) { } } } + +func ParseEnvVars(envVars []client.EnvVar) models.EnvVars { + var parsedEnvVars models.EnvVars + for _, envVar := range envVars { + parsedEnvVars = append(parsedEnvVars, models.EnvVar{Name: *envVar.Name, Value: *envVar.Value}) + } + return parsedEnvVars +} diff --git a/api/config/config.go b/api/config/config.go index 0417d16c9..02a520cea 100644 --- a/api/config/config.go +++ b/api/config/config.go @@ -21,6 +21,7 @@ import ( "strings" "time" + "github.com/caraml-dev/merlin/client" mlpcluster "github.com/caraml-dev/mlp/api/pkg/cluster" "github.com/caraml-dev/mlp/api/pkg/instrumentation/newrelic" "github.com/caraml-dev/mlp/api/pkg/instrumentation/sentry" @@ -360,9 +361,10 @@ type PyFuncPublisherConfig struct { // KServe inference services type InferenceServiceDefaults struct { // TODO: Remove UserContainerCPUDefaultLimit when KServe finally allows default CPU limits to be removed - UserContainerCPUDefaultLimit string `json:"userContainerCPUDefaultLimit" default:"8"` - UserContainerCPULimitRequestFactor float64 `json:"userContainerLimitCPURequestFactor" default:"0"` - UserContainerMemoryLimitRequestFactor float64 `json:"userContainerLimitMemoryRequestFactor" default:"2"` + UserContainerCPUDefaultLimit string `json:"userContainerCPUDefaultLimit" default:"8"` + UserContainerCPULimitRequestFactor float64 `json:"userContainerLimitCPURequestFactor" default:"0"` + UserContainerMemoryLimitRequestFactor float64 `json:"userContainerLimitMemoryRequestFactor" default:"2"` + DefaultEnvVarsWithoutCPULimits []client.EnvVar `json:"defaultEnvVarsWithoutCPULimits"` } // SimulationFeastConfig feast config that aimed to be used only for simulation of standard transformer diff --git a/api/config/deployment.go b/api/config/deployment.go index 15bf67742..727890dda 100644 --- a/api/config/deployment.go +++ b/api/config/deployment.go @@ -17,6 +17,7 @@ package config import ( "time" + "github.com/caraml-dev/merlin/client" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" ) @@ -55,6 +56,7 @@ type DeploymentConfig struct { UserContainerCPUDefaultLimit string UserContainerCPULimitRequestFactor float64 UserContainerMemoryLimitRequestFactor float64 + DefaultEnvVarsWithoutCPULimits []client.EnvVar } type ResourceRequests struct { diff --git a/api/config/environment.go b/api/config/environment.go index 78292a3c8..b06962f82 100644 --- a/api/config/environment.go +++ b/api/config/environment.go @@ -189,5 +189,6 @@ func ParseDeploymentConfig(envCfg *EnvironmentConfig, cfg *Config) DeploymentCon UserContainerCPUDefaultLimit: cfg.InferenceServiceDefaults.UserContainerCPUDefaultLimit, UserContainerCPULimitRequestFactor: cfg.InferenceServiceDefaults.UserContainerCPULimitRequestFactor, UserContainerMemoryLimitRequestFactor: cfg.InferenceServiceDefaults.UserContainerMemoryLimitRequestFactor, + DefaultEnvVarsWithoutCPULimits: cfg.InferenceServiceDefaults.DefaultEnvVarsWithoutCPULimits, } } diff --git a/api/models/resource_request.go b/api/models/resource_request.go index cdc7dfac3..803301b2f 100644 --- a/api/models/resource_request.go +++ b/api/models/resource_request.go @@ -29,6 +29,8 @@ type ResourceRequest struct { MaxReplica int `json:"max_replica"` // CPU request of inference service CPURequest resource.Quantity `json:"cpu_request"` + // CPU limit of inference service + CPULimit resource.Quantity `json:"cpu_limit"` // Memory request of inference service MemoryRequest resource.Quantity `json:"memory_request"` // GPU name From fb3669d9f78e0023319fa982fc4edabfb10178fa Mon Sep 17 00:00:00 2001 From: ewezy Date: Fri, 17 May 2024 11:17:29 +0800 Subject: [PATCH 04/26] Add unit tests to test parsing of default env vars when cpu limits are not set --- api/cluster/resource/templater_gpu_test.go | 2 +- api/cluster/resource/templater_test.go | 505 ++++++++++++++++++--- 2 files changed, 436 insertions(+), 71 deletions(-) diff --git a/api/cluster/resource/templater_gpu_test.go b/api/cluster/resource/templater_gpu_test.go index 3daef7242..36ab706a2 100644 --- a/api/cluster/resource/templater_gpu_test.go +++ b/api/cluster/resource/templater_gpu_test.go @@ -58,7 +58,7 @@ var ( "nvidia.com/gpu": resource.MustParse("1"), }, Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("8"), + corev1.ResourceCPU: resource.MustParse("10"), corev1.ResourceMemory: ScaleQuantity(defaultModelResourceRequests.MemoryRequest, 2), "nvidia.com/gpu": resource.MustParse("1"), }, diff --git a/api/cluster/resource/templater_test.go b/api/cluster/resource/templater_test.go index 78806279f..6b8a31d27 100644 --- a/api/cluster/resource/templater_test.go +++ b/api/cluster/resource/templater_test.go @@ -18,10 +18,12 @@ import ( "bytes" "encoding/json" "fmt" + "sort" "strconv" "testing" "time" + "github.com/caraml-dev/merlin/client" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" kserveconstant "github.com/kserve/kserve/pkg/constants" "github.com/stretchr/testify/assert" @@ -59,7 +61,7 @@ var ( corev1.ResourceMemory: defaultModelResourceRequests.MemoryRequest, }, Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("8"), + corev1.ResourceCPU: resource.MustParse("10"), corev1.ResourceMemory: ScaleQuantity( defaultModelResourceRequests.MemoryRequest, userContainerMemoryLimitRequestFactor, @@ -80,7 +82,7 @@ var ( corev1.ResourceMemory: defaultTransformerResourceRequests.MemoryRequest, }, Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("8"), + corev1.ResourceCPU: resource.MustParse("10"), corev1.ResourceMemory: ScaleQuantity(defaultTransformerResourceRequests.MemoryRequest, 2), }, } @@ -92,17 +94,36 @@ var ( MemoryRequest: resource.MustParse("1Gi"), } + userResourceRequestsWithCPULimits = &models.ResourceRequest{ + MinReplica: 1, + MaxReplica: 10, + CPURequest: resource.MustParse("1"), + MemoryRequest: resource.MustParse("1Gi"), + CPULimit: resource.MustParse("8"), + } + expUserResourceRequests = corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: userResourceRequests.CPURequest, corev1.ResourceMemory: userResourceRequests.MemoryRequest, }, Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("8"), + corev1.ResourceCPU: resource.MustParse("10"), corev1.ResourceMemory: ScaleQuantity(userResourceRequests.MemoryRequest, 2), }, } + expUserResourceRequestsWithCPULimits = corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: userResourceRequestsWithCPULimits.CPURequest, + corev1.ResourceMemory: userResourceRequestsWithCPULimits.MemoryRequest, + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: userResourceRequestsWithCPULimits.CPULimit, + corev1.ResourceMemory: ScaleQuantity(userResourceRequestsWithCPULimits.MemoryRequest, 2), + }, + } + testPredictorScale, testTransformerScale = 3, 5 defaultDeploymentScale = DeploymentScale{ @@ -171,9 +192,23 @@ var ( SamplingRatioRate: 0.1, } - userContainerCPUDefaultLimit = "8" + userContainerCPUDefaultLimit = "10" userContainerCPULimitRequestFactor = float64(0) userContainerMemoryLimitRequestFactor = float64(2) + + defaultWorkersEnvVarName = "WORKERS" + defaultWorkersEnvVarValue = "2" + defaultEnvVarsWithoutCPULimits = []client.EnvVar{ + { + Name: &defaultWorkersEnvVarName, + Value: &defaultWorkersEnvVarValue, + }, + } + + expDefaultEnvVarWithoutCPULimits = corev1.EnvVar{ + Name: defaultWorkersEnvVarName, + Value: defaultWorkersEnvVarValue, + } ) func TestCreateInferenceServiceSpec(t *testing.T) { @@ -311,7 +346,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{}, + Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, }, }, }, @@ -375,6 +410,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, Env: []corev1.EnvVar{ + expDefaultEnvVarWithoutCPULimits, { Name: "env1", Value: "env1Value", }, @@ -436,7 +472,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{}, + Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, }, }, }, @@ -492,7 +528,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{}, + Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, }, }, }, @@ -545,7 +581,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{}, + Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, }, }, }, @@ -600,7 +636,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{}, + Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, }, }, }, @@ -655,7 +691,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{}, + Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, }, }, }, @@ -700,7 +736,6 @@ func TestCreateInferenceServiceSpec(t *testing.T) { "sample": "true", }, }, - Spec: kservev1beta1.InferenceServiceSpec{ Predictor: kservev1beta1.PredictorSpec{ PyTorch: &kservev1beta1.TorchServeSpec{ @@ -710,7 +745,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{}, + Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, }, }, }, @@ -722,6 +757,66 @@ func TestCreateInferenceServiceSpec(t *testing.T) { }, }, }, + { + name: "pyfunc spec with cpu limits specified", + modelSvc: &models.Service{ + Name: modelSvc.Name, + ModelName: modelSvc.ModelName, + ModelVersion: modelSvc.ModelVersion, + Namespace: project.Name, + ArtifactURI: modelSvc.ArtifactURI, + Type: models.ModelTypePyFunc, + Options: &models.ModelOption{ + PyFuncImageName: "gojek/project-model:1", + }, + ResourceRequest: userResourceRequestsWithCPULimits, + Metadata: modelSvc.Metadata, + Protocol: protocol.HttpJson, + }, + resourcePercentage: queueResourcePercentage, + deploymentScale: defaultDeploymentScale, + exp: &kservev1beta1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{ + Name: modelSvc.Name, + Namespace: project.Name, + Annotations: map[string]string{ + knserving.QueueSidecarResourcePercentageAnnotationKey: queueResourcePercentage, + "prometheus.io/scrape": "true", + "prometheus.io/port": "8080", + kserveconstant.DeploymentMode: string(kserveconstant.Serverless), + knautoscaling.InitialScaleAnnotationKey: fmt.Sprint(testPredictorScale), + }, + Labels: map[string]string{ + "gojek.com/app": modelSvc.Metadata.App, + "gojek.com/component": models.ComponentModelVersion, + "gojek.com/environment": testEnvironmentName, + "gojek.com/orchestrator": testOrchestratorName, + "gojek.com/stream": modelSvc.Metadata.Stream, + "gojek.com/team": modelSvc.Metadata.Team, + "sample": "true", + }, + }, + Spec: kservev1beta1.InferenceServiceSpec{ + Predictor: kservev1beta1.PredictorSpec{ + PodSpec: kservev1beta1.PodSpec{ + Containers: []corev1.Container{ + { + Name: kserveconstant.InferenceServiceContainerName, + Image: "gojek/project-model:1", + Env: createPyFuncDefaultEnvVarsWithProtocol(modelSvc, protocol.HttpJson).ToKubernetesEnvVars(), + Resources: expUserResourceRequestsWithCPULimits, + LivenessProbe: probeConfig, + }, + }, + }, + ComponentExtensionSpec: kservev1beta1.ComponentExtensionSpec{ + MinReplicas: &userResourceRequestsWithCPULimits.MinReplica, + MaxReplicas: userResourceRequestsWithCPULimits.MaxReplica, + }, + }, + }, + }, + }, { name: "pyfunc spec with liveness probe disabled", modelSvc: &models.Service{ @@ -768,8 +863,12 @@ func TestCreateInferenceServiceSpec(t *testing.T) { { Name: kserveconstant.InferenceServiceContainerName, Image: "gojek/project-model:1", - Env: models.MergeEnvVars(models.EnvVars{models.EnvVar{Name: envOldDisableLivenessProbe, Value: "true"}}, - createPyFuncDefaultEnvVarsWithProtocol(modelSvc, protocol.HttpJson)).ToKubernetesEnvVars(), + Env: models.MergeEnvVars(models.EnvVars{ + {Name: defaultWorkersEnvVarName, Value: defaultWorkersEnvVarValue}, + {Name: envOldDisableLivenessProbe, Value: "true"}, + }, + createPyFuncDefaultEnvVarsWithProtocol(modelSvc, protocol.HttpJson), + ).ToKubernetesEnvVars(), Resources: expDefaultModelResourceRequests, }, }, @@ -848,9 +947,15 @@ func TestCreateInferenceServiceSpec(t *testing.T) { PodSpec: kservev1beta1.PodSpec{ Containers: []corev1.Container{ { - Name: kserveconstant.InferenceServiceContainerName, - Image: "gojek/project-model:1", - Env: models.MergeEnvVars(createPyFuncPublisherEnvVars(modelSvcWithSchema, pyfuncPublisherConfig), createPyFuncDefaultEnvVarsWithProtocol(modelSvcWithSchema, protocol.HttpJson)).ToKubernetesEnvVars(), + Name: kserveconstant.InferenceServiceContainerName, + Image: "gojek/project-model:1", + Env: models.MergeEnvVars( + models.EnvVars{ + {Name: defaultWorkersEnvVarName, Value: defaultWorkersEnvVarValue}, + }, + createPyFuncPublisherEnvVars(modelSvcWithSchema, pyfuncPublisherConfig), + createPyFuncDefaultEnvVarsWithProtocol(modelSvcWithSchema, protocol.HttpJson), + ).ToKubernetesEnvVars(), Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, }, @@ -936,9 +1041,16 @@ func TestCreateInferenceServiceSpec(t *testing.T) { PodSpec: kservev1beta1.PodSpec{ Containers: []corev1.Container{ { - Name: kserveconstant.InferenceServiceContainerName, - Image: "gojek/project-model:1", - Env: models.MergeEnvVars(createPyFuncPublisherEnvVars(modelSvcWithSchema, pyfuncPublisherConfig), models.MergeEnvVars(models.EnvVars{{Name: envPublisherSamplingRatio, Value: "0.5"}}, createPyFuncDefaultEnvVarsWithProtocol(modelSvcWithSchema, protocol.HttpJson))).ToKubernetesEnvVars(), + Name: kserveconstant.InferenceServiceContainerName, + Image: "gojek/project-model:1", + Env: models.MergeEnvVars( + createPyFuncPublisherEnvVars(modelSvcWithSchema, pyfuncPublisherConfig), + models.EnvVars{ + {Name: defaultWorkersEnvVarName, Value: defaultWorkersEnvVarValue}, + {Name: envPublisherSamplingRatio, Value: "0.5"}, + }, + createPyFuncDefaultEnvVarsWithProtocol(modelSvcWithSchema, protocol.HttpJson), + ).ToKubernetesEnvVars(), Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, }, @@ -996,9 +1108,15 @@ func TestCreateInferenceServiceSpec(t *testing.T) { PodSpec: kservev1beta1.PodSpec{ Containers: []corev1.Container{ { - Name: kserveconstant.InferenceServiceContainerName, - Image: "gojek/project-model:1", - Env: models.MergeEnvVars(createPyFuncPublisherEnvVars(modelSvc, pyfuncPublisherConfig), createPyFuncDefaultEnvVarsWithProtocol(modelSvc, protocol.HttpJson)).ToKubernetesEnvVars(), + Name: kserveconstant.InferenceServiceContainerName, + Image: "gojek/project-model:1", + Env: models.MergeEnvVars( + models.EnvVars{ + {Name: defaultWorkersEnvVarName, Value: defaultWorkersEnvVarValue}, + }, + createPyFuncPublisherEnvVars(modelSvc, pyfuncPublisherConfig), + createPyFuncDefaultEnvVarsWithProtocol(modelSvc, protocol.HttpJson), + ).ToKubernetesEnvVars(), Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, }, @@ -1058,7 +1176,11 @@ func TestCreateInferenceServiceSpec(t *testing.T) { { Name: kserveconstant.InferenceServiceContainerName, Image: "gojek/project-model:1", - Env: models.MergeEnvVars(models.EnvVars{models.EnvVar{Name: envDisableLivenessProbe, Value: "true"}}, + Env: models.MergeEnvVars( + models.EnvVars{ + {Name: defaultWorkersEnvVarName, Value: defaultWorkersEnvVarValue}, + {Name: envDisableLivenessProbe, Value: "true"}, + }, createPyFuncDefaultEnvVarsWithProtocol(modelSvc, protocol.HttpJson)).ToKubernetesEnvVars(), Resources: expDefaultModelResourceRequests, }, @@ -1114,7 +1236,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expUserResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{}, + Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, }, }, }, @@ -1176,9 +1298,14 @@ func TestCreateInferenceServiceSpec(t *testing.T) { PodSpec: kservev1beta1.PodSpec{ Containers: []corev1.Container{ { - Name: kserveconstant.InferenceServiceContainerName, - Image: "gcr.io/custom-model:v0.1", - Env: createDefaultPredictorEnvVars(modelSvc).ToKubernetesEnvVars(), + Name: kserveconstant.InferenceServiceContainerName, + Image: "gcr.io/custom-model:v0.1", + Env: models.MergeEnvVars( + models.EnvVars{ + {Name: defaultWorkersEnvVarName, Value: defaultWorkersEnvVarValue}, + }, + createDefaultPredictorEnvVars(modelSvc), + ).ToKubernetesEnvVars(), Resources: expDefaultModelResourceRequests, }, }, @@ -1237,9 +1364,14 @@ func TestCreateInferenceServiceSpec(t *testing.T) { PodSpec: kservev1beta1.PodSpec{ Containers: []corev1.Container{ { - Name: kserveconstant.InferenceServiceContainerName, - Image: "gcr.io/custom-model:v0.1", - Env: createDefaultPredictorEnvVars(modelSvc).ToKubernetesEnvVars(), + Name: kserveconstant.InferenceServiceContainerName, + Image: "gcr.io/custom-model:v0.1", + Env: models.MergeEnvVars( + models.EnvVars{ + {Name: defaultWorkersEnvVarName, Value: defaultWorkersEnvVarValue}, + }, + createDefaultPredictorEnvVars(modelSvc), + ).ToKubernetesEnvVars(), Resources: expUserResourceRequests, Command: []string{ "./run.sh", @@ -1309,7 +1441,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{}, + Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, }, }, }, @@ -1393,7 +1525,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{}, + Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, }, }, }, @@ -1456,7 +1588,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{}, + Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, }, }, }, @@ -1520,7 +1652,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expUserResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{}, + Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, }, }, }, @@ -1583,7 +1715,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{}, + Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, }, }, }, @@ -1668,7 +1800,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{}, + Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, }, }, }, @@ -1723,7 +1855,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, Ports: grpcServerlessContainerPorts, - Env: []corev1.EnvVar{}, + Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, LivenessProbe: probeConfigUPI, }, }, @@ -1779,11 +1911,17 @@ func TestCreateInferenceServiceSpec(t *testing.T) { PodSpec: kservev1beta1.PodSpec{ Containers: []corev1.Container{ { - Name: kserveconstant.InferenceServiceContainerName, - Image: "gojek/project-model:1", - Resources: expDefaultModelResourceRequests, - Ports: grpcServerlessContainerPorts, - Env: models.MergeEnvVars(models.EnvVars{models.EnvVar{Name: envGRPCOptions, Value: "{}"}}, createPyFuncDefaultEnvVarsWithProtocol(modelSvc, protocol.UpiV1)).ToKubernetesEnvVars(), + Name: kserveconstant.InferenceServiceContainerName, + Image: "gojek/project-model:1", + Resources: expDefaultModelResourceRequests, + Ports: grpcServerlessContainerPorts, + Env: models.MergeEnvVars( + models.EnvVars{ + {Name: defaultWorkersEnvVarName, Value: defaultWorkersEnvVarValue}, + {Name: envGRPCOptions, Value: "{}"}, + }, + createPyFuncDefaultEnvVarsWithProtocol(modelSvc, protocol.UpiV1), + ).ToKubernetesEnvVars(), LivenessProbe: probeConfigUPI, }, }, @@ -1839,7 +1977,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, Ports: grpcServerlessContainerPorts, - Env: []corev1.EnvVar{}, + Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, LivenessProbe: probeConfigUPI, }, }, @@ -1902,9 +2040,14 @@ func TestCreateInferenceServiceSpec(t *testing.T) { PodSpec: kservev1beta1.PodSpec{ Containers: []corev1.Container{ { - Name: kserveconstant.InferenceServiceContainerName, - Image: "gcr.io/custom-model:v0.1", - Env: createDefaultPredictorEnvVars(modelSvc).ToKubernetesEnvVars(), + Name: kserveconstant.InferenceServiceContainerName, + Image: "gcr.io/custom-model:v0.1", + Env: models.MergeEnvVars( + models.EnvVars{ + {Name: defaultWorkersEnvVarName, Value: defaultWorkersEnvVarValue}, + }, + createDefaultPredictorEnvVars(modelSvc), + ).ToKubernetesEnvVars(), Resources: expDefaultModelResourceRequests, Ports: grpcServerlessContainerPorts, }, @@ -1932,6 +2075,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { UserContainerCPUDefaultLimit: userContainerCPUDefaultLimit, UserContainerCPULimitRequestFactor: userContainerCPULimitRequestFactor, UserContainerMemoryLimitRequestFactor: userContainerMemoryLimitRequestFactor, + DefaultEnvVarsWithoutCPULimits: defaultEnvVarsWithoutCPULimits, } tpl := NewInferenceServiceTemplater(*deployConfig) @@ -1941,6 +2085,10 @@ func TestCreateInferenceServiceSpec(t *testing.T) { return } assert.NoError(t, err) + + // Sort all env vars in both expected and actual inference service specs before comparing + sortInferenceServiceSpecEnvVars(tt.exp.Spec) + sortInferenceServiceSpecEnvVars(infSvcSpec.Spec) assert.Equal(t, tt.exp, infSvcSpec) }) } @@ -2071,7 +2219,7 @@ func TestCreateInferenceServiceSpecWithTransformer(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{}, + Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, }, }, }, @@ -2084,11 +2232,16 @@ func TestCreateInferenceServiceSpecWithTransformer(t *testing.T) { PodSpec: kservev1beta1.PodSpec{ Containers: []corev1.Container{ { - Name: "transformer", - Image: "ghcr.io/gojek/merlin-transformer-test", - Command: []string{"python"}, - Args: []string{"main.py"}, - Env: createDefaultTransformerEnvVars(modelSvc).ToKubernetesEnvVars(), + Name: "transformer", + Image: "ghcr.io/gojek/merlin-transformer-test", + Command: []string{"python"}, + Args: []string{"main.py"}, + Env: models.MergeEnvVars( + models.EnvVars{ + {Name: defaultWorkersEnvVarName, Value: defaultWorkersEnvVarValue}, + }, + createDefaultTransformerEnvVars(modelSvc), + ).ToKubernetesEnvVars(), Resources: expDefaultTransformerResourceRequests, LivenessProbe: transformerProbeConfig, }, @@ -2158,7 +2311,99 @@ func TestCreateInferenceServiceSpecWithTransformer(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{}, + Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, + }, + }, + }, + ComponentExtensionSpec: kservev1beta1.ComponentExtensionSpec{ + MinReplicas: &defaultModelResourceRequests.MinReplica, + MaxReplicas: defaultModelResourceRequests.MaxReplica, + }, + }, + Transformer: &kservev1beta1.TransformerSpec{ + PodSpec: kservev1beta1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "transformer", + Image: "ghcr.io/gojek/merlin-transformer-test", + Command: []string{"python"}, + Args: []string{"main.py"}, + Env: models.MergeEnvVars( + models.EnvVars{ + {Name: defaultWorkersEnvVarName, Value: defaultWorkersEnvVarValue}, + }, + createDefaultTransformerEnvVars(modelSvc), + ).ToKubernetesEnvVars(), + Resources: expUserResourceRequests, + LivenessProbe: transformerProbeConfig, + }, + }, + }, + ComponentExtensionSpec: kservev1beta1.ComponentExtensionSpec{ + MinReplicas: &userResourceRequests.MinReplica, + MaxReplicas: userResourceRequests.MaxReplica, + }, + }, + }, + }, + }, + { + name: "custom transformer with cpu limits specified", + modelSvc: &models.Service{ + Name: modelSvc.Name, + ModelName: modelSvc.ModelName, + ModelVersion: modelSvc.ModelVersion, + Namespace: project.Name, + ArtifactURI: modelSvc.ArtifactURI, + Type: models.ModelTypeTensorflow, + Options: &models.ModelOption{}, + Metadata: modelSvc.Metadata, + Transformer: &models.Transformer{ + Enabled: true, + Image: "ghcr.io/gojek/merlin-transformer-test", + Command: "python", + Args: "main.py", + ResourceRequest: userResourceRequestsWithCPULimits, + }, + Logger: &models.Logger{ + DestinationURL: loggerDestinationURL, + Transformer: &models.LoggerConfig{ + Enabled: false, + Mode: models.LogRequest, + }, + }, + Protocol: protocol.HttpJson, + }, + deploymentScale: defaultDeploymentScale, + exp: &kservev1beta1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{ + Name: modelSvc.Name, + Namespace: project.Name, + Annotations: map[string]string{ + knserving.QueueSidecarResourcePercentageAnnotationKey: queueResourcePercentage, + kserveconstant.DeploymentMode: string(kserveconstant.Serverless), + knautoscaling.InitialScaleAnnotationKey: fmt.Sprint(testTransformerScale), + }, + Labels: map[string]string{ + "gojek.com/app": modelSvc.Metadata.App, + "gojek.com/component": models.ComponentModelVersion, + "gojek.com/environment": testEnvironmentName, + "gojek.com/orchestrator": testOrchestratorName, + "gojek.com/stream": modelSvc.Metadata.Stream, + "gojek.com/team": modelSvc.Metadata.Team, + "sample": "true", + }, + }, + Spec: kservev1beta1.InferenceServiceSpec{ + Predictor: kservev1beta1.PredictorSpec{ + Tensorflow: &kservev1beta1.TFServingSpec{ + PredictorExtensionSpec: kservev1beta1.PredictorExtensionSpec{ + StorageURI: &storageUri, + Container: corev1.Container{ + Name: kserveconstant.InferenceServiceContainerName, + Resources: expDefaultModelResourceRequests, + LivenessProbe: probeConfig, + Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, }, }, }, @@ -2176,7 +2421,7 @@ func TestCreateInferenceServiceSpecWithTransformer(t *testing.T) { Command: []string{"python"}, Args: []string{"main.py"}, Env: createDefaultTransformerEnvVars(modelSvc).ToKubernetesEnvVars(), - Resources: expUserResourceRequests, + Resources: expUserResourceRequestsWithCPULimits, LivenessProbe: transformerProbeConfig, }, }, @@ -2241,7 +2486,7 @@ func TestCreateInferenceServiceSpecWithTransformer(t *testing.T) { Container: corev1.Container{ Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, - Env: []corev1.EnvVar{}, + Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, Ports: grpcServerlessContainerPorts, LivenessProbe: probeConfigUPI, }, @@ -2256,11 +2501,16 @@ func TestCreateInferenceServiceSpecWithTransformer(t *testing.T) { PodSpec: kservev1beta1.PodSpec{ Containers: []corev1.Container{ { - Name: "transformer", - Image: "ghcr.io/gojek/merlin-transformer-test", - Command: []string{"python"}, - Args: []string{"main.py"}, - Env: createDefaultTransformerEnvVars(modelSvcGRPC).ToKubernetesEnvVars(), + Name: "transformer", + Image: "ghcr.io/gojek/merlin-transformer-test", + Command: []string{"python"}, + Args: []string{"main.py"}, + Env: models.MergeEnvVars( + models.EnvVars{ + {Name: defaultWorkersEnvVarName, Value: defaultWorkersEnvVarValue}, + }, + createDefaultTransformerEnvVars(modelSvcGRPC), + ).ToKubernetesEnvVars(), Resources: expDefaultTransformerResourceRequests, LivenessProbe: transformerProbeConfigUPI, Ports: grpcServerlessContainerPorts, @@ -2334,7 +2584,7 @@ func TestCreateInferenceServiceSpecWithTransformer(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{}, + Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, }, }, }, @@ -2350,6 +2600,10 @@ func TestCreateInferenceServiceSpecWithTransformer(t *testing.T) { Name: "transformer", Image: standardTransformerConfig.ImageName, Env: models.MergeEnvVars(models.EnvVars{ + { + Name: defaultWorkersEnvVarName, + Value: defaultWorkersEnvVarValue, + }, { Name: transformerpkg.DefaultFeastSource, Value: standardTransformerConfig.DefaultFeastSource.String(), @@ -2438,7 +2692,7 @@ func TestCreateInferenceServiceSpecWithTransformer(t *testing.T) { Container: corev1.Container{ Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, - Env: []corev1.EnvVar{}, + Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, LivenessProbe: probeConfigUPI, Ports: grpcRawContainerPorts, }, @@ -2456,6 +2710,10 @@ func TestCreateInferenceServiceSpecWithTransformer(t *testing.T) { Name: "transformer", Image: standardTransformerConfig.ImageName, Env: models.MergeEnvVars(models.EnvVars{ + { + Name: defaultWorkersEnvVarName, + Value: defaultWorkersEnvVarValue, + }, { Name: transformerpkg.DefaultFeastSource, Value: standardTransformerConfig.DefaultFeastSource.String(), @@ -2552,9 +2810,21 @@ func TestCreateInferenceServiceSpecWithTransformer(t *testing.T) { PodSpec: kservev1beta1.PodSpec{ Containers: []corev1.Container{ { - Name: kserveconstant.InferenceServiceContainerName, - Image: "gojek/project-model:1", - Env: models.MergeEnvVars(models.EnvVars{models.EnvVar{Name: envGRPCOptions, Value: "{}"}}, createPyFuncDefaultEnvVarsWithProtocol(modelSvc, protocol.UpiV1)).ToKubernetesEnvVars(), + Name: kserveconstant.InferenceServiceContainerName, + Image: "gojek/project-model:1", + Env: models.MergeEnvVars( + models.EnvVars{ + { + Name: defaultWorkersEnvVarName, + Value: defaultWorkersEnvVarValue, + }, + { + Name: envGRPCOptions, + Value: "{}", + }, + }, + createPyFuncDefaultEnvVarsWithProtocol(modelSvc, protocol.UpiV1), + ).ToKubernetesEnvVars(), Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfigUPI, Ports: grpcRawContainerPorts, @@ -2573,6 +2843,10 @@ func TestCreateInferenceServiceSpecWithTransformer(t *testing.T) { Name: "transformer", Image: standardTransformerConfig.ImageName, Env: models.MergeEnvVars(models.EnvVars{ + { + Name: defaultWorkersEnvVarName, + Value: defaultWorkersEnvVarValue, + }, { Name: transformerpkg.DefaultFeastSource, Value: standardTransformerConfig.DefaultFeastSource.String(), @@ -2678,9 +2952,15 @@ func TestCreateInferenceServiceSpecWithTransformer(t *testing.T) { PodSpec: kservev1beta1.PodSpec{ Containers: []corev1.Container{ { - Name: kserveconstant.InferenceServiceContainerName, - Image: "gojek/project-model:1", - Env: models.MergeEnvVars(models.EnvVars{models.EnvVar{Name: envGRPCOptions, Value: "{}"}}, createPyFuncDefaultEnvVarsWithProtocol(modelSvc, protocol.HttpJson)).ToKubernetesEnvVars(), + Name: kserveconstant.InferenceServiceContainerName, + Image: "gojek/project-model:1", + Env: models.MergeEnvVars( + models.EnvVars{ + {Name: defaultWorkersEnvVarName, Value: defaultWorkersEnvVarValue}, + {Name: envGRPCOptions, Value: "{}"}, + }, + createPyFuncDefaultEnvVarsWithProtocol(modelSvc, protocol.HttpJson), + ).ToKubernetesEnvVars(), Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, }, @@ -2698,6 +2978,10 @@ func TestCreateInferenceServiceSpecWithTransformer(t *testing.T) { Name: "transformer", Image: standardTransformerConfig.ImageName, Env: models.MergeEnvVars(models.EnvVars{ + { + Name: defaultWorkersEnvVarName, + Value: defaultWorkersEnvVarValue, + }, { Name: transformerpkg.DefaultFeastSource, Value: standardTransformerConfig.DefaultFeastSource.String(), @@ -2752,6 +3036,7 @@ func TestCreateInferenceServiceSpecWithTransformer(t *testing.T) { UserContainerCPUDefaultLimit: userContainerCPUDefaultLimit, UserContainerCPULimitRequestFactor: userContainerCPULimitRequestFactor, UserContainerMemoryLimitRequestFactor: userContainerMemoryLimitRequestFactor, + DefaultEnvVarsWithoutCPULimits: defaultEnvVarsWithoutCPULimits, } tpl := NewInferenceServiceTemplater(*deployConfig) @@ -2762,6 +3047,10 @@ func TestCreateInferenceServiceSpecWithTransformer(t *testing.T) { } assert.NoError(t, err) + + // Sort all env vars in both expected and actual inference service specs before comparing + sortInferenceServiceSpecEnvVars(tt.exp.Spec) + sortInferenceServiceSpecEnvVars(infSvcSpec.Spec) assert.Equal(t, tt.exp, infSvcSpec) }) } @@ -4214,8 +4503,9 @@ func TestCreateTransformerSpec(t *testing.T) { cpuRequest := resource.MustParse("1") memoryRequest := resource.MustParse("1Gi") - cpuLimit := resource.MustParse("8") + cpuLimit := resource.MustParse("10") memoryLimit := ScaleQuantity(memoryRequest, 2) + customCPULimit := resource.MustParse("8") // Liveness probe config for the transformers transformerProbeConfig := createLivenessProbeSpec(protocol.HttpJson, "/") @@ -4324,6 +4614,81 @@ func TestCreateTransformerSpec(t *testing.T) { }, }, }, + { + "standard transformer with cpu limits specified", + args{ + &models.Service{ + Name: modelSvc.Name, + ModelName: modelSvc.ModelName, + ModelVersion: modelSvc.ModelVersion, + Namespace: modelSvc.Namespace, + Protocol: protocol.HttpJson, + }, + &models.Transformer{ + TransformerType: models.StandardTransformerType, + Image: standardTransformerConfig.ImageName, + Command: "python", + Args: "main.py", + ResourceRequest: &models.ResourceRequest{ + MinReplica: 1, + MaxReplica: 1, + CPURequest: cpuRequest, + MemoryRequest: memoryRequest, + CPULimit: customCPULimit, + }, + EnvVars: models.EnvVars{ + {Name: transformerpkg.JaegerCollectorURL, Value: "NEW_HOST"}, // test user overwrite + }, + }, + &config.DeploymentConfig{ + StandardTransformer: standardTransformerConfig, + UserContainerCPUDefaultLimit: userContainerCPUDefaultLimit, + UserContainerCPULimitRequestFactor: userContainerCPULimitRequestFactor, + UserContainerMemoryLimitRequestFactor: userContainerMemoryLimitRequestFactor, + }, + }, + &kservev1beta1.TransformerSpec{ + PodSpec: kservev1beta1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "transformer", + Image: standardTransformerConfig.ImageName, + Command: []string{"python"}, + Args: []string{"main.py"}, + Env: models.MergeEnvVars(models.EnvVars{ + {Name: transformerpkg.DefaultFeastSource, Value: standardTransformerConfig.DefaultFeastSource.String()}, + { + Name: transformerpkg.FeastStorageConfigs, + Value: `{"1":{"redisCluster":{"feastServingUrl":"localhost:6866","redisAddress":["10.1.1.2","10.1.1.3"],"option":{"poolSize":5,"minIdleConnections":2}}},"2":{"bigtable":{"feastServingUrl":"localhost:6867","project":"gcp-project","instance":"instance","appProfile":"default","option":{"grpcConnectionPool":4,"keepAliveInterval":"120s","keepAliveTimeout":"60s","credentialJson":"eyJrZXkiOiJ2YWx1ZSJ9"}}}}`, + }, + {Name: transformerpkg.FeastServingKeepAliveEnabled, Value: "true"}, + {Name: transformerpkg.FeastServingKeepAliveTime, Value: "30s"}, + {Name: transformerpkg.FeastServingKeepAliveTimeout, Value: "1s"}, + {Name: transformerpkg.FeastGRPCConnCount, Value: "5"}, + {Name: transformerpkg.JaegerCollectorURL, Value: "NEW_HOST"}, + {Name: transformerpkg.JaegerSamplerParam, Value: standardTransformerConfig.Jaeger.SamplerParam}, + {Name: transformerpkg.JaegerDisabled, Value: standardTransformerConfig.Jaeger.Disabled}, + }, createDefaultTransformerEnvVars(modelSvc)).ToKubernetesEnvVars(), + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: cpuRequest, + corev1.ResourceMemory: memoryRequest, + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: customCPULimit, + corev1.ResourceMemory: memoryLimit, + }, + }, + LivenessProbe: transformerProbeConfig, + }, + }, + }, + ComponentExtensionSpec: kservev1beta1.ComponentExtensionSpec{ + MinReplicas: &one, + MaxReplicas: one, + }, + }, + }, { "custom transformer", args{ From 7fe8aa8dc4bc75f17e423e9cf2e92b35bc0f676e Mon Sep 17 00:00:00 2001 From: ewezy Date: Fri, 17 May 2024 12:06:38 +0800 Subject: [PATCH 05/26] Fix environment service unit tests --- api/service/environment_service_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/api/service/environment_service_test.go b/api/service/environment_service_test.go index 46db6da95..77615a624 100644 --- a/api/service/environment_service_test.go +++ b/api/service/environment_service_test.go @@ -197,6 +197,7 @@ func TestGetEnvironment(t *testing.T) { MinReplica: 1, MaxReplica: 4, CPURequest: resource.MustParse("1"), + CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), GPURequest: resource.MustParse("0"), }, @@ -223,6 +224,7 @@ func TestGetEnvironment(t *testing.T) { MinReplica: 1, MaxReplica: 4, CPURequest: resource.MustParse("1"), + CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), GPURequest: resource.MustParse("0"), }, @@ -253,6 +255,7 @@ func TestGetEnvironment(t *testing.T) { MinReplica: 1, MaxReplica: 4, CPURequest: resource.MustParse("1"), + CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), }, CreatedUpdated: models.CreatedUpdated{ @@ -320,6 +323,7 @@ func TestGetDefaultEnvironment(t *testing.T) { MinReplica: 1, MaxReplica: 4, CPURequest: resource.MustParse("1"), + CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), GPURequest: resource.MustParse("0"), }, @@ -346,6 +350,7 @@ func TestGetDefaultEnvironment(t *testing.T) { MinReplica: 1, MaxReplica: 4, CPURequest: resource.MustParse("1"), + CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), GPURequest: resource.MustParse("0"), }, @@ -376,6 +381,7 @@ func TestGetDefaultEnvironment(t *testing.T) { MinReplica: 1, MaxReplica: 4, CPURequest: resource.MustParse("1"), + CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), }, CreatedUpdated: models.CreatedUpdated{ @@ -443,6 +449,7 @@ func TestGetDefaultPredictionJobEnvironment(t *testing.T) { MinReplica: 1, MaxReplica: 4, CPURequest: resource.MustParse("1"), + CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), GPURequest: resource.MustParse("0"), }, @@ -469,6 +476,7 @@ func TestGetDefaultPredictionJobEnvironment(t *testing.T) { MinReplica: 1, MaxReplica: 4, CPURequest: resource.MustParse("1"), + CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), GPURequest: resource.MustParse("0"), }, @@ -499,6 +507,7 @@ func TestGetDefaultPredictionJobEnvironment(t *testing.T) { MinReplica: 1, MaxReplica: 4, CPURequest: resource.MustParse("1"), + CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), }, CreatedUpdated: models.CreatedUpdated{ @@ -567,6 +576,7 @@ func TestListEnvironment(t *testing.T) { MinReplica: 1, MaxReplica: 4, CPURequest: resource.MustParse("1"), + CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), GPURequest: resource.MustParse("0"), }, @@ -594,6 +604,7 @@ func TestListEnvironment(t *testing.T) { MinReplica: 1, MaxReplica: 4, CPURequest: resource.MustParse("1"), + CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), GPURequest: resource.MustParse("0"), }, @@ -627,6 +638,7 @@ func TestListEnvironment(t *testing.T) { MinReplica: 1, MaxReplica: 4, CPURequest: resource.MustParse("1"), + CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), GPURequest: resource.MustParse("0"), }, @@ -654,6 +666,7 @@ func TestListEnvironment(t *testing.T) { MinReplica: 1, MaxReplica: 4, CPURequest: resource.MustParse("1"), + CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), GPURequest: resource.MustParse("0"), }, @@ -687,6 +700,7 @@ func TestListEnvironment(t *testing.T) { MinReplica: 1, MaxReplica: 4, CPURequest: resource.MustParse("1"), + CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), }, CreatedUpdated: models.CreatedUpdated{ From 8efd86f57a4863662a017e84fe8efd5168b455dd Mon Sep 17 00:00:00 2001 From: ewezy Date: Fri, 17 May 2024 13:53:26 +0800 Subject: [PATCH 06/26] Fix model service deployment unit tests --- api/queue/work/model_service_deployment_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/queue/work/model_service_deployment_test.go b/api/queue/work/model_service_deployment_test.go index 403858322..1efd73980 100644 --- a/api/queue/work/model_service_deployment_test.go +++ b/api/queue/work/model_service_deployment_test.go @@ -34,6 +34,7 @@ func TestExecuteDeployment(t *testing.T) { MinReplica: 0, MaxReplica: 1, CPURequest: resource.MustParse("1"), + CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), GPURequest: resource.MustParse("0"), }, @@ -534,6 +535,7 @@ func TestExecuteDeployment(t *testing.T) { MinReplica: 0, MaxReplica: 1, CPURequest: resource.MustParse("1"), + CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), GPUName: "NVIDIA P4", GPURequest: resource.MustParse("1"), @@ -741,6 +743,7 @@ func TestExecuteRedeployment(t *testing.T) { MinReplica: 0, MaxReplica: 1, CPURequest: resource.MustParse("1"), + CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), GPURequest: resource.MustParse("0"), }, From 7f2e7c4229de8d76e0def6877bca4e7549991623 Mon Sep 17 00:00:00 2001 From: ewezy Date: Fri, 17 May 2024 17:09:32 +0800 Subject: [PATCH 07/26] Add config parsing tests --- api/config/config_test.go | 319 ++++++++++++++++++ api/config/testdata/base-configs-1.yaml | 151 +++++++++ .../testdata/invalid-duration-format.yaml | 2 + api/config/testdata/invalid-file-format.yaml | 9 + api/config/testdata/invalid-type.yaml | 2 + 5 files changed, 483 insertions(+) create mode 100644 api/config/testdata/base-configs-1.yaml create mode 100644 api/config/testdata/invalid-duration-format.yaml create mode 100644 api/config/testdata/invalid-file-format.yaml create mode 100644 api/config/testdata/invalid-type.yaml diff --git a/api/config/config_test.go b/api/config/config_test.go index 89f1f1ed5..044666261 100644 --- a/api/config/config_test.go +++ b/api/config/config_test.go @@ -20,11 +20,17 @@ import ( "testing" "time" + "github.com/caraml-dev/merlin/client" "github.com/caraml-dev/merlin/pkg/transformer/feast" "github.com/caraml-dev/merlin/pkg/transformer/spec" + mlpcluster "github.com/caraml-dev/mlp/api/pkg/cluster" + "github.com/caraml-dev/mlp/api/pkg/instrumentation/newrelic" + "github.com/caraml-dev/mlp/api/pkg/instrumentation/sentry" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/protobuf/types/known/durationpb" + v1 "k8s.io/api/core/v1" + clientcmdapiv1 "k8s.io/client-go/tools/clientcmd/api/v1" ) func TestFeastServingURLs_URLs(t *testing.T) { @@ -311,3 +317,316 @@ func TestStandardTransformerConfig_ToFeastStorageConfigs(t *testing.T) { }) } } + +func setupNewEnv(envMaps ...map[string]string) { + os.Clearenv() + + for _, envMap := range envMaps { + for key, val := range envMap { + os.Setenv(key, val) + } + } +} + +func TestLoad(t *testing.T) { + zeroSecond, _ := time.ParseDuration("0s") + twoMinutes := 2 * time.Minute + oneMinute := 1 * time.Minute + + envVarNameFoo := "foo" + envVarValueBar := "bar" + + tests := map[string]struct { + filepaths []string + env map[string]string + want *Config + wantErr bool + }{ + "multiple file": { + filepaths: []string{"testdata/base-configs-1.yaml", "testdata/bigtable-config.yaml"}, + want: &Config{ + Environment: "dev", + Port: 8080, + LoggerDestinationURL: "kafka:logger.destination:6668", + MLObsLoggerDestinationURL: "mlobs:kafka.destination:6668", + Sentry: sentry.Config{ + DSN: "", + Enabled: false, + Labels: map[string]string{ + "foo": "bar", + }, + }, + NewRelic: newrelic.Config{ + Enabled: true, + AppName: "merlin-test", + Labels: map[string]interface{}{}, + IgnoreStatusCodes: []int{400, 401, 402}, + }, + NumOfQueueWorkers: 2, + SwaggerPath: "swaggerpath.com", + DeploymentLabelPrefix: "caraml.com/", + PyfuncGRPCOptions: "{}", + DbConfig: DatabaseConfig{ + Host: "localhost", + Port: 5432, + User: "merlin", + Password: "merlin", + Database: "merlin", + MigrationPath: "file://db-migrations", + ConnMaxIdleTime: zeroSecond, + ConnMaxLifetime: zeroSecond, + MaxIdleConns: 0, + MaxOpenConns: 0, + }, + ClusterConfig: ClusterConfig{ + InClusterConfig: false, + EnvironmentConfigPath: "/app/cluster-env/environments.yaml", + EnvironmentConfigs: []*EnvironmentConfig{}, + }, + ImageBuilderConfig: ImageBuilderConfig{ + ClusterName: "test-cluster", + GcpProject: "test-project", + ArtifactServiceType: "gcs", + BaseImage: BaseImageConfig{ + ImageName: "ghcr.io/caraml-dev/merlin/merlin-pyfunc-base:0.0.0", + DockerfilePath: "pyfunc-server/docker/Dockerfile", + BuildContextURI: "git://github.com/caraml-dev/merlin.git#refs/tags/v0.0.0", + BuildContextSubPath: "python", + }, + PredictionJobBaseImage: BaseImageConfig{ + ImageName: "ghcr.io/caraml-dev/merlin/merlin-pyspark-base:0.0.0", + DockerfilePath: "batch-predictor/docker/app.Dockerfile", + BuildContextURI: "git://github.com/caraml-dev/merlin.git#refs/tags/v0.0.0", + BuildContextSubPath: "python", + MainAppPath: "/home/spark/merlin-spark-app/main.py", + }, + BuildNamespace: "caraml", + DockerRegistry: "test-docker.pkg.dev/test/caraml-registry", + BuildTimeout: "30m", + KanikoImage: "gcr.io/kaniko-project/executor:v1.21.0", + KanikoServiceAccount: "kaniko-merlin", + KanikoAdditionalArgs: []string{"--test=true", "--no-logs=false"}, + DefaultResources: ResourceRequestsLimits{ + Requests: Resource{ + CPU: "1", + Memory: "4Gi", + }, + Limits: Resource{ + CPU: "1", + Memory: "4Gi", + }, + }, + Retention: 48 * time.Hour, + Tolerations: []v1.Toleration{ + { + Key: "purpose.caraml.com/batch", + Value: "true", + Operator: v1.TolerationOpEqual, + Effect: v1.TaintEffectNoSchedule, + }, + }, + NodeSelectors: map[string]string{ + "purpose.caraml.com/batch": "true", + }, + MaximumRetry: 3, + K8sConfig: &mlpcluster.K8sConfig{ + Cluster: &clientcmdapiv1.Cluster{ + Server: "https://127.0.0.1", + CertificateAuthorityData: []byte("some_string"), + }, + AuthInfo: &clientcmdapiv1.AuthInfo{ + Exec: &clientcmdapiv1.ExecConfig{ + APIVersion: "some_api_version", + Command: "some_command", + InteractiveMode: clientcmdapiv1.IfAvailableExecInteractiveMode, + ProvideClusterInfo: true, + }, + }, + Name: "dev-server", + }, + SafeToEvict: false, + SupportedPythonVersions: []string{"3.8.*", "3.9.*", "3.10.*"}, + }, + BatchConfig: BatchConfig{ + Tolerations: []v1.Toleration{ + { + Key: "purpose.caraml.com/batch", + Value: "true", + Operator: v1.TolerationOpEqual, + Effect: v1.TaintEffectNoSchedule, + }, + }, + NodeSelectors: map[string]string{ + "purpose.caraml.com/batch": "true", + }, + }, + AuthorizationConfig: AuthorizationConfig{ + AuthorizationEnabled: true, + KetoRemoteRead: "http://mlp-keto-read:80", + KetoRemoteWrite: "http://mlp-keto-write:80", + Caching: &InMemoryCacheConfig{ + Enabled: true, + KeyExpirySeconds: 600, + CacheCleanUpIntervalSeconds: 750, + }, + }, + MlpAPIConfig: MlpAPIConfig{ + APIHost: "http://mlp.caraml.svc.local:8080", + }, + FeatureToggleConfig: FeatureToggleConfig{ + MonitoringConfig: MonitoringConfig{ + MonitoringEnabled: true, + MonitoringBaseURL: "https://test.io/merlin-overview-dashboard", + MonitoringJobBaseURL: "https://test.io/batch-predictions-dashboard", + }, + AlertConfig: AlertConfig{ + AlertEnabled: true, + GitlabConfig: GitlabConfig{ + BaseURL: "https://test.io/", + DashboardRepository: "dashboards/merlin", + DashboardBranch: "master", + AlertRepository: "alerts/merlin", + AlertBranch: "master", + }, + WardenConfig: WardenConfig{ + APIHost: "https://test.io/", + }, + }, + ModelDeletionConfig: ModelDeletionConfig{ + Enabled: false, + }, + }, + ReactAppConfig: ReactAppConfig{ + DocURL: []Documentation{ + { + Label: "Merlin User Guide", + Href: "https://guide.io", + }, + { + Label: "Merlin Examples", + Href: "https://examples.io", + }, + }, + DockerRegistries: "docker.io", + Environment: "dev", + FeastCoreURL: "https://feastcore.io", + HomePage: "/merlin", + MerlinURL: "https://test.io/api/merlin/v1", + MlpURL: "/api", + OauthClientID: "abc.apps.clientid.com", + UPIDocumentation: "https://github.com/caraml-dev/universal-prediction-interface/blob/main/docs/api_markdown/caraml/upi/v1/index.md", + }, + UI: UIConfig{ + StaticPath: "ui/build", + IndexPath: "index.html", + }, + StandardTransformerConfig: StandardTransformerConfig{ + FeastServingURLs: []FeastServingURL{}, + FeastBigtableConfig: &FeastBigtableConfig{ + IsUsingDirectStorage: true, + ServingURL: "10.1.1.3", + Project: "gcp-project", + Instance: "instance", + AppProfile: "default", + PoolSize: 3, + KeepAliveInterval: &twoMinutes, + KeepAliveTimeout: &oneMinute, + }, + FeastGPRCConnCount: 10, + FeastServingKeepAlive: &FeastServingKeepAliveConfig{ + Enabled: false, + Time: 60 * time.Second, + Timeout: time.Second, + }, + ModelClientKeepAlive: &ModelClientKeepAliveConfig{ + Enabled: false, + Time: 60 * time.Second, + Timeout: 5 * time.Second, + }, + ModelServerConnCount: 10, + DefaultFeastSource: spec.ServingSource_BIGTABLE, + Jaeger: JaegerConfig{ + SamplerParam: "0.01", + Disabled: "true", + }, + Kafka: KafkaConfig{ + CompressionType: "none", + MaxMessageSizeBytes: 1048588, + ConnectTimeoutMS: 1000, + SerializationFmt: "protobuf", + LingerMS: 100, + NumPartitions: 24, + ReplicationFactor: 3, + AdditionalConfig: "{}", + }, + SimulatorFeastClientMaxConcurrentRequests: 100, + }, + MlflowConfig: MlflowConfig{ + TrackingURL: "https://mlflow.io", + ArtifactServiceType: "gcs", + }, + PyFuncPublisherConfig: PyFuncPublisherConfig{ + Kafka: KafkaConfig{ + Brokers: "kafka:broker.destination:6668", + Acks: 0, + CompressionType: "none", + MaxMessageSizeBytes: 1048588, + ConnectTimeoutMS: 1000, + SerializationFmt: "protobuf", + LingerMS: 500, + NumPartitions: 24, + ReplicationFactor: 3, + AdditionalConfig: "{}", + }, + SamplingRatioRate: 0.01, + }, + InferenceServiceDefaults: InferenceServiceDefaults{ + UserContainerCPUDefaultLimit: "100", + UserContainerCPULimitRequestFactor: 0, + UserContainerMemoryLimitRequestFactor: 2, + DefaultEnvVarsWithoutCPULimits: []client.EnvVar{ + { + Name: &envVarNameFoo, + Value: &envVarValueBar, + }, + }, + }, + ObservabilityPublisher: ObservabilityPublisher{ + KafkaConsumer: KafkaConsumer{ + AdditionalConsumerConfig: map[string]string{}, + }, + DeploymentTimeout: 30 * time.Minute, + }, + }, + }, + "missing file": { + filepaths: []string{"testdata/this-file-should-not-exist.yaml"}, + wantErr: true, + }, + "invalid duration format": { + filepaths: []string{"testdata/invalid-duration-format.yaml"}, + wantErr: true, + }, + "invalid file format": { + filepaths: []string{"testdata/invalid-file-format.yaml"}, + wantErr: true, + }, + "invalid type": { + filepaths: []string{"testdata/invalid-type.yaml"}, + wantErr: true, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + setupNewEnv(tt.env) + var emptyCfg Config + got, err := Load(&emptyCfg, tt.filepaths...) + if (err != nil) != tt.wantErr { + t.Errorf("FromFiles() error = %v, wantErr %v", err, tt.wantErr) + return + } + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/api/config/testdata/base-configs-1.yaml b/api/config/testdata/base-configs-1.yaml new file mode 100644 index 000000000..65dfc7ceb --- /dev/null +++ b/api/config/testdata/base-configs-1.yaml @@ -0,0 +1,151 @@ +Environment: dev +Port: 8080 +LoggerDestinationURL: kafka:logger.destination:6668 +MLObsLoggerDestinationURL: mlobs:kafka.destination:6668 +Sentry: + DSN: "" + Enabled: false + Labels: + foo: bar +NewRelic: + Enabled: true + AppName: merlin-test + IgnoreStatusCodes: + - 400 + - 401 + - 402 +NumOfQueueWorkers: 2 +SwaggerPath: swaggerpath.com +DeploymentLabelPrefix: caraml.com/ +PyfuncGRPCOptions: "{}" +DbConfig: + Host: localhost + Port: 5432 + User: merlin + Password: merlin + Database: merlin +ClusterConfig: + EnvironmentConfigPath: /app/cluster-env/environments.yaml + InClusterConfig: false +ImageBuilderConfig: + ClusterName: test-cluster + GcpProject: test-project + ArtifactServiceType: gcs + BaseImage: + ImageName: ghcr.io/caraml-dev/merlin/merlin-pyfunc-base:0.0.0 + DockerfilePath: pyfunc-server/docker/Dockerfile + BuildContextURI: git://github.com/caraml-dev/merlin.git#refs/tags/v0.0.0 + BuildContextSubPath: python + PredictionJobBaseImage: + ImageName: ghcr.io/caraml-dev/merlin/merlin-pyspark-base:0.0.0 + DockerfilePath: batch-predictor/docker/app.Dockerfile + BuildContextURI: git://github.com/caraml-dev/merlin.git#refs/tags/v0.0.0 + BuildContextSubPath: python + MainAppPath: /home/spark/merlin-spark-app/main.py + BuildNamespace: caraml + DockerRegistry: test-docker.pkg.dev/test/caraml-registry + BuildTimeout: 30m + KanikoImage: gcr.io/kaniko-project/executor:v1.21.0 + KanikoServiceAccount: kaniko-merlin + KanikoAdditionalArgs: + - --test=true + - --no-logs=false + DefaultResources: + Requests: + CPU: "1" + Memory: 4Gi + Limits: + CPU: "1" + Memory: 4Gi + Retention: 48h + Tolerations: + - Key: purpose.caraml.com/batch + Value: "true" + Operator: Equal + Effect: NoSchedule + NodeSelectors: + purpose.caraml.com/batch: "true" + MaximumRetry: 3 + K8sConfig: + name: dev-server + cluster: + server: https://127.0.0.1 + certificate-authority-data: c29tZV9zdHJpbmc= + user: + exec: + apiVersion: some_api_version + command: some_command + interactiveMode: IfAvailable + provideClusterInfo: true + SafeToEvict: false + SupportedPythonVersions: + - 3.8.* + - 3.9.* + - 3.10.* +BatchConfig: + Tolerations: + - Effect: NoSchedule + Key: purpose.caraml.com/batch + Operator: Equal + Value: "true" + NodeSelectors: + purpose.caraml.com/batch: "true" +AuthorizationConfig: + AuthorizationEnabled: true + KetoRemoteRead: http://mlp-keto-read:80 + KetoRemoteWrite: http://mlp-keto-write:80 + Caching: + Enabled: true + KeyExpirySeconds: 600 + CacheCleanUpIntervalSeconds: 750 +MlpAPIConfig: + APIHost: http://mlp.caraml.svc.local:8080 +FeatureToggleConfig: + MonitoringConfig: + MonitoringEnabled: true + MonitoringBaseURL: https://test.io/merlin-overview-dashboard + MonitoringJobBaseURL: https://test.io/batch-predictions-dashboard + AlertConfig: + AlertEnabled: true + GitlabConfig: + BaseURL: https://test.io/ + DashboardRepository: dashboards/merlin + DashboardBranch: master + AlertRepository: alerts/merlin + AlertBranch: master + WardenConfig: + APIHost: https://test.io/ + ModelDeletionConfig: + Enabled: false +ReactAppConfig: + DocURL: + - Label: Merlin User Guide + Href: https://guide.io + - Label: Merlin Examples + Href: https://examples.io + DockerRegistries: docker.io + Environment: dev + FeastCoreURL: https://feastcore.io + HomePage: /merlin + MerlinURL: https://test.io/api/merlin/v1 + MlpURL: /api + OauthClientID: abc.apps.clientid.com + UPIDocumentation: https://github.com/caraml-dev/universal-prediction-interface/blob/main/docs/api_markdown/caraml/upi/v1/index.md +MlflowConfig: + TrackingURL: https://mlflow.io + ArtifactServiceType: gcs +PyFuncPublisherConfig: + Kafka: + Brokers: kafka:broker.destination:6668 + Acks: 0 + MaxMessageSizeBytes: "1048588" + LingerMS: 500 + AdditionalConfig: '{}' + SamplingRatioRate: 0.01 +InferenceServiceDefaults: + UserContainerCPUDefaultLimit: 100 + UserContainerCPULimitRequestFactor: 0 + UserContainerMemoryLimitRequestFactor: 2 + DefaultEnvVarsWithoutCPULimits: + - Name: foo + Value: bar diff --git a/api/config/testdata/invalid-duration-format.yaml b/api/config/testdata/invalid-duration-format.yaml new file mode 100644 index 000000000..cca5edf48 --- /dev/null +++ b/api/config/testdata/invalid-duration-format.yaml @@ -0,0 +1,2 @@ +DbConfig: + ConnMaxIdleTime: 30rr diff --git a/api/config/testdata/invalid-file-format.yaml b/api/config/testdata/invalid-file-format.yaml new file mode 100644 index 000000000..9551df9cd --- /dev/null +++ b/api/config/testdata/invalid-file-format.yaml @@ -0,0 +1,9 @@ +Environment: dev +Port: 8080 +LoggerDestinationURL: kafka:logger.destination:6668 +MLObsLoggerDestinationURL: +Sentry: mlobs:kafka.destination:6668 + DSN: "" + Enabled: false + Labels: + foo: bar diff --git a/api/config/testdata/invalid-type.yaml b/api/config/testdata/invalid-type.yaml new file mode 100644 index 000000000..c4912c60f --- /dev/null +++ b/api/config/testdata/invalid-type.yaml @@ -0,0 +1,2 @@ +Port: + Value: 9999 \ No newline at end of file From c6ff954c7a2b3e88cfedd059683e86fba41900ba Mon Sep 17 00:00:00 2001 From: ewezy Date: Fri, 17 May 2024 17:20:04 +0800 Subject: [PATCH 08/26] Make env var setters return errors that will then be checked --- api/config/config_test.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/api/config/config_test.go b/api/config/config_test.go index 044666261..f49a9fdb8 100644 --- a/api/config/config_test.go +++ b/api/config/config_test.go @@ -318,14 +318,19 @@ func TestStandardTransformerConfig_ToFeastStorageConfigs(t *testing.T) { } } -func setupNewEnv(envMaps ...map[string]string) { +func setupNewEnv(envMaps ...map[string]string) error { os.Clearenv() + var err error for _, envMap := range envMaps { for key, val := range envMap { - os.Setenv(key, val) + err = os.Setenv(key, val) + if err != nil { + return err + } } } + return nil } func TestLoad(t *testing.T) { @@ -619,7 +624,7 @@ func TestLoad(t *testing.T) { for name, tt := range tests { t.Run(name, func(t *testing.T) { - setupNewEnv(tt.env) + assert.NoError(t, setupNewEnv(tt.env)) var emptyCfg Config got, err := Load(&emptyCfg, tt.filepaths...) if (err != nil) != tt.wantErr { From 1d4661de7ac63e99514c4f4b374804cb89940be2 Mon Sep 17 00:00:00 2001 From: ewezy Date: Mon, 20 May 2024 13:12:38 +0800 Subject: [PATCH 09/26] Add component to make cpu limits configurable in ui --- .../components/AutoscalingPolicyFormGroup.js | 12 ++- .../forms/components/CPULimitsFormGroup.js | 46 +++++++++++ .../forms/components/ImageBuilderSection.js | 77 ++++++++++--------- .../forms/components/ResourcesPanel.js | 2 + .../components/forms/steps/ModelStep.js | 6 ++ .../version_endpoint/VersionEndpoint.js | 1 + 6 files changed, 106 insertions(+), 38 deletions(-) create mode 100644 ui/src/pages/version/components/forms/components/CPULimitsFormGroup.js diff --git a/ui/src/pages/version/components/forms/components/AutoscalingPolicyFormGroup.js b/ui/src/pages/version/components/forms/components/AutoscalingPolicyFormGroup.js index 9c248fda5..c75b4b3a9 100644 --- a/ui/src/pages/version/components/forms/components/AutoscalingPolicyFormGroup.js +++ b/ui/src/pages/version/components/forms/components/AutoscalingPolicyFormGroup.js @@ -109,7 +109,9 @@ export const AutoscalingPolicyFormGroup = ({ Autoscaling Policy determines the condition for increasing or decreasing number of replicas. - }> + } + fullWidth + > @@ -117,19 +119,23 @@ export const AutoscalingPolicyFormGroup = ({ Metrics Type * - }> + } + fullWidth + > - + diff --git a/ui/src/pages/version/components/forms/components/CPULimitsFormGroup.js b/ui/src/pages/version/components/forms/components/CPULimitsFormGroup.js new file mode 100644 index 000000000..26a1387ae --- /dev/null +++ b/ui/src/pages/version/components/forms/components/CPULimitsFormGroup.js @@ -0,0 +1,46 @@ +import React, { Fragment } from "react"; +import { FormLabelWithToolTip, useOnChangeHandler } from "@caraml-dev/ui-lib"; +import { EuiDescribedFormGroup, EuiFieldText, EuiFormRow } from "@elastic/eui"; + + +export const CPULimitsFormGroup = ({ + resourcesConfig, + onChangeHandler, + errors = {}, +}) => { + const {onChange} = useOnChangeHandler(onChangeHandler); + + return ( + CPU Limits

} + description={ + + CPU limits are not set at the platform level. Use this field to limit the + amount of CPU that the model is able to consume. + + } + fullWidth + > + + } + isInvalid={!!errors.cpu_limit} + error={errors.cpu_limit} + fullWidth + > + onChange("cpu_limit")(e.target.value)} + isInvalid={!!errors.cpu_limit} + name="cpu_limit" + fullWidth + /> + + + ) +} \ No newline at end of file diff --git a/ui/src/pages/version/components/forms/components/ImageBuilderSection.js b/ui/src/pages/version/components/forms/components/ImageBuilderSection.js index b30cfaf46..406ddf73a 100644 --- a/ui/src/pages/version/components/forms/components/ImageBuilderSection.js +++ b/ui/src/pages/version/components/forms/components/ImageBuilderSection.js @@ -1,64 +1,71 @@ import { FormLabelWithToolTip, useOnChangeHandler } from "@caraml-dev/ui-lib"; -import { - EuiFieldText, - EuiFlexGroup, - EuiFlexItem, - EuiFormRow, +import { + EuiFieldText, + EuiFlexGroup, + EuiFlexItem, + EuiFormRow, EuiTitle, } from "@elastic/eui"; export const ImageBuilderSection = ({ - imageBuilderResourceConfig, - onChangeHandler, - errors = {}, - }) => { + imageBuilderResourceConfig, + onChangeHandler, + errors = {}, +}) => { + const {onChange} = useOnChangeHandler(onChangeHandler); - const { onChange } = useOnChangeHandler(onChangeHandler); - - return ( + return ( + <> + +

{"Image Building Resources"}

+ - + - } - isInvalid={!!errors.cpu_request} - error={errors.cpu_request} - fullWidth + } + isInvalid={!!errors.cpu_request} + error={errors.cpu_request} + fullWidth > - onChange("cpu_request")(e.target.value)} isInvalid={!!errors.cpu_request} name="cpu" - /> + fullWidth + /> - + - + - } - isInvalid={!!errors.memory_request} - error={errors.memory_request} - fullWidth + } + isInvalid={!!errors.memory_request} + error={errors.memory_request} + fullWidth > - onChange("memory_request")(e.target.value)} isInvalid={!!errors.memory_request} name="memory" - /> + fullWidth + /> - + -)} \ No newline at end of file + + ) +} \ No newline at end of file diff --git a/ui/src/pages/version/components/forms/components/ResourcesPanel.js b/ui/src/pages/version/components/forms/components/ResourcesPanel.js index 4cf32323c..8338e7eb4 100644 --- a/ui/src/pages/version/components/forms/components/ResourcesPanel.js +++ b/ui/src/pages/version/components/forms/components/ResourcesPanel.js @@ -140,6 +140,7 @@ export const ResourcesPanel = ({ onChange={(e) => onChange("cpu_request")(e.target.value)} isInvalid={!!errors.cpu_request} name="cpu" + fullWidth /> @@ -162,6 +163,7 @@ export const ResourcesPanel = ({ onChange={(e) => onChange("memory_request")(e.target.value)} isInvalid={!!errors.memory_request} name="memory" + fullWidth /> diff --git a/ui/src/pages/version/components/forms/steps/ModelStep.js b/ui/src/pages/version/components/forms/steps/ModelStep.js index bc2bb3236..e33971751 100644 --- a/ui/src/pages/version/components/forms/steps/ModelStep.js +++ b/ui/src/pages/version/components/forms/steps/ModelStep.js @@ -12,6 +12,7 @@ import { EnvVariablesPanel } from "../components/EnvVariablesPanel"; import { LoggerPanel } from "../components/LoggerPanel"; import { ResourcesPanel } from "../components/ResourcesPanel"; import { ImageBuilderSection } from "../components/ImageBuilderSection"; +import { CPULimitsFormGroup } from "../components/CPULimitsFormGroup"; export const ModelStep = ({ version, isEnvironmentDisabled = false, maxAllowedReplica, setMaxAllowedReplica }) => { const { data, onChangeHandler } = useContext(FormContext); @@ -45,6 +46,11 @@ export const ModelStep = ({ version, isEnvironmentDisabled = false, maxAllowedRe id="adv config" buttonContent="Advanced configurations"> + Date: Mon, 20 May 2024 13:48:13 +0800 Subject: [PATCH 10/26] Update swagger docs and autogenerated openapi files --- api/client/model_resource_request.go | 36 ++++++++++++++++++++ python/sdk/client/models/resource_request.py | 4 ++- swagger.yaml | 2 ++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/api/client/model_resource_request.go b/api/client/model_resource_request.go index 05a3e75a2..ccd65559a 100644 --- a/api/client/model_resource_request.go +++ b/api/client/model_resource_request.go @@ -22,6 +22,7 @@ type ResourceRequest struct { MinReplica *int32 `json:"min_replica,omitempty"` MaxReplica *int32 `json:"max_replica,omitempty"` CpuRequest *string `json:"cpu_request,omitempty"` + CpuLimit *string `json:"cpu_limit,omitempty"` MemoryRequest *string `json:"memory_request,omitempty"` GpuName *string `json:"gpu_name,omitempty"` GpuRequest *string `json:"gpu_request,omitempty"` @@ -140,6 +141,38 @@ func (o *ResourceRequest) SetCpuRequest(v string) { o.CpuRequest = &v } +// GetCpuLimit returns the CpuLimit field value if set, zero value otherwise. +func (o *ResourceRequest) GetCpuLimit() string { + if o == nil || IsNil(o.CpuLimit) { + var ret string + return ret + } + return *o.CpuLimit +} + +// GetCpuLimitOk returns a tuple with the CpuLimit field value if set, nil otherwise +// and a boolean to check if the value has been set. +func (o *ResourceRequest) GetCpuLimitOk() (*string, bool) { + if o == nil || IsNil(o.CpuLimit) { + return nil, false + } + return o.CpuLimit, true +} + +// HasCpuLimit returns a boolean if a field has been set. +func (o *ResourceRequest) HasCpuLimit() bool { + if o != nil && !IsNil(o.CpuLimit) { + return true + } + + return false +} + +// SetCpuLimit gets a reference to the given string and assigns it to the CpuLimit field. +func (o *ResourceRequest) SetCpuLimit(v string) { + o.CpuLimit = &v +} + // GetMemoryRequest returns the MemoryRequest field value if set, zero value otherwise. func (o *ResourceRequest) GetMemoryRequest() string { if o == nil || IsNil(o.MemoryRequest) { @@ -255,6 +288,9 @@ func (o ResourceRequest) ToMap() (map[string]interface{}, error) { if !IsNil(o.CpuRequest) { toSerialize["cpu_request"] = o.CpuRequest } + if !IsNil(o.CpuLimit) { + toSerialize["cpu_limit"] = o.CpuLimit + } if !IsNil(o.MemoryRequest) { toSerialize["memory_request"] = o.MemoryRequest } diff --git a/python/sdk/client/models/resource_request.py b/python/sdk/client/models/resource_request.py index fae8c0445..fc25eb77f 100644 --- a/python/sdk/client/models/resource_request.py +++ b/python/sdk/client/models/resource_request.py @@ -32,10 +32,11 @@ class ResourceRequest(BaseModel): min_replica: Optional[StrictInt] = None max_replica: Optional[StrictInt] = None cpu_request: Optional[StrictStr] = None + cpu_limit: Optional[StrictStr] = None memory_request: Optional[StrictStr] = None gpu_name: Optional[StrictStr] = None gpu_request: Optional[StrictStr] = None - __properties: ClassVar[List[str]] = ["min_replica", "max_replica", "cpu_request", "memory_request", "gpu_name", "gpu_request"] + __properties: ClassVar[List[str]] = ["min_replica", "max_replica", "cpu_request", "cpu_limit", "memory_request", "gpu_name", "gpu_request"] model_config = { "populate_by_name": True, @@ -88,6 +89,7 @@ def from_dict(cls, obj: Dict) -> Self: "min_replica": obj.get("min_replica"), "max_replica": obj.get("max_replica"), "cpu_request": obj.get("cpu_request"), + "cpu_limit": obj.get("cpu_limit"), "memory_request": obj.get("memory_request"), "gpu_name": obj.get("gpu_name"), "gpu_request": obj.get("gpu_request") diff --git a/swagger.yaml b/swagger.yaml index e6cc23940..ddab6be5d 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -1995,6 +1995,8 @@ components: type: integer cpu_request: type: string + cpu_limit: + type: string memory_request: type: string gpu_name: From b22ab30a7517f9f5d8510356358110f5b807ce71 Mon Sep 17 00:00:00 2001 From: ewezy Date: Mon, 20 May 2024 15:17:38 +0800 Subject: [PATCH 11/26] Make model page display cpu limits if configured --- ui/src/components/ResourcesConfigTable.js | 8 ++++++++ .../version/components/forms/DeployModelVersionForm.js | 3 +++ ui/src/services/version_endpoint/VersionEndpoint.js | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/ui/src/components/ResourcesConfigTable.js b/ui/src/components/ResourcesConfigTable.js index cce0d4e9e..d0bee6412 100644 --- a/ui/src/components/ResourcesConfigTable.js +++ b/ui/src/components/ResourcesConfigTable.js @@ -21,6 +21,7 @@ import React from "react"; export const ResourcesConfigTable = ({ resourceRequest: { cpu_request, + cpu_limit, memory_request, min_replica, max_replica, @@ -47,6 +48,13 @@ export const ResourcesConfigTable = ({ }, ]; + if (cpu_limit !== "0") { + items.push({ + title: "CPU Limit", + description: cpu_limit, + }); + } + if (gpu_name !== undefined && gpu_name !== "") { items.push({ title: "GPU Name", diff --git a/ui/src/pages/version/components/forms/DeployModelVersionForm.js b/ui/src/pages/version/components/forms/DeployModelVersionForm.js index 357f7bdeb..b2205db26 100644 --- a/ui/src/pages/version/components/forms/DeployModelVersionForm.js +++ b/ui/src/pages/version/components/forms/DeployModelVersionForm.js @@ -56,6 +56,9 @@ export const DeployModelVersionForm = ({ const onSubmit = () => { // versionEndpoint toJSON() is not invoked, binding that causes many issues + if (versionEndpoint?.resource_request?.cpu_limit === "") { + delete versionEndpoint.resource_request.cpu_limit; + } if (versionEndpoint?.image_builder_resource_request?.cpu_request === "") { delete versionEndpoint.image_builder_resource_request.cpu_request; } diff --git a/ui/src/services/version_endpoint/VersionEndpoint.js b/ui/src/services/version_endpoint/VersionEndpoint.js index dc001a45f..4eccd3d1b 100644 --- a/ui/src/services/version_endpoint/VersionEndpoint.js +++ b/ui/src/services/version_endpoint/VersionEndpoint.js @@ -26,7 +26,7 @@ export class VersionEndpoint { min_replica: process.env.REACT_APP_ENVIRONMENT === "production" ? 2 : 0, max_replica: process.env.REACT_APP_ENVIRONMENT === "production" ? 4 : 2, cpu_request: "500m", - cpu_limit: "", + cpu_limit: "0", memory_request: "512Mi", }; From 9c82fd2191d04e73138a45142a15d22744c8d117 Mon Sep 17 00:00:00 2001 From: ewezy Date: Mon, 20 May 2024 15:48:06 +0800 Subject: [PATCH 12/26] Update SDK to expose cpu limit field --- python/sdk/merlin/endpoint.py | 1 + python/sdk/merlin/environment.py | 1 + python/sdk/merlin/merlin.py | 8 ++++++++ python/sdk/merlin/resource_request.py | 11 +++++++++++ python/sdk/test/cli_integration_test.py | 2 ++ python/sdk/test/client_test.py | 2 +- python/sdk/test/integration_test.py | 8 +++++--- python/sdk/test/merlin_test.py | 8 +++++++- 8 files changed, 36 insertions(+), 5 deletions(-) diff --git a/python/sdk/merlin/endpoint.py b/python/sdk/merlin/endpoint.py index 081196221..74f7377c3 100644 --- a/python/sdk/merlin/endpoint.py +++ b/python/sdk/merlin/endpoint.py @@ -78,6 +78,7 @@ def __init__(self, endpoint: client.VersionEndpoint, log_url: str = None): min_replica=transformer.resource_request.min_replica, max_replica=transformer.resource_request.max_replica, cpu_request=transformer.resource_request.cpu_request, + cpu_limit=transformer.resource_request.cpu_limit, memory_request=transformer.resource_request.memory_request ) diff --git a/python/sdk/merlin/environment.py b/python/sdk/merlin/environment.py index 6374ac1b5..e202f967d 100644 --- a/python/sdk/merlin/environment.py +++ b/python/sdk/merlin/environment.py @@ -27,6 +27,7 @@ def __init__(self, env: client.Environment): self._default_resource_request = ResourceRequest(env.default_resource_request.min_replica, env.default_resource_request.max_replica, env.default_resource_request.cpu_request, + env.default_resource_request.cpu_limit, env.default_resource_request.memory_request) @property diff --git a/python/sdk/merlin/merlin.py b/python/sdk/merlin/merlin.py index 5ba69cf4c..c1a929880 100644 --- a/python/sdk/merlin/merlin.py +++ b/python/sdk/merlin/merlin.py @@ -63,6 +63,11 @@ def cli(): required=False, help="The CPU resource requirement requests for this deployment. Example: 100m.", ) +@click.option( + "--cpu-limit", + required=False, + help="The maximum CPU resource this deployment can consume. Example: 100m.", +) @click.option( "--memory-request", required=False, @@ -78,6 +83,7 @@ def deploy( min_replica, max_replica, cpu_request, + cpu_limit, memory_request, ): merlin.set_url(url) @@ -91,6 +97,8 @@ def deploy( resource_request.max_replica = int(max_replica) if cpu_request is not None: resource_request.cpu_request = cpu_request + if cpu_limit is not None: + resource_request.cpu_limit = cpu_limit if memory_request is not None: resource_request.memory_request = memory_request diff --git a/python/sdk/merlin/resource_request.py b/python/sdk/merlin/resource_request.py index 89b2baf53..242b6b689 100644 --- a/python/sdk/merlin/resource_request.py +++ b/python/sdk/merlin/resource_request.py @@ -26,6 +26,7 @@ def __init__( min_replica: Optional[int] = None, max_replica: Optional[int] = None, cpu_request: Optional[str] = None, + cpu_limit: Optional[str] = None, memory_request: Optional[str] = None, gpu_request: Optional[str] = None, gpu_name: Optional[str] = None, @@ -33,6 +34,7 @@ def __init__( self._min_replica = min_replica self._max_replica = max_replica self._cpu_request = cpu_request + self._cpu_limit = cpu_limit self._memory_request = memory_request self._gpu_request = gpu_request self._gpu_name = gpu_name @@ -44,6 +46,7 @@ def from_response(cls, response: client.ResourceRequest) -> ResourceRequest: min_replica=response.min_replica, max_replica=response.max_replica, cpu_request=response.cpu_request, + cpu_limit=response.cpu_limit, memory_request=response.memory_request, gpu_request=response.gpu_request, gpu_name=response.gpu_name @@ -73,6 +76,14 @@ def cpu_request(self) -> Optional[str]: def cpu_request(self, cpu_request): self._cpu_request = cpu_request + @property + def cpu_limit(self) -> Optional[str]: + return self._cpu_limit + + @cpu_limit.setter + def cpu_limit(self, cpu_limit): + self._cpu_limit = cpu_limit + @property def memory_request(self) -> Optional[str]: return self._memory_request diff --git a/python/sdk/test/cli_integration_test.py b/python/sdk/test/cli_integration_test.py index 57eb42b25..5dfac8018 100644 --- a/python/sdk/test/cli_integration_test.py +++ b/python/sdk/test/cli_integration_test.py @@ -40,6 +40,7 @@ def deployment_info(): 'min_replica': '1', 'max_replica': '1', 'cpu_request': '100m', + 'cpu_limit': '2', 'memory_request': '128Mi', } return info @@ -136,6 +137,7 @@ def test_cli_deployment_undeployment_with_resource_request(deployment_info, runn '--min-replica', deployment_info['min_replica'], '--max-replica', deployment_info['max_replica'], '--cpu-request', deployment_info['cpu_request'], + '--cpu-limit', deployment_info['cpu_limit'], '--memory-request', deployment_info['memory_request'], ] ) diff --git a/python/sdk/test/client_test.py b/python/sdk/test/client_test.py index 1f2a85dc2..b22cccb15 100644 --- a/python/sdk/test/client_test.py +++ b/python/sdk/test/client_test.py @@ -47,7 +47,7 @@ def api_client(mock_url): created_at = "2019-08-29T08:13:12.377Z" updated_at = "2019-08-29T08:13:12.377Z" -default_resource_request = cl.ResourceRequest(min_replica=1, max_replica=1, cpu_request="100m", memory_request="128Mi") +default_resource_request = cl.ResourceRequest(min_replica=1, max_replica=1, cpu_request="100m", cpu_limit="2", memory_request="128Mi") env_1 = cl.Environment( id=1, name="dev", diff --git a/python/sdk/test/integration_test.py b/python/sdk/test/integration_test.py index 1231a2c9b..02f13e7b5 100644 --- a/python/sdk/test/integration_test.py +++ b/python/sdk/test/integration_test.py @@ -1231,7 +1231,7 @@ def test_redeploy_model(integration_test_url, project_name, use_google_oauth, re # Deploy using raw_deployment with CPU autoscaling policy endpoint = merlin.deploy( v1, - resource_request=merlin.ResourceRequest(1, 1, "123m", "234Mi"), + resource_request=merlin.ResourceRequest(1, 1, "123m", "2", "234Mi"), env_vars={"green": "TRUE"}, autoscaling_policy=merlin.AutoscalingPolicy( metrics_type=merlin.MetricsType.CPU_UTILIZATION, target_value=50 @@ -1249,6 +1249,7 @@ def test_redeploy_model(integration_test_url, project_name, use_google_oauth, re # Check the deployment configs of v1 assert endpoint.resource_request.cpu_request == "123m" + assert endpoint.resource_request.cpu_limit == "2" assert endpoint.resource_request.memory_request == "234Mi" assert endpoint.env_vars == {"green": "TRUE"} assert endpoint.deployment_mode == DeploymentMode.RAW_DEPLOYMENT @@ -1269,8 +1270,8 @@ def test_redeploy_model(integration_test_url, project_name, use_google_oauth, re metrics_type=merlin.MetricsType.CPU_UTILIZATION, target_value=90 ), deployment_mode=DeploymentMode.RAW_DEPLOYMENT, - transformer = Transformer(image="gcr.io/kubeflow-ci/kfserving/image-transformer:latest", - resource_request=merlin.ResourceRequest(0, 1, "100m", "250Mi")), + transformer=Transformer(image="gcr.io/kubeflow-ci/kfserving/image-transformer:latest", + resource_request=merlin.ResourceRequest(0, 1, "100m", "3", "250Mi")), ) resp = requests.post(f"{new_endpoint.url}", json=req) @@ -1282,6 +1283,7 @@ def test_redeploy_model(integration_test_url, project_name, use_google_oauth, re # Check that the deployment configs of v2 have remained the same as those used in v1 assert endpoint.resource_request.cpu_request == "123m" + assert endpoint.resource_request.cpu_limit == "3" assert endpoint.resource_request.memory_request == "234Mi" assert endpoint.env_vars == {"green": "TRUE"} assert endpoint.deployment_mode == DeploymentMode.RAW_DEPLOYMENT diff --git a/python/sdk/test/merlin_test.py b/python/sdk/test/merlin_test.py index 691a8b7fc..55be538b1 100644 --- a/python/sdk/test/merlin_test.py +++ b/python/sdk/test/merlin_test.py @@ -24,7 +24,13 @@ # get global mock responses that configured in conftest responses = pytest.responses -default_resource_request = cl.ResourceRequest(min_replica=1, max_replica=1, cpu_request="100m", memory_request="128Mi") +default_resource_request = cl.ResourceRequest( + min_replica=1, + max_replica=1, + cpu_request="100m", + cpu_limit="0", + memory_request="128Mi", +) env_1 = cl.Environment( id=1, name="dev", From b69c7723a99fe9cd57944e670aaecd9c14ee4910 Mon Sep 17 00:00:00 2001 From: ewezy Date: Mon, 20 May 2024 16:20:54 +0800 Subject: [PATCH 13/26] Fix sdk deploy method --- python/sdk/merlin/model.py | 4 ++++ python/sdk/test/integration_test.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/python/sdk/merlin/model.py b/python/sdk/merlin/model.py index 975ccbe37..7a013777a 100644 --- a/python/sdk/merlin/model.py +++ b/python/sdk/merlin/model.py @@ -1268,6 +1268,7 @@ def deploy( min_replica=resource_request.min_replica, max_replica=resource_request.max_replica, cpu_request=resource_request.cpu_request, + cpu_limit=resource_request.cpu_limit, memory_request=resource_request.memory_request, ) if ( @@ -1748,6 +1749,7 @@ def _get_default_resource_request( env.default_resource_request.min_replica, env.default_resource_request.max_replica, env.default_resource_request.cpu_request, + env.default_resource_request.cpu_limit, env.default_resource_request.memory_request, ) @@ -1762,6 +1764,7 @@ def _get_default_resource_request( min_replica=resource_request.min_replica, max_replica=resource_request.max_replica, cpu_request=resource_request.cpu_request, + cpu_limit=resource_request.cpu_limit, memory_request=resource_request.memory_request, ) @@ -1807,6 +1810,7 @@ def _create_transformer_spec( min_replica=resource_request.min_replica, max_replica=resource_request.max_replica, cpu_request=resource_request.cpu_request, + cpu_limit=resource_request.cpu_limit, memory_request=resource_request.memory_request, ) diff --git a/python/sdk/test/integration_test.py b/python/sdk/test/integration_test.py index 02f13e7b5..ade7bcc4e 100644 --- a/python/sdk/test/integration_test.py +++ b/python/sdk/test/integration_test.py @@ -1237,7 +1237,7 @@ def test_redeploy_model(integration_test_url, project_name, use_google_oauth, re metrics_type=merlin.MetricsType.CPU_UTILIZATION, target_value=50 ), deployment_mode=DeploymentMode.RAW_DEPLOYMENT, - transformer = Transformer(image="gcr.io/kubeflow-ci/kfserving/image-transformer:latest"), + transformer=Transformer(image="gcr.io/kubeflow-ci/kfserving/image-transformer:latest"), ) with open(os.path.join("test/transformer", "input.json"), "r") as f: From 8a49a81b3703f765717120a0ca0ef95e38ebf70d Mon Sep 17 00:00:00 2001 From: ewezy Date: Mon, 20 May 2024 16:28:14 +0800 Subject: [PATCH 14/26] Revert redundant changes to cpu limit of default configs in sdk tests --- python/sdk/test/client_test.py | 2 +- python/sdk/test/merlin_test.py | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/python/sdk/test/client_test.py b/python/sdk/test/client_test.py index b22cccb15..1f2a85dc2 100644 --- a/python/sdk/test/client_test.py +++ b/python/sdk/test/client_test.py @@ -47,7 +47,7 @@ def api_client(mock_url): created_at = "2019-08-29T08:13:12.377Z" updated_at = "2019-08-29T08:13:12.377Z" -default_resource_request = cl.ResourceRequest(min_replica=1, max_replica=1, cpu_request="100m", cpu_limit="2", memory_request="128Mi") +default_resource_request = cl.ResourceRequest(min_replica=1, max_replica=1, cpu_request="100m", memory_request="128Mi") env_1 = cl.Environment( id=1, name="dev", diff --git a/python/sdk/test/merlin_test.py b/python/sdk/test/merlin_test.py index 55be538b1..691a8b7fc 100644 --- a/python/sdk/test/merlin_test.py +++ b/python/sdk/test/merlin_test.py @@ -24,13 +24,7 @@ # get global mock responses that configured in conftest responses = pytest.responses -default_resource_request = cl.ResourceRequest( - min_replica=1, - max_replica=1, - cpu_request="100m", - cpu_limit="0", - memory_request="128Mi", -) +default_resource_request = cl.ResourceRequest(min_replica=1, max_replica=1, cpu_request="100m", memory_request="128Mi") env_1 = cl.Environment( id=1, name="dev", From 42c751dc690dcdbfd1a9b0958697c4fe23f86785 Mon Sep 17 00:00:00 2001 From: ewezy Date: Mon, 20 May 2024 17:09:26 +0800 Subject: [PATCH 15/26] Fix incorrect change to sdk integration test --- python/sdk/test/integration_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/sdk/test/integration_test.py b/python/sdk/test/integration_test.py index ade7bcc4e..3f4fc0dfe 100644 --- a/python/sdk/test/integration_test.py +++ b/python/sdk/test/integration_test.py @@ -1283,7 +1283,7 @@ def test_redeploy_model(integration_test_url, project_name, use_google_oauth, re # Check that the deployment configs of v2 have remained the same as those used in v1 assert endpoint.resource_request.cpu_request == "123m" - assert endpoint.resource_request.cpu_limit == "3" + assert endpoint.resource_request.cpu_limit == "2" assert endpoint.resource_request.memory_request == "234Mi" assert endpoint.env_vars == {"green": "TRUE"} assert endpoint.deployment_mode == DeploymentMode.RAW_DEPLOYMENT From 80248a117830d40e2c2b1f2a8c03819cd8be4395 Mon Sep 17 00:00:00 2001 From: ewezy Date: Mon, 20 May 2024 17:49:34 +0800 Subject: [PATCH 16/26] Fix integration tests --- python/sdk/test/integration_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/sdk/test/integration_test.py b/python/sdk/test/integration_test.py index 3f4fc0dfe..c6d4791ea 100644 --- a/python/sdk/test/integration_test.py +++ b/python/sdk/test/integration_test.py @@ -504,7 +504,7 @@ def test_resource_request( # Upload the serialized model to MLP merlin.log_model(model_dir=model_dir) - resource_request = ResourceRequest(1, 1, "100m", "256Mi") + resource_request = ResourceRequest(1, 1, "100m", "2", "256Mi") endpoint = merlin.deploy( v, environment_name=default_env.name, @@ -560,7 +560,7 @@ def test_resource_request_with_gpu( # Upload the serialized model to MLP merlin.log_model(model_dir=model_dir) - resource_request = ResourceRequest(1, 1, "100m", "256Mi", **gpu_config) + resource_request = ResourceRequest(1, 1, "100m", "2", "256Mi", **gpu_config) endpoint = merlin.deploy( v, environment_name=default_env.name, @@ -1271,7 +1271,7 @@ def test_redeploy_model(integration_test_url, project_name, use_google_oauth, re ), deployment_mode=DeploymentMode.RAW_DEPLOYMENT, transformer=Transformer(image="gcr.io/kubeflow-ci/kfserving/image-transformer:latest", - resource_request=merlin.ResourceRequest(0, 1, "100m", "3", "250Mi")), + resource_request=merlin.ResourceRequest(0, 1, "100m", "2", "250Mi")), ) resp = requests.post(f"{new_endpoint.url}", json=req) From ad7d6537065fab9bec9aff8ff29af0d21dc14363 Mon Sep 17 00:00:00 2001 From: ewezy Date: Tue, 21 May 2024 10:58:17 +0800 Subject: [PATCH 17/26] Update tool tip and form group descriptions for cpu limit form --- .../components/forms/components/CPULimitsFormGroup.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ui/src/pages/version/components/forms/components/CPULimitsFormGroup.js b/ui/src/pages/version/components/forms/components/CPULimitsFormGroup.js index 26a1387ae..a4a0bd1ff 100644 --- a/ui/src/pages/version/components/forms/components/CPULimitsFormGroup.js +++ b/ui/src/pages/version/components/forms/components/CPULimitsFormGroup.js @@ -15,8 +15,7 @@ export const CPULimitsFormGroup = ({ title={

CPU Limits

} description={ - CPU limits are not set at the platform level. Use this field to limit the - amount of CPU that the model is able to consume. + Use this field to override the platform-level default CPU limit. } fullWidth @@ -25,7 +24,8 @@ export const CPULimitsFormGroup = ({ label={ } isInvalid={!!errors.cpu_limit} From 450bd94cf48765d7dfa36cc0625e04bc5a0204b0 Mon Sep 17 00:00:00 2001 From: ewezy Date: Tue, 21 May 2024 11:16:36 +0800 Subject: [PATCH 18/26] Update docs --- docs/images/override_cpu_limits.png | Bin 0 -> 64952 bytes .../01_deploying_a_model_version.md | 24 ++++++++++++++++++ .../01_deploying_a_model_version.md | 24 ++++++++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 docs/images/override_cpu_limits.png diff --git a/docs/images/override_cpu_limits.png b/docs/images/override_cpu_limits.png new file mode 100644 index 0000000000000000000000000000000000000000..36219274b03c1b0f12f119cc8d7767cbb277b40b GIT binary patch literal 64952 zcmdSAby!u+_dZOg3KAk9-K}(olynNxNOw0JQcyzb(A^!<(jd~^-QC^wZuDtB?{$6u z{mJEFbM~H@HEXSzxz}3v_Lr3weTsyG1OWl@RP3FQJOl)cGz0_`7Xlpc%@Oq6PY4L4 z3S&V*SusIDLRlM217kCN2#9z7k&QZ(i>8t(hyWcsX(lfQ$HSX!tmhA!>QuJ zmdmh8Plk6x8c~ZV@FkSS?m&2|!tm?T$dGr&fBRa2ex=0^rN|Gj%1`hT8d=+CY#oZ* z6GhefH3GC8A$|`$cPLz`Ck!o&uQn_tT&6bk#FM-x>2J6@@SmHW55P!!+b*DehqLgK zK0xRYU~oe8YjS;!B!G&a#?Sl?nbA{{$g~4e=dETa9|n>z|Dh~e1d=}?wjx5Iw|FXJ z4$>6t=-bA(6e+d?m^sic5cY!51GpRLK}~uw&$gQl4xYpymMx&NBPGFS|KPCVRfj3| zp;^SaAe=!+_9<>o{PxNn5rqUo?kzK_;7_=Z#Ac*V#Q5b=5lbP<7vcDjDAwMsAw=*MujgIBee zzvWK&PUx7yIxw8EpAlE0PX~fRS{9kD*4UNRQR(rcgK3%yTg}^97W$Srt$4nF6qo$(5rmCjul|!SZ zQ_fT5EU;6_Rof&{`D!L?07ma*JmNg!JyJP>K2k&S)e$*NC6%?I+NP|CIF87UK)N+ossjx{uGGA3zBp!|EO6$vBQ=rKa&1K8=p%%hq!mLD6*3T8M=t7L7kNo&P zF0bAs#w2P>brd>JS@C-Gz{J4h&cv;PxMJ31ebmJWp*N;2>(D6Q$khnTxZdv7Y(igZ z_OQg}=Z)PHB6}8liqn^;!ds(T*(0ksU;JX>L`f8M>)W1xb;U`}+LY3jLYGuyDs-RP zH71GUjN^@i>P72?>y_@6QGG(^QTU|rtE!Z$O!=EBr77ETER+3#h8&(8!STa^#m$I~ zSA)(u$lI!$A{#F@_QvXqxC*k1v2rNJrt=&%EoHR?=SBH68rAR1?>r%Uo4WapWG$@P z7GMPvn|1UUsj;zM3i{Rf3HUL#X!z~=rHibI42sZv#R*D!2@Z;FqiS1iD{K=aha)eO zM3UN)#E(Ohip?_4Qj?sP6z%Eh)#+7=XNf(HH;uFHIqu!=AyAX0t)sJ{HKE;8U80SR z!j77zvs7tT>QgdS8dYK|{Ah}0ntJN69lSZQX)!E3Y?|db8EXE4)tS|vb+!_xGPRP% zJl;&J_jPR8Yu)9bN^Gs&gk8qnvk9J3irr?HVl5s;p6?9HkN{xMUG$3nFw?V z8u~Spd85c^6754zyT<#Jmt1EYKkfEu{@~sId;xnEc~Nnle<^yce98CpHPXAMImj=O z?4HpgaN~Y@ii9}za-?-|`GCD5Ays>Pfvg$pZ3j^p(R-r)w}o%XQt8STRGQ0JdtJDv z?JA8N6kZ}Me=0G6Iw6W7QZ771j3tEs#f@MUw}N*|u3WgB zwuZ$I2mDe2mkDhfOYH?sNc{6E(mmBWhq~7d9FWPV;BRanor>;fdTm|N6VU_F_j{R= ztBmF%FQ<-~){|BFHJUW6m=WVK;_V1Gnb?BVTJWUTNG&Dsmr9(~oER>7Z*1;7n-pZp zHzW=vsU$XKP9cL!d@F-cj|e(tmH8{`mr zc@0ukJ5b5!5=-DUu#7#ZnlKKKRFR(|D`Pk?e59MKz)+2=E4Pn5EKSp|?}m=v*Ah?- zD_Lq2uZ`5FnJ)~`7?saZF0ad3|J+2d6%ZLn)~b=uqqKkX{5buf{fs5pw6f?@^#?<> zs)@3y+?6w(Vr9Mx*fw>8-VMKI9 z<8+<&aU@c9K%&|^H9fVJ$kfPQHK|IAiK}Yr!o0Gns+y1SdzCEHB9+Hg^K*-3>F(Kc zeM5=K?l11#Jd_uT+HlI^0%pI@I@c!e`%ey^As60Sbm&dl@`?HW^!nb#Y_(mq?YZjuPG7L=;Q5{-nM3FP@%}kq zhugu`$GMh4(QaTFMRUtJ%O5J98~5%0=jimXs=#vor#hme z8wURXY<333fdK(TgcF2O6$D{#VoYv3Y@8FyONxVJC!IZfEG!5IM@ZE75d0fYAu5$p zN$7Br;}H1SF)LglCI;n7$k*o&RX~!8&(bVON8NiuC*b?hlRZ5;e!jZbn7tp-rp-gY z$iBmZtw20{iMcxl;8-2~4`K#VQV`U@H39?_BpL)Xa0LncKzufafcbR|0YMJ@hJb($ z_k(~3exm{ZWmBO3ybB|p0{!PT6xYLp0t$j+V!&?&T^oIU3tJ;gJLedCPXIgf8Y_OV z`yeIBscUIY|549UN1xuw-0GnT1h*3>aA~e@_mR-a+|0t3(}{=p*Atw;^~2W;#Du>d zu`}f%{vahwC}?S;PsmFDn*KF0FA^akA-9d50jIo>$REvtzj%m^?Ch*K85kTL9qApJ z=q+sw8QySka4@`PWME{Z1D>F>b+)kk=tO5>OY*yuKl>5Vx7D>Vwz4y}v><%w_oI%b zy&VrR@xwrW{rfGazLW9aBU#w~F)d(%3=el0-q62h_+QQJj1B&Onmye4-R#%Aeh5$!v6H@;ijc86pjALLUKX}D+`oo-bmwnDf46*Zt8XJ{X$~~BDE*&Hpz1 zHROlChkWzqHRC_W{Ku_-G<}!@r;Lp;FxZ|2F5ZZvlJq zB5^bPWB>C=k%q*Xyl&9*Y(ABpHE{Bb-xNHOe}s$uAq*3`C|JU(RYRV?3@ z5_)O=+Ythx>^G>Q5B@YOi$7^k0#wXRlML?;qhfECx%TKB7ldrzGEaoR(19j%k)wbJ zuZHZS#2EKg_X=Oe8io75Z3qnsgUS!_uMbK^$bNP|%?WIcwCSWb$cEEx%%%SNdQTE! zCIS$BCOQIXjFz6@y)pb8V}9#l%fCHjBdy)KH<^~?JoO6ge~5PgCH+lEYq%!`_3QhM z%3zX`k|$-$5gcY&bsv_}VnZZF& z_VXg$8epoWJRh)A)MW^AY!76aS?LN*KteeojLC310X(uMnR;l?c%_ zG9wvGMND}uOueE2Z-RWc;+({x-yD%qwm%ey5U zl_jJob^&k)+M;}#(5FJ?+nWL!3atMjE*uoj(Lgg%doY>UQi&cHj>753I{-!v=4IAaJq~fwEwnHYv81>Ugxw+#hgK7;}|LXw!5ME1Ep$Q4Z zpHV0#I5=@=2MO@l(>iT?(J5q(Pa%Kqy={l4(zq9KO3AaYz7rFoE-OO1#L{j*%aT)L zm#Nmcl?4>U$Jr23x{sQbLwGxxk3~L%Cz||SGsLni45O2CUp|)I(^Csk z59_!ifgo;F?^U+!l@Vs+7W-G>N8SQs^GoO_Kg;U4lys1Mw_M0oG7F@8@?JSVQi!J~ zACeyK_3IKe3e|Al_Y}J4JN}a}ii}ZuD3AOI^*{mgX{cU+$9KnJwwb^d6Re7F=nB@t zN(mPM$xs7kk9IzOLJ3F;D4ivuE04V|Vr@S*z0gA-{MQXwDfA1xc^*IB)Cj`}BhIZu zgd<`d9@WXd^m93d&Z(}z_F3`&5d0o_BBuSYsr~Sf0=qVf$Z83jZLpcVU8dg&&Gmmg z&WCK&L((D?Z{|(7*CfW;emm(|`9B;=6Il-v*BV}$yM-2e?b1nu2ln}Y_H*FRd$bNz z=+4(Lt)n3a2{quwl?0?U8?)7}#+8s(3C4(kprD8>DIAI6EGbfIYU-9Z4v!th%Nyf> zbuCPH;ivLb^^@*mMzf1y6(!{?N#oeNI}GFDEc>YTPNB!P4G?dFljdGO(5nh4D8{)+ zb_ZqZb%vL^U5dw^FS~KLU3^iWTc%d2b;*pNe&3}Wor?cI^h^WU|KW>akdTbD*78r1 z>H5vNgDv8@7cLvk@EP~IAsr*<h@pj zj<&YLV`4Vw<3mFUNs?Rz=yzOlM+#>7Ikl`xo`9CbIP5OI)dpGAvDwGXOjRWpigrKq zI5M7W0DZ3Q@$?d0DeW!}#S&@Q=~JtDdu{K>D+N`ev0zX(H^+_E>RMah&r) z6J#xDn69cjoXlg&=1QXK)UY>>g>mbvvU$_`8P{G!vWMmdJ(J^}V;Ud)89{`*0ZcHt z<<&}&=hH_f?Wp39DhZK%6TU1n+IhZOm(MS%^oqwtF^W!$C|xwfHhH80#HCVUQo&_( z^m9O<)OcC-QoBu2Z7siey~AMp0uZG}ja`&@Bu!_LMy(kS zaBoXfxhq-1JyL8xsOr>g;+veN>#mb_+bjeFtDS0CaLlFAnPl~L-+f2QkeeqixGQPHBU5!;rnB13{GU zGS0_^Zyajv^P`5Y{kC_`K<9HLPjN;Z`DRr_o7u&(5sj0n-{)=U`3D#r zSS67+BiH@bWvz;4Q<3=TH+C_sX8R$~Ip2m^jn;UfhNX`{^=T&v3ir_lC~E}ei_ac= zd}?^M$+50A@ZN5!2gRTeO@kprz1jwQ499wmTFzg$J$TYn%r2Jk#oCljv)$R#jS-wg91KiM{r>4{+i1|}!dE!)VvEVO+;w4C0JqKlhWzO8WF*BqpJ-7CX2TJ)T8 zxBxSaM0l)tUOQRmcuh@r^@O!>bu-&d6Ab71oo?n};DNs^(A z?V1hm@~P}N7TuMy{sz?D;Upew&C^b$9}JJ&m{QRzl;p5|NM2q&q)$pp3UhXh;3@Hj z%q-`dON)UEM9b;+3lrh|J5BPz!NIigEt`*QTsnfs_v$x+u}TV^+!|{ZvB$mYy`=(v zi3TOg>(75AdvL1ZuvyS%S8Z#H#%w4i^dh0ee$4Ul~7OGX1hM>Ht=|Om#A-cWNjXry zj7y{pJVPvRnT^r}U^VVZ^CMznC^?CRebC7yqI&Ya^bWgFD^ez#V{5?c@6Unc3sozg zm?rC~eS85Q4hHMD2PewcyPTSB55OAhR<*B9Xj)!J2%~pAM*a-2b$Hg{R90!jcIw0S zZXEC1yIB>UJceeJ7+&bm6sZPz!KMMysWPKf5%{)`w^aV7b|~M{Hvf=mN&83h^5joN zKxL=GFrfE-gjDY+`4@f??Bkd}3LCxNJ}0!2r|eOOc_h@09N`QBu2lSU%08$o%3Y1x zWxegnU7Hi5w0G~mL;x6Rs>0-5WEWbcJ(rRctT8ox48LYn)&%*EP>+WZ$<1CT1v z%}LwfZ&2EZiGa&&)baWjVl1P$PsYR~Z$swNX3%|_0|$Qs3K0?UXs|uGc`cOKxs;=V3Tl(#>!f2b-E-~Op**SD1mx_W#b`O7s zH&vUeRCFT&xHJQcY1_NZ0LyB|M( z?1Z>~9`LC3{q(&-!pS|G{cd+DgP|D_0VC1VU9Mq$?!MCpg@;_C2dn4y`n;Q|pO<0Q zRb9WTLSG`9zU=AER(`Sm9Ln*k|3{h5>UAy32bi?xakOd3=-j1PyL&TLZd?|G%O+Q8 zF}tQ$twkZ}CQ9DG)9H@raTz(TxA;SNPq)u6n03%j{FW2#1wQ%W^N(m%j>$WdWy&}) z+)|&P;guBM{Br+}Mt)V*JjA5_a5>B1FyTP=UfIQRVAk)x7g~k+c1DAzLlc2~(QO z>DJK=GQ(6I4vUFiHT{cZ77NBdc5+G#NGJ;5A3efubO-HAWR+d;dyV57_91)G5{ilu zibaZ@iR;ya1u62Ox0dy7Z%-0uPZxE3z-I>5<*--fIII-MzU3$}6m;oI_ES%L` zHIPaFI1?*58k0sTAecOXdzB|wAv>5>kWE|;|ymAyJxzFSh)rKJq{O!VwvPTz|HYf`7*L3YXlOiZ4FRpp4fCscP7 zeRR!QU?H4%LWWHKrputO&DAFMddYpQHMl^vLRF;UqSUBK@E$29dZa%{C-!vF&UFg_ z*OQIn21OLqc^Q}6Iyt1|_)H`c@}xg8a%58zgyI%Gx6o-kW!J86E4uK@Oh7xAcDLqjwdAA_Pq>?XuNL@o;jB z73UX)O2xBviL2Z$viMLFbv) z>3)45&uIV;Li*c-L=rHNn6M5j_zP88R<8-xCR>ASUF-Hw>n&}xG_jt!Z8TrcUdOT} z_=YpBZ{GnRF|@{hyCXn9a2C18fL@s?E6(i3F|dHGIZpKrOV1)IB9q`x^kDpoOI1xD z%0Jc@(nd#M{MC$+sE4|Vh|`8$9~F2$${(Ggnqj2?G+HuTy~jQKp~B^>>wK(ux`F}o zXt7rR&5PXkwNcJJb*8uZfLBd$Kg}`%7Pt@Sc6+}u(;-pS;o7@zecG7Qq5Y<2ZAyiC9qTb5Cfgh5nXOq|> zyDcUGayg@x5{uOl;P_LGTgIoH#W!OOw>?@fYrAG0Z^uEc*1B`dEk&jh_Qzxu;;UVI zH>xbAQ;n!wlo)vOg~qpjPtV@3ml{U?=#3-mQ*bWj)a-nM!e@dnP!K{ghh8NU{K`LC zmI;mAp`9L=KUagMEhwBSb~Jb8S0ZmkVIj`c#lOgQ=>D)-zS@S2<2=EZ5k{K%L+WNb z!P?X&7>9Ozp%zZ8(rj|;bTmKHL*!?By9CuSOa(oNg1`ns*x2dw#qQg=qaY)#hBM+`Lv_ZrDA%xGz_j z&LWt4bjl{lp4M!i;DWeIv$`3!qcjH9k1q>4BftC+-(4trKBwSa%_#kWmZMx;r18me zVSz%Z@w>^89Eq-INcgke{%_aSQj67FT}cLBjd*9d$j9sVdi{eCfn#Bp48CV6(T$Qj zLgLZ%`W|)MzB<2vmotXd+;-wP;K15G8#vni33jPtL_P%3%9@~AUOXv2wj^hh@7UJz zy*yVfxuWykJYaCKvughU5%g-|pzbbs#;&)}AH8jYD)A};Pbz8E-0w0onTyjm$2whP zNfOZ}IB^G9lN2-}&cFTgh#u93^zMVIxOhMz5erKzFa7&-`&Q@MTr~>v{R&%hC||;F zK&FL@EX$y`?zSo}E{gJ8f%O4pu6a@)IC(S0eF_x! zJ5TMYg^bF6xIHbp{D*8!1TaUnS^j({VXVZzvf!vWP~7h+WcO)r9|3YNH=uDR8xiKe z%TIt!`BRWk*%3#$BpQD?=086O=b>PvkEGtgKEm9jv56KJeXpyqc~Jt!Jy6QwAYr_fQ*!uF zyp@xZureULz}_R7dr+9{Q4z%OC}aVbMf92C!tbg}$#xyQG$Xox&AAj~;JdxNO#mtB zckfC4bdpRVp$}2|pv2B~uVUfchsJU~w#T$i$>{%dX7ccj&)ueTX)2ZU71yJKyq_cEiUge|`Fpo^{5e z?CJNZs|sg&-k|~Pt$QL7lzvb(Sz?M90$?2{Q=^B*+OQ}89dkt{A`ylrRTG)={aRVKlHjw9kDDb&<5#k zB;#&kPrqT0p_Q92KpmE_&n@#NWZN0`%OwXJexJIC*Z zGXcfrxEr-Q?YL44Ms`;vMaH^|+jq(NQwK=WgmnvfSq3lx(fs{1mZ2JJfklQU?y5}K ze0xtF$ENos!*cgM2oDbr)uNFkI+iTAKYX(wA#vu#WJC}_`uhNa^`IT!+ zK5$MBW$eY7o?2h-d>QTZALf(g2JDFua?MYJ^nqVA#Ask8Hrr1R!6t%I?gwZ!gzxfg zn~c-AyMV=HX*GcPUtx3ZsN|{^=9M#E$1xj~nq7)JTOK*(%BI!@x!5M$?59F9+IPX! z?iU69o|Fy^DuC-%t4xS|;CRDRzedt1S6)kNuNItW`zuhsc_X)1ef}Icb@qnND|ypM zFhse}OSumk?~X3BnaG%#6`4cR1R@F6=t3pv9bn}Wj;kzY4S>4gevrv4wl$XZ*MAmO zWF=vNc?{zUmbuYbZ47)=`>4S5mNZM`lomKbuFCqy-j-YPsKP)M(g*1-TuvJcI~4(^Xmx*X|7?g=%Vk z&-NJjZtLF7C7*fmWRJh+SvhN9Gn|BD{Iyh&G_@a6CAHFYx$pB0qOh*7_ur@OpPEls zzS%$Inbkjw@$A-*c8H9>QnwTh-0-rPt(hY9Z`cR;pK@J7(Iu;f?gxX2USJu%>n@Gx zvcs_$e>4^1w|h0m&nN=_fM=+b22cgo@;w#^y;LtDZ{Ox`R?WZYE(Gm|H#p9aaMDpw zQBAI8XJvKP*y94FI4ZX6q$x!p4%6#4n|@)ch0AL5w0$g*QNQb5i{DeB`+ywd@jV_O z1h(_1Rgs6|x%>8t!%fIy-?7;3dY0RHCG~pc2mN~YCx7s64yG&A?~?&ZLt92Y;n`&< zg&!Y{vX(6vq*1cO;K+sVa-hkvUspDJ4MfW1gSz_NB3vp*fw2zh3S2Zw1q*${{8xPU z1yQ5<<2A>@_)-xnWqcI^4vtxq6Xv{ZKqB4Qpyba=Y^x-&RlcbuV$RlvIoq`bQ6(CP zfYco}TZj5?Fy*pWxo>xn&%CX#$fh!=H(SbggD*3+{MgvNz$ceG@2Qn-o=q#`HJ2I> zzi><{ay|NXsRL?;UGKzmoY5JV00Jk7+WUq3y^>WR(Yit3|0`$9zlgLmiB6FWx~mY1 zU#JRZo+;6dzFF^s*Xxb5rx*=Q_P7%^@HmO(wwg~wLql8QRfgWW;y6B}NUl9zyZzo| zq2*?c<%s`h;(CDqN^iJQ(|%ohdIX(3%>4tj%!x1 z$o^2dKO!_#JA2M3$yW3254l5iTrytX<|pU8chMsKax{HfWTuLaQYb?VV^>(+8M^J< z3@xk=;EtaNshp?&>q$*sw)()Oy6Y(ZY;9+6)r~SQlL8ZggyPWx=b5Q)qP71XG|un1Yx7xeoYt#Z~7sH+iNJlG`| zzVf(WyuNQ0g&MVUaH4i4;~KH}Zwfqo&GhbUa>StV{kSif zd1>~5Fj{5#u^jBZ$)~tMS6_KY*NPkO4)ZFw9Cpf)eYQJ(xb_T>mx>!Ejtz~$34=Z5D46r>w9{$rJ{j(WPl ziClQi3xBX3@6`<~pTzCeshQH(6JdRpelboDjpQ+fVhg-K7LbM=`xR9*ULybG*c+@`vO%G8J=VBkvAm534_E}8S$AwX1i5` z4LEZc8ac($&_n!rHAEGJK5gg1*RXcoz#{Kzp=!Bp+8YDs$mYZ#iwoI4-HPNdwE6>e zfC|isba=|v)&V5%h`6K7R@Z$SyCw}CyGjt-dzic1uTU`Yj3B#gI0b-3GFQk%PMA6TTlmKXI3&cBO@zzV^%42O_&6k0#k^s6xT+PkuEz zp{e+R&@QZppLWn0xh2YeEn(SEsQcP``a#j2`~7Cfz3v&y7fz=M4e9XQSZ<@Yr?__& zkPfr>%ok!H>+EPP7bQ-6v;Dd8r?)p7I$6iGm$tD9`dLuMNMiSOR$IIc83jB&e-7R~ zp*R_SDyU^G?9kfUwHIps4cdzrYs*n`cn^!ulSKAqGJIc!lkkEb%tLIzgly;F<&K4b zjLhPJ;RvEQxUp#Z0tU-v7R)1m^7D&f=wPwk`Fw{$TE$q?QLz5Dj6?nyJmMb# z#;-10<$z7qx{jv&7v%jD)BFPZ0lon6*0IJlg85_gzsJN71<+^fVbK8jgRBo=`L7Sk z7XVZYXf;oHPC*0E*8ogK;re3r_hBf(Kc9YDUz38K3=c~udHb+i-N7Jj9}{{Mu9(5t zydbm+FGCRx%;nefe=R01O_f!cuJTLDEBwO!QP{)_Zc3w%TcW^NM-;LjlHo5+$qY$C z%K$MGKvSuPfmP&!HTtCV%7t7l3!5H-495M*Elf9~>dz zEik%Vj;+)`{o;2C0~zoeck6f~(ntMaJa}80?0Vd9Pyb5-1i}R@z@u)KSO?!d>W>l$ z=ubGckL6KZ(!>DZ*o#*4w6I70$znWs=-Wio-+}QjkpKI`Qwt~-9R^Mdkv!@zh4KLx z`*Myyrmr-j5?U=`Rr!lN>W^RUq4`ZLho9E`{h9@K}**kks#)O9xU!@z5 z7ik>z^W_Ifm9ZF=0u{G{`R^(Z8-2^#`oLm~!S$Dd`EQ-Cj_>9$W zw59RMC*kY4mb*J2NH4WoOhf0qn;S7cn^>b$Gy9g(?Hk?UgTm+Xwt9zg&trjz3tSUW$)xUsR#m>Y;$3-pxf-Vu4$uNwOc`Y0$M{&%|M=-Q#9b$c-0vDj?&`!ugOHj%G=l^fgwM(QsII)(LmxyTk_;=X)ms?|GKq%NF$w${9OKNWp=^r)2m+9@sRUEah6Vw*l z0)(ASKJC!i#>FesOzGVzMvEOa`E(Z+$JiOx7dIzm>UJNo#3Upt#s`kp6F`*-ptIOL zg>0#KDlmmiBomi?g!}DP&GB(rDhR0Fr-XO%CN9pv z<9}CnIjd6#3Xow$jf4*5Krzi>e~vDF?iGocSbK)<=QEqN*hEHkanX1q&P;c_$#OEK zA0$5{3CW~?wEW0u!8X@hozGt#NF|Yfpr1MK{9}gz=IZ;fQN50+7V$#d_!3mHJtmg! z_jH$q_x9u%y{Mu!h+9lrI(%5hk_5o)nK!k@WeFf9ip!PZ{*?#?=`f%aK)u=(S+;II z{es5z?5EiE*)HPb~;{w3UQhA_J@!+GmxD7;A>h~pB>NO(Hm75K|Evl}GuHREj zZxexD(eJZHeCGUsL*ge@f}b~WQ9=x55#wcK8>KX#U*pFL7;JS|5ax8Q@$uF{~B z*^_vkNJ>uL<-DFmVi@agWOHQ?)aW}3LU=33w+C-+a#fhxa>s6gA_D7Vxv_-bZE-`A zZdR+=0}HR)eq57IZc`wj;0gmD-=JRuAln_?o*$d3aXRb}Mmy9_Xk?ekWlNho>r5y( zIR38lp?8y#zN%;^>5U_|H*KF7BtJY9&D3c=15p*_~tI#POS@-Wf%$M5gMhoQY zqC)TIX$b>FD3&6=Jr2redl9*5&PifE_*(8P?FO6q()HELe2$$}^S9cf+1ydQZ-tRJ za!{lm_!z}k65H_ri*K^lF}XJ$D2jnj>w<%yRqq_ptZWyrcU_(0u7-)ici8raM1+$` zbT#{Y&bVLp0a6W70A(+`8<|9_k1^#V&FMvEb7YsnR#&s&XR{({i7f|XltIFF0uS0K z)8a-sgWjtSeA}zh?YQkGawBB7wy?+_6|R%{#_nR z7%=9S&ihc3CYi)6*KoI`5X*7P!&Rdnj3{g5va$%D36yadwHLGjM&YqdA@<)0RNjuBgfU2yS<1Ym;B(^>)pWju zoh)}Qd-{x{z0@VqTqJt~0rzykAq0;{Jgr-(yDyQe_e;pmb@>Flam{%Wm;R57LB{Qt z&7rZNjn0ItS@(FJtV?8%B*W>1YDq;!F+>3Kk|+nqhLKUF_4flAp#)=5pEirK+@Ca<4U zkm3F4!#OpU>O;_PWwDmXVcQT%r7#zLdF&BRQ@lZh!sFCPmv*gcSPv(aVXzB#l!*N% z{rZX&hr)hqq}d`V89zp))}hdrvU@y|=Du`SflVqp-dI9`ype7+?>#u-%5GABcE>0t zzc=T_?tXi+n?W{#enZDYHPFFaCK|K*g#^I3luTS@!U~IfQ*>%LW;sK8c!;>>^yL%w z{ik%%Ua$k}dyK7Nl&mL*v2T z)<`!--uTi(_5v=kaDpB4hhQDyUbjsypdB zh!qvXWKW{Nc_Qz_wT?1cpC+%&Qh~s(I7yAT>5~h*a8|t1(v6- ze(ea8#%!?C;FHi=Ml-60x)PQ_{bd&K5^d;K#4zkfmiLUqYIm%|zLl2qi%FVkreMi8 zrp4TROU4}MGiCY<3#p~9{rKx6v|9ENUFjj*dfS8ca?GbSTjI6%0Fi*asYJ_<`;9uj zdwFB*+LCI)`|cS-{LBJ{fCPKBsS2(V2Gu^XKCm}jjOReeF;<7iook5Q{-lJw!tf4p zl`4HcZ#}>{9y(p>ChMe~%2V(+lg$>I%NjO1$OE#t#-nw)%qFrjamiC3q|L`Vc=fY0 zJM*y>W{v6p%yRxdf09ZGM`)^pwn%r zgwXrtgL7qOopndl9KuvUZ~Hy(Lf)yXCvcw} z=vIEAm(CBNb_j=4GZ`(Qa-2W*Omt_m>);0%;1#y)-QxfSlN6f3LsHXYS?Y3tmN_3X z)NwvmyT89kMm4Nb6q>HyvaWS3UjbB}R7Z|?p-^K-bqyZpWdApPpAz9u5~94>Y;G>q z*yfXe-3CMhioVW(>80H1@&Ay*f)rP{MRWIMbDtldEYCT$~nUmr} zK1Hv=M4AurdDuWBpY&Lfc>RMzG3OLnEvM}kSTit1%pq^(pA3F}LWI{#q^NtupaXx; zU{)ae9Ha#YWmo=18hNNyQ|v#%O*OI;*U~2zX`8AHp-=?H;=M)zcu;uo-GAQ#aieKc z#G7}K$9)ur!Gr=~7(`919qgm%!;Jb_jTjob0_ow0fQ-ycOdDjRGn=cJg$}fjXxP}~a$(Qc-n9JWL{>_e z`ZECY^2S`aHf_F==YoQLfOv$8xn;62!DN{o&4T_qI>_>Qik~8A)P^jux>}JpiE-e? z#iheO!6;khnAv4Sd_kbe$hXD*8_#PY9o&va$Bp>%FCreSZ~rJ;6X(Skcn9qE#FD52 z(BwFh0}$dVWV_%qsp}ljD6i@KKU922)^x7?kOh4KDFXxrNjPTLUp=M!2UFe}GYye` z2tRv&T>zr^`=z6cXoz{O+imwjRTyrd8sTK}ZI|s=l~3^iMk)a}9W>Se{yTGWBjLGU zd}rTiL#K-YweZa9#x}C35>=|II;)*%TK-o8u(5%0pxDMm{w4@ld$h>xRPd1ZMYb!mM2Fg zc>g7iRVknWX{DeeAo_PNm2rSX|J=c`XP(kBZ9}0}JdCgTFF}AFwZDSFnCaY50`v>! zRxMyE+mx2$m@*C7Q_j=lfPcxcA`C=~&T`AIV4#Eu$;gDC;c?Qgc1IuQiB7_@t-0TW zY42<^&88cM#}oj{PPzH4I(^N8nPWW;_u%nqJ7NYqJaV39w)s@90f|jg-BQp;)h?5y<_A zOCxAP-qJf6pPMp4*cr`Bi`3tF_o_84F<{?~9G5yDE9I(``8{#DdCrlp5&vWr*>!!Q zM7MO*Z9c$1_hV!Gi?h>ee5hvnCdEYUdj`TUSB*e*x*~kF}tElot+CE?^9tWYjp1shdoP8(EofH4V4}0il@lUF5;)+sm0b|0|-7N z3%gowEQoguLg>SX7+l@aEta%)7VWgl`l2l&Fu4qPwfjE2tbc2RUiky=uwhTQWy7WP z9LO*z!A2aUQe#&=<9eSi-{`JUeN!rdl#i_r@0Eh3{Znk^cMHlK4IjhJ`^x=^=dTd(>t{;!MeJ+wYC*MJ{kARGm`faq zBIC@5tjylx!hu6Vx=~#;)uv%_R6a&V~Mf7euk4na%5&yWdY)>nVo;|rM>tu z-bqP;>*1Hk9L6DwYpVK#*g32YLQnQ=KihF{6IWl)eeOLM80R!stFi!#JjH$I^nH83 zJ2f?RD8pKC@xOvqa@5|UM~>wQKKBNHQN4xly~=9dM^A1~+@EldCwF9mS! ztmw5;?ngu6Py(F7Iz@3UQ$eo5967=1=m~S-V}TD5p#Zs&krp8zGAy7?6ic9+o}(Fk z{d*Z|L0R&!sae5x_m`;L4~st=;UcMaam>@&k5w%{U3!D7zp4`a8H;!e@{1>8?s#e1TYu_mcM0z92@>4hT@UVh zH(z(UJN?~y?~hln-hEZSs#Dai;_S0!jkV?+bB;Neho{0w=Q|*}Ku4M1y%cn!c^L7* zu*4EH#no!9_NRmsuzWhNYYgfYb?K2Bp9=79W+F1jbXgdfb`~yk3b4&H!;pSPtN!W* zaAP%_T@I*}MHW-fQlH{9XlvpNoU5`II5)vgoaf`Y zCnrDi9E^sY!H5jYD>{R2x5J%X2sV2rlkKI$jW`-$Z zG_#x0}fRNHkUwS;D>&C5(rD8`?o2<&X-MnwV&S-xFg& z(3c}Lah7MD0(fGc43DrY^j=F1_vy-|y9i&YWLjFeD(!Trx%2nTz`&T zpg6cJIel(*oU3u@F8QHYD5{ZGrIAMs21)`7;>i&=V#np7UgJa$V{Ed5*&KYyWm6NZ zj)ViqYM1O&0DIRSJ9S!(&$|R~4H&!&|375(PdS7NlrLdn>`U-3O&h&?22&c}FLzuI zM#`}2nx^evsujCtxIwP}2t$f=-Zba(y3yWhQ8j0tm{0SkXt{gX71f7=Ab@*6`S{xNW7-{`kbF zo9!d`^Q*Rk*O8g8kPpjnfelTn`N2#6Kt7ylH}_B={y;3nBbI39hS;@>8}1EZ*YW|l z`BbpsUo8YEEc}{0eY~n3k-4k69OjZ(Es1R8T$vr5%NOVi4of>RKYYT#1?EKOrs{CG zKl0B$*sw7qojHDv_OGVcBGB^;%RX)43Qr>q#t%_XzfztkUpIHT(_Kx%1mo&&)z9rNggFGrR% zi@<6^G74GZmd0YG^rqF2eM=GhsnhfyYv`{_HvyMjEk+0AwaXv~^5GTfo*wNs&QwnR zTCXi0y{V)*C(h8{yhNj?T%^VaFAy?^IdFpADEX%STfnXDGv@raTl;oF>p!xFU*7Vs z-!THxIFWtic|1^ zPbmDp{ej%Kn`$KB9+cX}y`c&#Z=X1w8)aas4?P5YYWG2cS|U z0ORrt5Zj1%L+oIctCE{#2WS0X@*UUL<#{l3-lXzGMAE%+w3L7h6xBx5kOZIG!k^r) z+MtMzDEIW8lUd$FjDLEd-aWab;V-E8y#U_JKwCQ!r1z5s*2WqGaX?Z43VviI0#%2r zFzXLF_#ZSyz6p3KnoS3=#e!|P5M6g3S;GIId%wJ7NnlP1y3Osd{Khh#c#iAS6>v!b zQ2!dYKaT|DpGZ&doNV5&+xiQ}z=a^m7bz3~nd;y4?LSBUioiI^FFJGmj@qZ6TzpC` ziUu%NymfCR8jG{rCL z^Y<470d$g3*l?%(S79B54Uk23{huNFKVXQE@uQn_$Gqvw?)4*=F8z@HF#(WiAX$4w zK~icvrq7-O(WhrNR62jJ62I2-r6)j#5Lf@>#6RYcz_X{B)0QL|_g`>odjhBDnmFy> zVEOY1*%$Z^3`3HC!j5HC?fnfqJL5CzZc10E57H`yk-v_|L?L-2%zlqe;&#A z|Jf+ztdfUEM^XWm4G}*ADLYH;_}-6n)w#O4UQk`1FWvJMP23isMk*Tp+_E&DB`9A^ z6PEaGTnSq(&#lS%s~&8@e}o(0qH>{ta)tjgx4KY1KBwZ5Ep1en>?Qf}{a({!K3VMpkTj?L00u1GFjYPN+us)>*HHYv9j@)AX+d zRtqX%VW=f`k1c@`iAtkBhy*(iOYR8~?y<+L38oAeghsX=pz_OjygF~T?K3;xLIr($ zYMzp4YC}`G*|2cb!;h9n5ybUVjK*~rwCtHG_V&=bVZ$iz87jTUX0grEIcn2!TJl$H z`7|ymT4LXw;66);MJaA{9*Jes*$Z;FPNMFE2opoEQ>{jn$;hjFYSK| z6Qe7#IXyX%n!AjwPpuKVwe!vjV#?MM$~UO7Nomr?w4OID^tv0HIQc%0$g~lw>byHB zU)l8&&vc>&BA9*CDpD@?uxhKh(yY+aFc-C&A?vN^*?S@dl#EPbPe-HBP2TaG7VTs{ zdlBk~ek>V=G3zQk)_(E$)Bf|c?Md-ok&k_a{CCt!-g73U{O6Ze6>U}JK2tEx?!zi; zvEtlom&x^?$aK^;LkJ#Y+$=A8gQkT>tIdrqC$o%IJa`VoBemI&<#yCtn^@Sv!{(X| z_#v_`qt2;{2_w{)LH{iE2jxHe`U7SsByX-8I`!JU`L2E=U3$-h$IO0RJ?Q;{kFsO} zC7S}uy9$H3_BC?X^6K8A$BcZBuLTzbH>r7MgzKk`vptz&ToMy!)q=X$4|>>FJ2<#a zBR7Hvb9nP~Ggrg0xG!QEsxBx+YR zC{i+qIC&w;`@S2b=N_u_er{|CBq+;HtmLQ@O)Vq;R&1anI*jGKagsG36OYopnPIF{ zh)g`gFKOX(=!~T`zw7wHr3aukW(Q9^3xuJ!PN~o?@$kssXB9-yK@d5$W_C-GX_FYs z>vQSx$ty;;`i!&78~@PlT<=z4xvSKvh=Qs#$y!V?8*D$a7ssSj7^yV6fBwMgo#?2r z_!w_#`D|->#0V|8z$k#-MOKfpGOl6LO@ht1YJfTJecb2ZatAq<^wYDc68-4n;D5g3 zCwDIR55E!555<9P+NWxWK`;t+B_k6dboa}1om<0Y(!8Gu^mLJtWIlHx1El>g?rbMv z?FNSq&P&0c_cO>;q)au018`*Z4U^!9x2y2*CXQ1yl7veMY#`Zn#O~>$d=|*2l68d~ z!AN-Z8OFH_le3}PBb1C>k|I-VBg+Ayv1Tm$VvSur9k^OXZaPE#V*O(_y-CkLn7U5~ z-)tL+>fVL7#Dk+us)Cu%MM`rZ0(r$kdn$Poe>yC^FSAU(D|O(##36r*-@4c+49KdD zQc5>3X;_GrpD{p0j7+iDdbych@Ut;gIQG5CS9;S6(vP&V6~9#<0*Rb8RqARQiKT7B zMI>>LR}ni|GdP_yIB{Dhn-O*~dzIVQ_G>jxg~pDcFyhRQh*)vAUKAp7ottS4=xB@O zdO93pPbxr}T8Z9~IP}924otCHi2u`7Ku!j{1pm{PF-zj3@ADxQv(i1oKJsGm-@fq0 zh&~Dy?FP3gg6UkthuZ}o<#dww*J6!IgF~!;t&(Uhj1o2ZI95hY~*vvA!=cZ_Z zrgk1{I^g<|Y0FW8*b6rV)v>AwZUm__LnW02gcG0asq=?ILTws-}CQZy-$O zWa4dUmKw{B85Sa+`DojWh{<{woo`{5uH!EHZ|;u262LiZ7cbKf&x1SXz&LmWiz8hn zZz{}w@G;A*f9v0#6SZa>6Lci`@L*Uk>i+Cq*fs23wGDj)AGq1018mN!9oDU5i654v zr5VLnoafvbi5)aqvOGI=&c+hLzE`}R^&Yj#VXHPZmLOh02a$w38QR3YSAgk3+{N7&(?076?l69dYEL?Dpq6@oIU) z>*SnDxfV$ZG^zP5*qxc?3kR-fRd&!K$zoL88wH7!;)!P?y?+!$|LHTp>H4ib+QaKY zx#NF;q_q(3vp5YqAx(NYz1STcR-q(^nE)-9m$cl|Ca2o?*j^+UrHd9TUaM3%DQ^Uow75)K67`zCYz?zydn zjgz_7+W~iM(v8!`xgOzD^j9gOOjAXys22EsXr_~=b?)EOOnPOFM|?Uwl5kWhjq@gH z*s|EVXr`lAs`#bLL<8`+5QXzPl2w1O@)})?rmRZGjO;L0_8Sx>IzI1_t(-0>aXA6+ z@;Y+`(jncpgRS=MbSj0Sn9YPLAKWJ1)bsb zM-6!(#?5T?SEa!plX;y4$T%fCa$pWaa{0g0J{;>GPu^ha2`(6hRyBRhD}FDlC$;7} zoEOk0_pEEw`2Dpf3NUh>31&Wn&mF}OL&_*RzEQ09Hj=Pk6!m6`mLdo*#(%xpt zCOR&l$q=_LQ0xsv$HCHnv$;IIrRHLqPc%Cc&Li7NBq6}X)miFfu4b}qGP`m7=vE-V zPF3GYgHYxIX>1VR54T17lqfk zN~2uxwJDTAE@x-9ALD2^0{g|rM-{l}wP(hgQ-fe+&vw{mH|_XsBD4l5u+#=tbmYb3|t`|Ud19~&GUdQ2fz{ZLFn`9A)+I)B&$R8UUx#}J%YGDzSot>SShAtr~ zS;6nL=UR(pBELJbLxeW%&9FA6e=>)hk~(IyD~Zcvu3jCDeO=xWeDRIq$LaAn&Jf!U zAdjJ7>$=O*$xAnibHr2hnhoKLoK6UetS_3($s%z)9xd`m)_jOhFhH}0q??(Ds2|+F zsOG&f!bKe2j`FUMlGyYTg5qhB#c88@w>z*i>n>wck`^6F1}p8wd41b(KmNCtc^UPa z!sI!L5Dzki>|ed600~_b!#%t5x4k7^pr{;@JQP&wO}f0-7U=A~ILlFQIjiT~KFo}( ze55hlQL1lF^ACm$O$ewZp2_DkEkQ2A$i^2$*;r7)p+lt9FfT|KYgpnx7Agt-W?P>C zhs?3k3IXB)ox#)H2>qM#3qX6eQb3ga(;oi`QeWUi_;6ea_WvgQmrsP>Ml$5T&O#=9 zBK+&3qyMQ3{VgJO)ci#FlN5>n@i>2+5BlBn!VH7k@L+rKhU-W=huI=@M`)f&Y^p35yVtI+ixdSnAFG&k2<$wBBdj82aSNj5?IH`-W# zj^}ex!Y2KK992+IZJU#t_8a(Pcn!7+g{ZERQCyp!mkzBlKqE)V!m&9c4Z30G%fmVH zuRf1DgIt%|lo}OAsSQc`cZ;6qtof%Fy!rCwZ&P=dv(cb)n=ezNDNrcETvc%aw=dP# zat9J69388qUo)$^FF8hy=WAVdpdm1HCq3$(%(2;&DER8|>=gLumI-Wn9_PHCZq?KE zI3}$>S}n2H{EB(tes;eR5~mjdeY!cr<8@fS&N7uB=2Dm=eJy~w>`F}*~RLYEahyb zWbyHb^)mZnt2?U*vUJ^4V(E1(F8gBW>Dd>Y3mw6e1C@zrHKs$!l%!di*<%`}#vHka ztT^AiX0$8cWr<>k?hJ|%cv2h&6VU$<Im9)n#H4x7R$yROHS5#Orm=UuZP1 zWg`(+qI0TZ$2^gChSqk-`b#Zw$yJX$FA6OV4m&ccVD<7|hGoU@}`18Fy z-WT0>>&p62UPP6Ew3*X@@B$~Y|G@^$&SE_indLz&;;C(u&1200CXQ9k>p5-WS{C!~AdPy1k4h*@ofgZ}`f?<4Arl}60{denzbDv-k=gf(N{0|)DlG!?n-HL? zvjLsa=g0;z`P9<@TE6iwfp!)1rRL`OOy+l>YwH7Tt<4i^_g-UO_qwn>J>@O$^BABp zsqU=HK1H)l3umAE*$0oCd$9%`?}WR%HP7q5JE6^hA--y#o)o?f-fQy*&yhDLU~VTC zAG>B5B4T1NtK;m@qh&>g`AH>R5WWyaLQiZuMT3udNTaLq$?3 z7)-PTDduacXtY|VcwU0dyu9+9!KAJ3O7pow9t)y2b4gTA7Wrf8eiL-2-0kmH3kF>G zkY0W4K_l-W+vv65+mxYH&i=w;HZLAYB2!R!*1 zi^GmguD2t8?8-Q+-edB0@0IQ`tr#5LmZBM-c7;UJxG_;` zl32r5Ms;Ejdl^vJ;<~yO)LeMmsS#zp&_cRR?P~lpUZT8!hVwoZkh9FEwO>UQmfPihTyp7FWbZ^F zbOn#C3;w0({98?sUcx88=z&4C?Sbs3j~&G&gR}7#eL2zr zJj-e&qL3wPWd-)*$r*1B_5Pko_AXOcyaKq@#sV+In9Q6y*&nCrwj-Gp#$#k&@8<(8 z#`CjU*~Mud2Dvpy9Ck7-Yf0L)`c-?$)tila zqctRBX$>`+mOW0k3MOVNj0gJ@>`yllDOGBUJd0AxZPj)s^DF7R`+r6-iSTxJ-?OwU zlUo9dj(MwGEVs*n8O(rYRT+&-k@Wl<2voxVeBjtYW_GC7pMnq(5h3HzwUt}$(yHEP z=n%9J@QOC;@MI@d*xkF|i?TM}K)gJ4mk#!D0+U@Ha=SoqWv$BIF_;t?^`y#hIqrV* z5f-9k_BnX%_Z9~2%Z5~3ulhOh>yuKdydn05jgIoQf``~yDieXj#aZcx_8*_QUV1kz zHDGf%xAm-%gBZ~W4`#pAmMl!n8R+rr?aWr@*%MRbEy36u23rzZN;&b01e zdp~Bn6;4L7>2O_ZQC1&q@&qK#*H}-`5V@o=c%iVmri&Ja5wAl51oSu{7-Uayl19oqbSMMmq>VdbDDr|J1d|Ui2 zUM!sLBSQ(nmves)niNIxcQo#v_th#cO;^X+jTQZEm(Z&_JO_yLS}zHB#$!<^9QwbY zOg85rlzX+|7C6c6{S3O{dPN@gIib+;y}i9}>e;{%7I-!NNKEaxOFbB@<%|4 zCi}WMFtsEyeyni1$+gsY*i0`oak&JC&Rxdgzxu7J!KBCkV}6O~V4&;Fa9MB1cO{U2|$j>QOeU^Z0vr}@9B zXk8pE7B3ZXSsKUePUP&6L-3bnV6}(s(YHSIy03*wFh-LOjDt3oTkhRg{cGc0PZ0vK ziF20+YHJ~tyw$AwV|hOV?UhAvs1V}j-A;E~{XZfgkVOZglGhk8>O)4eXfW{#zOe9k z#|dQ|#SQ-)?UDR}ZtlfL3^Ix|uj6_Qz2)=y`fQD?E(_@kv#(GtV1_LQJMf(hyQP0n z?Z{y?Reo3Y$3aD9jT3?PXSESux9H6_x;`wCnT!7PWe1;b^^XL7QzDn!wd{?dFC|ZP z-P3EaC(S838QZyd*>o)C`CQuH&PE^gg=Hphql?7s*-n*xyE@8DDMD{cV~ak~cBtOu z{)|qAi!;#f>=ljElWFDS<2#A9zpYEhqSI;;Ua~njP*bOlzardB71lG;mJ0rch;&u#Vg@v{KnT;MtWZTeAZP8IYtN)xW+mMBDu3tg$p zUbo+m=3BnnYg{e4li+{Mj8R{3Kd-6~Ecf3cjSdv$`i4ajI!V3xy5@@R6@~I}D!Lg# z&~ooYL63jEL$v`+5E;c7vb5Xsmz9dyTzNkyrMoNm>`KLw@BW1!DqQ_?o~JrIQs1X_ z4yo>Cti%cA~%w~#!X(-O0oMR34&r#<`TqIm8GaU%l?7UaXgS=)DP*qwTdVl}u1TZ6(3 z009M!_viWE)ETCYo?HPxg>~>Ki2MYDCVj}`)Q`O>6<97S%W~CA`>zqu=}0<*Ub);{ zV4XwfHOs@eMy4~gcg(0Hji|wqRf#gdcP5%L|FBo$}Z$uy;p&%2(QJY{5IOLmxx3v|-Nwx!eXqPW6f3T8WVvNX z{?LMt?}TDD)YNnqADTU?ZgF=t^NW1bDLxb?MHq^O_5xLgj?Yx&w(MkTUe6SJKs|p6 ztV@Bb(#t2%?5`=>mib&nHhxTk$U5ENijo-2*&%a8{FR5aY&$|;O(~kiH_2BB%44R= zAxE3OqChw74l0h)4`enN*1t8sK8iAlUBx87d5KNL9=(T%PFyzmwm*;kD=gCYAXn<< zMz1Ukh%YZUB#u%9nxad1J+=>tPlR?p##4r&`%vhuc7~PefIcH)QP@RGMcc94)8H<_ zs{7PMjUL9v0Dz-ublD%bwgbu-3(^)iQ#BgnEyMElYvpm~uGg@?UVKk{;m=s3HQr{> zm4xwee+c}3w-Ipz|077;_dg%PJJj@5lDkkmt%v7uS2r*HwDbqGLi)R71XuK zr%Y4v61<@Ukwqr+GWeAr=U_U#Ukdd2Nf4HFW?SxutwCF}yRPxp+3@q)y}kicSQ?`! zVUpRDUHPFduYUZ>#fN|{w-kn*wcx~%F=(t6(W$)!o>bCnTG@!D$XfsTEo!;jz5xNX>5`hu*8S^L*e?#N(!&IGb0ku! z+Do`~FK>|WNdj&zVmc8px9h0L22tlr`l*;H_L3DJ2-(82+Gux1D_w~Uu{+jm8Cu3s5uGcL+>G%iPA)i zP)*aWMshqqaO>XsSkFIj*-4SzUwyF^AmK1?&`I1Ve2#eK^CPwE@8#e~r0u%2!M|q{NHwzWepvO9WigCpYNH+zdZ>sjuXOI^ocnCrpf%ryi*7K~@_c zNMvuQ7EbC{M_<9OT6UtL+m#(K_C0HTprN30B~VtCxHzQl>X8oTO{twaG$*jSy7@}) zw5-)wt=J!59!;(4tAZU(h1}ufM~VwtLn_3gka!J^$;zx8i~+)BwdVYy5pX*u4Mbu~ zlqR{!d|8XQ*a`&lP0Y2&uf21ELPZMeXkV^ZV$wP#BQ^HH?+V`rrym^jLPMWd9bV7az3waYHozbTIelzH z>{e()Vw65gVSBD!1Oxi$Dj%!QE-BW1;wBAABJiINvK<^~T1qVWp^Tj5s246;rCm8M zzwHwoC}{cX=|nF!!K4kL7?QhbWQaPEYvB!e_?>&ji_6y?LRae7_dlBW@`8r^B$#8- zK8Eo~Z;2IRsO(27G)suuN_UvIodXMLMH;4ym9X}QE(_>a9ryyn%5_)^%`R$MZRyCT zqOVJY<(FOObV0Ym9wT%`Y$)i*PtXv`e?~fPhEaxQQ$O-4BM1)*=Mqn@DY+U$&;bP* zwVgEM>Ix_TN~|occwjVS5!Z-pn}Bxue*df+xhQsAptD;2b}P+)!i9k{Aln_YQj}nt zPoEK@6W%}YZ31#Sh@wOXRzKtXhS{1?A_P zO@r$ju;|zm$~*`iD(&{w@?-{I+1};g!O`I1uv!(wy%g07rK&|y_Y{POuH~fojQqiM zbKIQEhqHPWas75A3%U?@#k<2v-MkOZKf<7sP8+J06hzh4tbDB@pcyq}hmfOk6O3(? z?C!1asc-IY`W)R^eFr8J>P_L56h*D}#}lqiBpR=y&(9`8!JYRf=dbj(Eg6x>=xgMW zg;t=kryIjUAN@=zW}9V;xh`UHs7?J_Ry_2hUk{$dvvi>O+v6JgE*CSV)-xrXP7@6x z7U`0CJ~pdru4sgEa9^dm5}UzB)7Y{Ge8v0QT&$v|cS>~NXd=;Zd1SA07r&h|xNrgN_%w3k1|56+DPs2noTr8hZ7BXn zVXCsTa|luxXhxWYuV#!3G$mfBGH12i%8DZ4Wnu_(gmGoSy=@*e%G1o98JYO0!Iv$V zuOLyZ3S-QK&g;w=^ooZ7S#N{AdK3+P=^MeCYO^zK%EEf3-rm|#uz*Jk0kbAol{N9{ z1eIh-$5uj8Ug4=U`CpNNIZ)<&^W_oc;)B55eN0-tW;JB@)ySvTvmM2cVEi(th{(2U zEr?^m?{Bx>ZHD-k!!2&_f39J->X_54bHzgQ?6zFC_kR|y?p!SxlF0Qe*G^DCH`_21 zOtVcf=wU&ujk#=LK(<%c1vmT2duGVZf?oP&1=Fy#EBeW8z)PJC-qT5#^H|*|3eo@K zjhz!~h>!MdOe4iLcTuSj-1Ma|#L$c0em|dHe=9VN{37)7Yh{f;>4CYaq?Ftx_3Wk$ z-r%kX&Tbb+f-7d|HBX95|KS1FrRFIaL2vdBL@e>(3DL_3bu;qNnFMCvO z?ONaXQlgg&dXcSY*1^7=K^?c$sh(OSH@&r`Y;d_u)wP(Lam#*F^StOd{XL_kl$u*W zR48LuajKsnn0fbY>+zNak3UePDfG0|qeKYwy85jRi*)c+$gPC6<&SMukuntdF}eie zOo-h-JpUqRlGZm2IDa9uTCcm{mOs~&o55L+9R|P0dsFj1$em;_%mGh*vQXIK1S(+EZr-QEcgzp{NVV_GHFA_aVE%yF+F^w&C zjf0E)E|*{OZYSor_RnW77bZeHQwQhzPO8E@22w^rQ^aAF9-|4PplcMksgz~!)9CJr z^E%9SlK|q9*}Z%k3QrwMuYqdV%fm6uH#=JJpydoIIS_9@Tk-;DY;$MhyH3wm> zt>1_X@&lZqT9_`zH|zkR44)ewLKMrNxoPpejGOQ7R;%I zuIO(dCPBuB1I7wpG|=jW63G1=pXB2b1u4+;>~gx~hghh@(u2@>$CBYDL+rba0{)l^ z1Ltp5yCYfMquJ(C*WP#Z_N(iZ^-g;=FYx$8V@tKssKAUdblO$Q51bx@uX`(usf~xj z73?n$c&S6U{{=IYufT$XV#!17WfO|eeZU%^lsD*_GG)}{a+pdZA7%P8D8?tz$XY1A ztw$+uV>G9a$==n+e6FFsBaA?9vPhd^fY0LfcUCvOTB5f9+f_iZ}O*>4Y7F~qs5SZDpkrFa-I%pgt>vSAMW;S zB%&GoJSR7mHvB7xy(JGzm&!CWYaX;0DOD7>JV!)i9!k~S^z)hCcc|}Saie)Bl7RUi zHSv zn@pzG%NMkocb`3hy@PH%YY)8A_HGZAOvLX$5$v4`eGZ8QO4D~5biC4=y;M5lSEp9vX zs>O?CdzEHYp_IHO-**l8uW9tkHA809xbO&U>30`g$_~(a@pw&|+T6R~Mj7bRst@0X zpv_uRm2_$6~z0vlYM8{Y)i(=8Yx zQ$v+7{~XI!;FU-sa|+*nvnZMqbSSl>XoIk-wLJ8*e2x6<1Bd(B_UJ|-QY4SYsDNbh zl-^JV*>~oRm8@qUL96~f_ZI^%2cI01JSe^oURoW^Ca_&1!dD7#J0TB z(aFrdw)*(4!Uwt>%vVQ?yu{}l-&%Q4D*x_r@-7O-x|{Ib_Yb2Fm9Y)`1uDH#w>7k9 z4`6PW4s|*~i7~;Dl|dbkY?(c|Gps@R7qCe98KY}rd9C!>AKE_)PGOw(p2BW!qQ0A$hJP>Vaa+~0lVY-m;BUU z13!;8YndJ&2WC>gu&u@-{2E6Sx0&WiLBM`bD2~YKvy%CKra@@7>PA>GSNe_H>G~)m zT{@31jYh>=#XMO`s2is#E7#ihl=bf5#3J2F*))2MiVrWaShRr#Or$pGv?QOyUQZlQ z823dht3p4F7Iseonsp6OfaLrpQR&4@s)|IqS5S zBj4`X>t&W85XeyGBNvE5+`yN{AP8CS&Ps^X*CC;lBYNID+MvR$qZ&a3y}oc+uvddl z|Bx%4#3$Cws^(+Yd6 zSxc-qTHw>&wY?OE3jU4mT`~I^YXVbloV89X&?G+-4xM*{dNOh#skY~Fk&+G?VSUVP z(X;XkiA<#W;T(S~1+-#=qj|PonoY;llv;1Jw-}HoF7BR89UAj4#n*$MMe=hBYh}6Aqu8!rRvRWwf zn$syWw5le?e`h>q4MMBC5-@+}LO>xqS+YPlQ)xW1LFfatIXAjKVYNqXmybbxcAc)x z;J=r_!WgNhVUo=6BS?U+jEQB_qM%MK)9;K z%GgyBwUT5_bN^HM&~zW?DYEQF)N#FRTft%$Mz=O}$?|mZnj(#&eJmk4Lw#fC3J3Lb z+Jp)wCP_4-_WNfI@Lq7gdI4}cpM6*#N)tOY><=*-!Y}i=J~0;53c{q!rgcM(uZ>s4 ziJ`i7PG@AFvXlDVFdyv#hWTVyy)su#JpEn3EGn%g3z+Kku=Q~_vAl|o>{6vr8O~m6 zS=0|hm+%XeLEL~=4P}l9N=~*Lp?B|1J6%hQ0{HXEgNJdnR4>Ez$tjy=ppuS;pMdpZ z6RFc4>%?u2WHT}LXx`XuxncLDmS;yW#@K9ksNDD?4;VQN6#HthgF;F_1IUHjQzK{# zE|VFpe&f(R+}bnUs6M>9z1gYFF*6`XMttC>x?m&Nl8fhkI#xE%VyzM zg$0gB){+JlGwvGO3oH*?pl%unxujq5L@q@ynBOcF&s*(7 zzIxe#q4BLq4fzlf6FMA^+r`o)PRL3)@iu77&6POl7waS(c_iup(MA~I+2 zVxEqFIE1x4Tx=4t7#l_@NmdJt>YFY^m_GQd)&c(OumuNb*f$W*(*j#%#-yO~BvK^1 zpC>Rnl&HLYf#CoR7#{veVh`13o7K{7C~|_u(Dp2`mnqt`k*s@`_7|T|?eDt~{gTi@ zu>#0Hgb9H8ggi4BgtofXb`BS-;Izv=arp{^oGzU);kOB`3*pyR%B?5(vB)2!OA0|D ziAf(%Cxtl(S2GF%>omO23C)LaLFhnb^r?<%d_K6);w45ch)m;Lr#Rkr4vps49b>f& z8K6!CC;7bg1zIby-KsE7Uj@G^eC+DI@hUpri%Bij9I2Kp3^a9fpj|F8*EF^2xt~2q zQ9>h5`s$j!IjA2#r>)I`c7u9iQc%mT#o*c=*grzLxdIWg^!sQ#NVC&T%&+>n|5#?2 zSajpib~(TJqu-C}34~SmX!+-v-c)e(s@UHP7y*{`PSNIFd1{}8uEAb@*b*uqcm(yN>?L%`{FgIs=Co~=F}nWxA!Q%Jb|3v*;BaE zulY9OnSW=?mp0UAm_#AC6J8})>dI$ecI8fDjXbCf?}_|oBx!D^oATF{zXRA0~!i`cL$bjk-VH0W8G$pkJ2`x&R8yb7_pf|>p!zIz~0OVeWj*UM}z zfw(&niA$vmkIqJc1{5Z?niZ~nAha>8{*J8L&>kd&+%e^A-$9}4m>~^bnd<`ra!|$#TLZ80h;}!j(@z`S*eO!F$F=2qDt)lHHv*Tzw}F~ zKU2B3op^63USlFq%|VbS^ZHQ6RxpmaB`o z9OKR9;bUR3&HP6BmdkuMmLR9me97yf(|;SvPWWUlFWGKg;X9GL3(;>Nf;3V#~%UB?Qa1;cuKePA}hkdHrfYPudQc&};Wx>faTy6ffs@U(Q02%3tPb0yG$dEyx z9;H4jVp4sqS4BQ!tq-F0zoJZFGCpc~wn_5Ocnt*VmvK;&N-{&2M0ShJv>Pg1D`?qU zsoH)xE*7bk*QD>y0McK}X}~kJ33$ZaqNv|-{U@QC4RrNKY~ZdH~_Kw z|LWwr*LG&|rYS&s7m+vc{|;RpNiPp84Y|b!fOJtIG?SZ1%4lN$w-0!S_LM{pd*ksd z9QGRyk0RlQRkld|jLQW{1jZT(Aq|~>yV93s(Dp1rJ1obYV;CT-F)f-_{O$jL@J~WN zHEBS~p!OnpBI2LHfnUcT2mo&MA+6j%>R*HNhw0O0grJhuWwiY_I{q93?vcg1!5KC~=nV@Tw03J9<>E!^ z<&sak&UVWsM}I=x-8mx(M3ZXqbR(5X99M1WQ!DwIDI#@dmoS4Qu`*cB9y#Tai$ z_}8)wL_wi|w?_1XLTs4tt@(+>X>7%Q*v8N~1E8Q~+L=lt5&Cp?Gj9`>XC?xT)Zhrl zr|n% zQFt4&Wug=#$t1FovOb*RS6u%Iy1!jNWD;M34|*;;CX_M&A0r0ZJdXy2Q7Yy29j}pR z&))+3&6ijKj$xGmYwgt;njfmsN@l11)}T%TqhV~ebi`c@C3YgqgMdc!_*77$03t${ zUuA`CDi0~Zl_GfD?4h%Sb^jAyCOIMk9X*cKSaFk>4(_7@`zsM*I2{eYlR)NA%@&U! z5})EU5yc%$Z6wTlh9a>v!vX^;6;6>qL3sN39e(&@tIW;?~*9_~$aj$$i}j7oKFqgyq8vVTf2lokqvl z3?Z1_7G7==iO7zdyV0B>tzu9E1L!3TB0iZ)!DRTLp37Jcopzl%m2y$RBl2ao^8!pH zmlavQZdRzO?|WcB-~!dsoUt;U#tWN5){6={-KKAS)0L*Be65Z%1UIfDt2sAVlPqb~ zx7tC?=lnXVIaI7hcVYa2t*xyp_%2TXCu~A(tF+lJix^FRhqg0Y{29Ve5?wB8W3oyIg?{(-;|!D+3Gw;QzqeP ztTZ%l9-rUs1lab{O>Wj@``c+oD6o@s53srPg6`J-`h+F>`rSt?}iI z*s8FW3+&>pvfrCKpamo&`6u|3GeG|DXDVcNCm#g>$uA+$+WD1-1`u4R>J`-04sb8g zX)f?_jiXYM&0BemE-L?X`7kJG??h$GmtdDe>`5Q%cO~6KNU%Tk8gh``K z!vDym0E4(sj<`D&bGkKD*otuTGU4d`hqJ5PLSUD1WuR3!J%4Z-pXzFys*&|k9n-KNnYkN*6n0fmBg4^)o&Q$v2f z6jmwBVYoMb?KV`f~cFQi?XMR7{pbBgvdBW~vRp zrqhQ%-&h*!2846(*2tvH{ZArnmVH|QjxSN~x>I4Ozynx&K&dJ8Ox1yatc67tv-jR>c;;{`77pN4AiZ&WN$R3(a*A#H`Y!rV$i6qj@LnL z&yks&=PzfOj2G{qXGZ}^F7%Dt9eCSrmZ{5H#C5YD8wh>!+XI2DYp2!2n;QUa+sm6A zz7?KVUlbaW^~fK{u6+6v=35_2dE8b@G0t|zawA#t6{)16T&Zl`AY(9P(maaho&R>- z7J)=lpi6Q*buda@rg!w)BO&Zq;>G9E^&V&YS%%4NF`er6$_40GAsMwBA01nez;lxR zAoy%Q&1=)a`Rj)N#ok{=)v+yYz-S;4EChE=@L)-BmxSQ%?jg9ldvJm~B)Gdf!QI`0 zt>DfA7A$U)v+X@+-#hO3et!&p^cXZfyXUN$v+Aj*s;+TIQg;PwPF#>xG zDnVIffNvY~b}bSp=^0hjQSiV&SL-4XB3GdYQ##%N^!Bv}=~Dh&tlfWSaV>U#$0$eM z?t~)jIuJt@3sjyQ$PsTxflUIsX=uu%aeiYotmq<)TitAf2(7wg7i^8h06H*J*O=g9Ymj8r;jn=P$l*MGEGLHJqCbYzI9~63`YQ2B3;bI|xrvR~y z7w3edC&LdRp^03Lz@P{cM?PcshPD9BIZ!VgZD52w;=zNe39qcN6M^22aE-h=zwHD4 zzzz(YbL5zX&Rg`p<9K(~EXudA@Nm!yUcQZ3L3IoPFAw|MT+45)>aAyJ$$a)HWfYpT zB~=cm9)~IZ0etRIdx4&}DgpZ|=R^k*#R(&hCw6d=Kr40~DqpW{eMu=gLMq^lZ7rQHEG|!wj z|F7PaUabHQw*|YcSr|Fd@Vi+`%_W$7NC8>8=T_h8rb7w(bw?^b-M~zgCdH0rCKrF% z>T|+M5R0k4YZ$Z>Qp%)>`Ja3YdBKsm zVVyV$qo)6r!|I?w5M5WL!s0G7km@*=O0sAlXzp9Qn(`po5>gq@A!|ja)sXcZpHpqp zWcU^0izA6|8QjWt+R#d@{H;|!xWtH8^6`#8J&is6SrOxCvH!C9juBmsH`ldYhecq^ z0|bwj%gs4R)Y2TTTe1Nf?gt5MZ%M1=7CK$enUN}LI@aQ|t+>~Q zaC@o!C^FjY9a&r>v1TTs+mGc~rG2H#r?~*iQ($imtqXL}RVRHEJ#u_{qk&1|$p-oN zL-*)b%oVSbulHqfbI?**W4LS0&FJF*9arBSh((|si6Y^#%GGBhy3<|9znd9633a+t zCf7hrV9(%j4?-qvmhCGgxn=PQ`vNLj)x^j75Pr|T- zlKVUX$qBn;-v9dL5nS;_2Wl_V=b=ym+@)BZL1?q_(61T0@t|RKyTKEEe@erk@OPgM zoSWJj&v#H(y~?y|CCGjtWJ<&)zQbmSgA-fFLH7}cq2CNAdx)(ue+X1@06I6zX&UJT zk{EvV%L5QQS%e5j3*2S5EY9F~gRTH1O6{hav!!E2hq}pJC@5)#o8RI3i&O^Yx|jcl zlK#!z2ZVhCG@g(Q^+^Ub^mPS?!&}GQ)$DrK`-;jqI&BG+QVohF=K~wUYB-@u#bTd| zFXndpvm~x}l;r3P27WzZM56Tiyh>wJfB7BkfHN?!XT#xv(GlRiwlToh_}=Z4kc*!Ca3I;;_tsLr!ei3pO6h@Z6r>ol|KT>GkL-2)a;>mkz1A-P{ncm<*IG*gJCQ*>YNZ78JC+%yL{}j#p^pVncI#F|z$)Rq)DEsXw8keBYi3X8RLat&wT;6^1=C_{2&NbJA| z5kwp{-(ifYWl7grj+Fho#Hc+IrSEX3a+d0?hXCmPQM5oAzl~yyz$Y25YQ{7op$vHKgD)GIJusO; zv0l2-UA@+2FT&#cZ!Rm~+8{i-HhrAciGK;rKQB7T0iW(o)+y#++tNQSeEWp}SG51_ z=mFKgI)rZ?y~*dFTLS)VXmYzpIKVEP#r@ymfakDnoee>Wf2~-wVA8Ag;|HaqWlvf# z8E-;gRA&EFzJ8Ib#q-C(KM=yofB$lQ`!POg!jnqo;Pb|2HYP14`sgF-F^Uj9@&~8N zw_j%6S0Q1!>0I!EGm3~N<+_mU;QFV`f0D6WF5$8oymGMT!N&qI#dF8vEu0fnO$y)T z#K?KE)=7vnM58SKe2|#TlLTtHln-G$^p%D;B6fra{EsgZzz;)PmVF~K;|<8yE-C*i180x4`?E+scT!k5Alk2^QU+Zz76&4sUjrpons|?QyCbzB zMn`@ETL+C{cKcT($!yLc%Z>JPQ^s~cV`KH(zDzq!?M;an#JsOl>ksW5xrKAwe|WFl zGTI$n7ozVUr?j*8@3NS@FX5Gi5B%U6G&Q;c;Zp`rj=A5;1S&oC(-`v)M!R(0yNyWI z;dn{ec(m#+c*2Vq$*UZ>{j0ngBxEzV&GxrKpsPQMwE9PGzf1e6g!IW0Ws@858p7CR z$ykTA@z#81rX*Sj9Ig1MZi5ky<8?n$GQay*$AgCRxhCm%)<5k^`)znZyDm3_wePNi z#ewE7Fpc)$GGcBItpuhA#Ynfe1MRN+b;Bw3AIfyve^#Zmb6CapNUApT9?z>L68tzp zvc4Y6XlIzmQjiLtvz#aV^5iJE6@rE!;hu~jR5+&YE}H=ry+tLZ@Gf0qm@8{+)7AMz zc3=#A?813lzn9$p4H)|LKE282XO-$q5bNBGw`JUR@vOR|^-`ti6$z&S3D1&m6$pY9 zo2v}5nyg+E+Z}I=g7Vn>Du}h54Vswm*+`L7jm>-Vi+F>+v-Q}_LP=&95vyU*Mr z8Dv6)OyAk`b8@*uzk4j_lo<`Bh###zsM1RYN8HNuKa?aJjioF!)qy!Oi*e901__Ll z&w%IW9A{0ut8g|Ek0#F;J->+Ow=Df+8~nFWcfma>YxkQ94CdI5-Vq+7tCn0?`sY%x zdox80H)mVZ$~vu1+SIbCG%inxIIM{IV9J!*hV4fhZBDDa=%cV=^T!mAScZ(Qv#d)z zilJJ}D7dBQQYv=s8QLkHYO)?PDVHu`?H)54yHocl>2uneC0)}peN6W~4RAI0--ByD z&&qA|M9@DCj)=8NUlN<74A};}?RUnv9-XNm8Ynb|M%C#iOUE&UlrN=~)|`win zMi&)2u!*U)d(5h%CAUqO);M34=wc8y9)OnTHzKY2YqcirDws-zaw>@?4>qdhd~0J^ zCjGsPJglPO@_}HWj@CQteFnYf^pT4BV|(r|2J4o`IN^<#*Uwmtwp>a}{NX1ei0Ucr z^DY$XKT(M%vd6b_xse%i>#hWQm#xr(rzyEiV?In4csULfU2~Zq(-~V|F;Y`F3CT4MsqzWMVF_w$e@op!YUt-qs~A*s>;A@c zk9eVcFpPvJVY)~~XO%%?z0s*z(&m!H^Ws;f!ydcfo1o)3S~Hvp_YPXm_USTtc`a>o zW2b8St7Z60?M!CTsPsojMxu6bJ7ef@Pv_=m{(wy5Ml&+X%W%WxhRW^Ms$+nJ>DY)5 znwy?iDZMCDMq}XXDEa*JPKD5^=>nf2kx5sX$pa4#ylPUKBC!*`3%l)0PZ~#_S+9(Y znQIfc?tZDCt+A)>B3Wm(WJxE3N&SVHE{0~tH49rN_I0kqF^iNF9XQK5^r271WF6t} z)BA%ktiw!5fIn$+)ARZiq;{DpYhz8XGv5?3n%K8%veT!+k0_mgkT-Mw^j1w?O4`=g zc~gc%mq@QXW_J|LrMc?FKp@IayxZ8-SD%5`8zD-iN;1fJ{lz#>P;kq~U5{6rY34E- zhZ@YoMlB!(+Sz`3?_${ zwwp8y5;xX($l)TAs)Zqw<_hgHI9&Ow!iJb2DXmM#!?jf7#o@r1m!2AIU(1V_10~~% zT}Y44KuXpFKXL6oB;OL5Pf2S5P(+UAQI(3`* z_>sq^Fc@gv-Fd|GO!4u34S4r-u>6%-+QFUePiFxrbw!g){l$`7SF2!8W@Vq}iBh>K zeH~RO-UOC|nz>$hxB(ofgD5mYV`j)4e(u+?7mn1$OCbV|W_+1Rkh^~0vnpXIe^VwP zoyOF#s%Z-*5lMYJpxf+KtxSx~Zu52^(q%gkz$`tlL<0)V@ZQs~{(s3~YO5_nv+R&+N2;Za;nfOh5~9mCd_yKJnwZ zg4;Hgi#(Zl;V2>oS>o-jLhl;8aa!pE&{8c_jBvL2lPq%0!?kTk>mCuCS(XFoKP)>V zD#5VWe(tlcIcqMPXmt6j2OANuhZmzVIZI20fMcnI=J?*ttB{d18V&2*nEoNADpzuS zJO=;m_u~4tn+rJH*qc{@XBr@&Q$aBrd@O#;4Zqu6PN=9I!?-IkAZms?!e%kyF4W?f zG(fkryUWxE_}uPpt5^key)(mA#P<_to6L&ab+PM}-X&_q5U6b|cT*AY!~sycJC90n zl=h|Lx0L7~DL>+T2V8+oH_ox|ndxN3j=iz#Aen%d)s`Ao0kECbTI3(bR5+&0tf#j%;CQbkRnh3NO9k(!`yYCCybw#J&LtLbT_mKie*q>aks`-`PrE;QXD_4hI*h=16L&6<2$umXyyq!4$%-2) z4xAn1)kh)4+|^HtWq8`$wF%F6Cx2~D7sd?veA{$bDO*G_M~Ca{xkpWAHLk1E*qetc zhTx9KYNnF+j>ZVgiVZlgPP?9*{zRya^P{ey`fzFxSUZUf>NHLa9h)k1ROIuOEo<}s zl5Ts{B~t9!50c^a6kYQwu4n?~o93z$=Q=EX_ak}#%@2a@_?`_bqAA78ejdI zl<@}Y^19W0=GOT-u)nKgyA^u8op{Lqb#D6n=^*=Z`xibxcWhIkudouGfw$k@yxrV! z;VTJ=p_KWRbZ~l|Lt*`=lK4(3PQZT?7_sqtmv-`z6$B!M9Fpy?2Y4=A>gmrFfcnK^Dm-p6# z?56Wg=3#u99FQbzq4n23>y_vYuY_a@IYvS(v8ZXO*00W2n~#5;UwGl}ZKe#sVDfNbpaiiU33KZhB%jfemtnY4)nez_EU-wot7)aF{T z9VuV!_tA5-78Pi!*iy@=Yrqjhu1-onL{xG+U1{#^X~8dtU|>7n;}NChDS8eX$sNr6 z<_}4blOSq9!cktcp5EQnaOaDJ;L|KiUL8B?usuV(c#bsM3Y$BRx;+b5FVv8^qWoNM z#?WMMJ`#@$XlhrttT+a14L61W7gURaZ5V<{RY4=taG&XwLD?>w@$IP3J%&MD!E~i6 zIgU~Db=PGq$*Vu(iY~UWt7Fe~TbO___REhQW6XLA&Z@0Q!I&LP&c~~XACR7Uo%}d7 z@P( z3t>z-b>9iz!HZ$yguz0nld|cKL%e;=qcW}6%iWc5{ChMj2VHtsv=FwD;wU3Gq`ewk zJ4)fo_$zn^3JeTlS~AgI#x+Y@I*%o*lZ z__`FSnw*_r%!e8mi}m zmky<^R(+mtDx!!+MI&z4z%-m-I-zg^H4a~YZKR;H zw~+-y;q;T?RO&$7l(HCE0=4eroyWa{1tkz#`eb_Xc>-!^I2w3)xa=AR1g_d2DVbEV z8Cq4JPv70-i3*4(@Fs65_x+7VVpd=wdbc|rl!Cl+sGbzR=s)5^$en%&RVUVSgfWMY zeU^xe2WJxgsYdJU-v1IhwvHc%32GUL8hEpu&;FVohPHOE7@ufI18y(S5goRRu(9YC zSFyDctt>|ptK+#PJ1*B39({wTtdY*bxiYDfX?EGoojtb(`y{;zn;Su1jAqB+#uuHo z-}KC;zK6&N`^ZUU9Bvf0%IO^nxO|8zmk%JvMLl}iU>g{4{hBHxd!t*uan(cAR+fSq zWurMOqIa<`)jWMBgteQ@mu)TrK!T8EzwnXD+8p6@fAD>0pqDwSv5OG+Z22@iYuH=q zjVeXKQY^2Xp?H980C_*N%GEo+V=stIYpwFvM%ncp5WdgsRv)gE{s3G(Zg@I|+Djo< zWapX_>grEpvD}VkVwDCko!0x)WlZL?KvD3p1O;2n)n~DXe?;$nRR!1Ebp^|_-l+(| zz7%;9>mo&oA&5;ssF$o$@(w1}tA%bfk*1?oKfQ{z^EEDF@3mWGiJcl98hOQZhuO>P z2?3?jPx8&5MEK;pL+SbDJ#xIuBgfcqV#TC}XRYV+XjIf*bJA{iF5i{ z3j4x{JXuCWPt~9IDI3E9=YyTz17j4aH*+Ph)nLNwgBqdm+2|tS5p+$VO$^J!yS3sakAp z(c7P9*9pMsW>h+c}6+u`#DBy zpLjkqDWQZ1mlc|ckU9IgyMm597Xy)tWfmAe*pOh;$n|eDzu>|3XqS5!ZVYqm(5H=Q zjjsnzMPyF6rBa1c?cC3ig{s@WRj&&qbglP2l5g(rGY5f$iCnK%4d0@HWR~WG)m9EU zLbSgiaIq)AG5FIH-3p1^Wt$e~so`x&pbFWW(}O!#l%bwYOOleyIHw6AYFg!r<@^wy zQ{(2HvmdgGHD_=jeVBMZ9u z9^+iBozxal0gl@nAxG-fEm0{f+B)>5AN;33YU|@JqvebAmWz}a_Ek*JER+pRq$?*E z>u@k;wyoARSt~!$ z5%*O$ z*R;m1f&~|)Y9^8#M3HMnwYl9!e)s%h4x8)miFrIyTXy@T$D(*AqISb{zFxrWS;Iq3 z)uuYw->qDaDxc3Ctq?R~q|NBpw83MP;}#FD=|YhVe?N`~XI8uU4vaqfqsZzQ`u>(c zPuC6~6z-9=fgt2tw^NR)VH@heOH)x4>G~|C<*1IUs-1d7ps4&mYZLG~rM1Gy7V2!`Qii$)>Ew@`C~%rEmxVMsJ_NM!~g9U=%D5%jQwU70IJD>{k! zp0bvw4y~K6Nn_dfwpu0q24hhzHW*)U==1m%w%e-Py1KT@40feGX*mmS4kJ2iU6!J3 zt7QofUHz)jj6TzPfG(|j6E za=WB;Z(0D?wSLj}rTXqa$*d?UX6=7Q;#dL^DGM}Lvbgm0{oaubcd64ks4egL{h0#x z;*iL?RyS!SJ?0_k5K=%CiqVHpeZ@O3+GY=8HkHKKblXbmTiOI5mED;>^jKBmjW3x;R^`mV{tr zeknICI_Ys|Yi#Nuj6plLkaw=jmVC86E48DeLcUlukdKun!ZN zbdZ$}15px{p6p5aPP+UV9{})7^7T6$!iE31U;*4PC2??g7z~V(01A%C1*@(z#6Yi7 zR*4*-JgIB`kPuG5qV|6hq22?S?t8!A2p^&$+`5YV>HXvwVx&`Z=`_}^JKJb?esP2BkcX3UqMc)XqvJ8$s%Kxo14Tn z<7H<7WY`~0`-}211;&$ff5WirV!#WCzWU8=_-#;rIe{Z&FQB;Gn~ggMF^la@7j{Wi zEFq$0ko-Yte5_Rk6aeD55prQHz3^B>zLo>xKX3f& z{eX*(Fd%BNCGzW3<8OWZ8}9Z+lmt=iy@i58!59DF#%UYb9Xoag_`FcfF160 ztCdToHwqHYmQvFxQdUnk{?3#BdvXCOLFanJ^tt-@x-OeU*Dw@nCz}-xBK+Q2-B~~4 zvyZuY2P@ucn3P{=fK2|F5l!cTK{l8pKB`chuWq&U<}a{l#xMm01n5+EEO$9v!evr< zJu~Wr3HX1dBKC#;?F9Taa{wPz+p)!@Crv-8Lpqo85mRppoGV+4b{w>%(Wv=6S)s$` zdcFSbf`C(l-Ex5rz?r^!xY`XC3Cs<(Jr&K^Zl~k0T8)7_-P}DFW6*k2u27;h0lE?T z9k2ibAJ-6pcv|@nah2r%$u%Vp$LwGs95X{k3NU%(OM)g0m$Wg(;%I$Y9}3#yXjRmT zRG0wN6gqi3)vKUI!U3tsXgiwT^@i0Zqxk4Rr@@&iGZe$+k4+cHG4A>Ylo7xP%g-<0 zJ;zZ5;8>IvY~iK<2`W_}{!fgC2`NW5b&(s7L&l;tlUE(HHR8L{>X+cNUlWG(Xn(9| z7mY6v%G!qBi~g)WwP1Uc#t*QtE+045rC<7| zuVr7|ly)2eniU}Qf8t3+(BJVS=7^5$zB?G`J3b#nE zeU!p#0Ch|TNW|)8Zw2aqdCV#Q{ANBb`sI`DU`1Oo_66c|)z`Wo^F}miAEC(2zUR?E z-Hc+gP4M(tuX7si(1+iuNxN03-0ZU((>NpGT%`l1{jj zY#+4-%VzQ-effBN{O}=z_f*weR%v}$^p@GPNeLqz27;#nSGB-O5ZpDZ5u+A};N8{+ z(nFKabA8 zhfX^mwYL)?Y2~k~0Dmdk<6`pzkqW=DNY$8sZft*Ef3N}c;ll~cC-C349EJy2(gW58AR_$dQvCDc4;IKjMfD#`dH~2$<<$D`;PwBGJP*u@fq$p=zbz;X z019ONzaRYfRR90E!h&!wZO(V33D`Yy7yu43_!q$I_G7){^SbM{WVT*w%s%IHIS~Ze zGYlQ}@cxgAV}$!y!|kIrV~HaIWiXX-nULj>q~9gnl~HgP8(_}F_aS4S%}n)Q}wO4w>fV+qPtCS$lpxW336F#j#E za9;so6@u$M(D(0XJO|}U&K)(q-JQ%u1&7A2QOkY>825Y7Brk$VAn7Ddyd6qO@L14< zv_238Cx+A3Kr)hefl9v%&JIX(jGAj0Tvtmd_IMfJ1`u3#C#s{SHpcS{V`;>pcQg{6MGIA2S3KRvFL}gTlFVts!!qgRqkLN^>dK!BNe_-t+9!it47rTpbYQ5%lmH~ z75st$7>(%f2mFLn1s`DibA)?}CTlg(l6kCs<6P@ojE25yE_%%hS*IZGUTxkiO{j}&xI%P1YsE$BT&g|r*#W_wOtq1o!`Kx%shlk-r5 zWs}Twv&ona{qY`~x5PgCp@Vk8dFoN?R;Kr;21xr5NGXuyx0n=5@jk1Rx-<4$$FK4P z;G~t(Jwfej9?zX{nIEEKI-ZV%i_g`o9Fmm^HQwhphM88GT@sqEey=@3yREgKv+``c z|4~^2)E-c`R4b3;Sw|TKIP0-LDLw)b<%b?QE|0A*4$!7l{@~LjK9h{+#H=55qa0+< zTp?pQihaym;5DzWW!l`D3qUS;F8D2VyUd+5|L19?`fp~J8cnFw{3!4ApW9kzd4)6f z;&1w@F*)5Fg6V)_5G>j+&iuA-&hbhZY}8@FZkI&rrt?DSVwr62Wis_^!L(S$-((Pv zs=e+J!~w#+gJuQE<>9=bt%-1zY?H!ocG+JWCP48*e@xav=hq=lKuqIx)R4W4BITh5 zxZ@LMgw!=O(?;Gvtp}Aw3FCo@>*c(q~SL@9X73!@ESFQ6SFN+qF}3EFV!FWcyL(N)HU4XxYBnf2?F>4joi+`s{njcVc7Dh%aQZ@AiW47`cC?35I8WvHy%Zi0r>`QAs8@c~)zUPb2}d~n5s zSyF14f^s`8Tr9@3!-bd}R$A$w`0wuTb47;OZ_zJ1a*c-a+Ajl0LX#^u+c?zIt@LYQ zQJ$&Mbriv0suU>+aXa63nL_cz4>n=@)KHmSasY63 z=O6+Pyb}s3k4(c7jYcVr`dcONz~F&Cf&?M$7a)Tvd;VT;n@J%jnwZz6AGq%!FD#AQ z_QNMBh&ISKet1y1_+p+QcObg+sS!XQzL)iPT? zlUD!Nsq7*S%(0H2LEXvwCouBnb47(j;UOnkoj)$TQfC?AimRtf6^AtP@Dfne3XOLz z2HgC-E`GgTXmV^`<=_jl+XhPB?#Q`a$(7N(4noW^0b897G%=f};UP!C7P(JAy# znn(l$N0IK;Fs9b;l{TXpcIus3tTajYfYs=?*U)UhZ+4lh0(tr~_}m>ZY*1mb`ul&r z&*>-XxoqPgJ6x=rJ^Elnm%?dU1?0TQvs*5fFte3Oa)Ofw5q6h%CDmJulIuDqkVyH@ z@H{pfi2<)Y!Dt|60NB#hL6|f$4Ci_<5%_$qKNC-ytaYGIBhW>@> zEZ(N~-GGs&6^i*2ibBPF_U0IB%XQ=EpRfkq>aM*#JzKWR!UIoGULeMCuQ?nS(j1TQ zo4!nCFL4|uJusw25BKXkf{oQsUxgEip_2V%EL*O^;kVJ<50s_3DDz~?0!lZC?F=Wk zsHAn&K9iuqGKZVxs37(w=5&#>emRR+@#G2fhWG~|rBh!{`baw3fJjg ztW;+7qO^|ur^u}lPCv>2HST=zs0DilP$-OGex^Tx9U!DaSawCG4GU;>2N#m(y)P}( zud+6ZszUBhU<>14-bR;VvrKuwrmI+WVH`5&CQ}*cq+IRmb3f*k+5kcmxi~bmMvzDn z?%5zFInU)5+o6NO($t4T%@8Z6iVu}v?LknK7?k>)=$JGv`_J$LN80XN6WO90AW%M| zs~*xuDT8K{V0p%QyTb8al}fQ1)n~)I>r<-tw+{&>?3EI7W%?u>GQ`@bB~gQ|nktP) zB@<=uSU!zTN@T-4MHnIy2~*n^{0cWmo~_pGavIY{I!FZB`_ON)*Da%5uNgx z_u}G@)ma9h%Xb)@SfbU+cJL{dLrY9Ku}aXut<*O^k6`5k>;~iY;~Kl4$^4#qdK<^3 z9fy6STkM8G&;62Kef%_*TugIob39+;8YX^ah9da|7XEC%Md1FfIP#?(XLqxH!#MS;mr`OFjQgujN55r*Zd&zvCTkRMqHPJD?-N2-n;` z*YugHF#gUr&^+F&a-d@c3JqVMm)ERf*@D0lF7-HLgc3PlhsIi9Ef1 zOzN{w<_~sZS2OL?^o2);GYm}Pi55sGA1{*^S!4Ky^Zu$1P#}k&cmMP&A6-if*cwhG zjBo}@UabB^<|vIx+~V(A@f@M? zHvF7tg&Jr0r|NPlH^|CuR9DVylzMQ5*}XjV74|l4?I%052uR<83WE_dVB{ zo2*v&W~qCB>Z*z#+F*FMGm#b41pSrICBAuB@cebA&ifLykZmS41?6VEn;Xy`y@Ln1o~tytE!Lm zJgmaCmz5*`sQo*%e=aJ<4~FJ$$0gP7Va2QExg$Zc2gDK@brjwCRg_fUy`jGQgW?5` znd^mp62OunBkL#N_efe(Hg4J3qn~sQKI~%MBD*KHsEYbF_Ikgxe)+SrRNF{+Agm0qTdZ3;(mM}F(cwya|a7pCZ@;w9|vWHthPCTNW8%d{UwZTv19+Y z+v+$qESK>%PVLOh41aY>idk`4nXSN#mlSM-37@icm@6yi9vj4_ zo6P6sPTib?Ag2NsectA=gA3)lYvG+znyDbeC_eqMCS01E8&{_*^?aT=7~T*q6}{FokKscSi;{%$w={|>*#^yY&Zht>U8EimYt|&TZ$GJeSk447bm|JN|4w zBl%~8xGZxdbk1E3P}ro?TNuY|a=FQt*u47j09(D3dKyiN|MZsoqnn#0^Z0a!_{x_! zOWkL7AN103$UjfsDN2s#+WU+HASe zO>|EmfP&(__-i8DLq*q| zLG1R#+upSmvg%$pGOL-J7??~YnYeVoWnXHY`D%ZqII&c#aj?~urZ1?3qraY8z0M=> z@_23T?EL%(lN5&F6W`}B+V;$CTe2z|3TCMmRkpoNoE|`&fqJP8Lh@_hA5Yg~fgf^V z@2tHKDvN(g$hdkK`(WA*9SN!1XX4R(_0@cB`Hp9{*_nCis(GoT^7vwxdA@9IZK`g~ ziic=(FLjo3rgBb;XF@v_(!fp5!aRdf-*|i>@Nh1@KUc16{$ffWqA=%x&xY{WD6uK6 zBNT_KEK>TKtaVt}J8WxSi&wByu`PI#< zdiwIF-q*#iE%$2r4%XklsOLFu>I3mvukxmHShuxq!cv#FL&=&C%2gMI5Xj78saJO9 z#?pDlGGm$B)4s|lxF0vKk$PQEgd6|#*GQ$ctm(v;+@fM&7SNYWV5;s3OI!@HN>&gT zPtH6;0rnfURG+fOA~YSB)7;w9qYL9GUhbJhq07#t;}%0a9fy#^s-=uPJ$pmbcF(?r zOs3ZPx@9|iRWjBVM;FA|6BTLAV(u#_gseh9=Qc5%M5RPc-z0;rvs}@0Tp80*T9r#N z4htCNOZgMSv~KXST-7H;E$<)F?7e(A#bt$cbI|)H_l>LeIuL1dYQ_blT1Cgz?N2ld zP`dLOSD8lb;E_E?0GZcCCDD7sio<;?DGq7AT&)_HO6m05q>&W1rQWEeK7Q!6;gB;- z_q8!|wCzTL>$!v%JQwKAcjew!#iRTn-c*mi^sDV~ zg4z6net3$bT3g70@!H+ar?U7SiDqa}s%NtEqA5Zu3&=e2;h@&obYXjFwV>?`5N8O# zPj-I77(Y18UE@Z3$uR0T-&u9PiX!SZ5C`Y74lhq&Q=|U*xwgrR+ktOR6FuHFnZ$RS z7j<(wz|slucZ6z>n}PYjbwlMXgOOZ2zbRW!=tp4(MecGOwtdi|=`T-~yP zJkgOucI}+Sx{q(&7{5SGMEfafJMVOIu}XOz$C}3wQ{%PpLk=U)Q}{sv$ldvc zR}?=tqkAkhNj{xp^*gttM$lors;+&hnOpODMVMu}St0)DZrzd`UOx zS%5_jD2NaCBWf!1xLq(ga&yg8Zf4?>C?cD_Q0Y7*vQF7*zWKG-{Q0NBn%y>8z4V!P zIDMAwbgHurivnb%hN3UUqjcujypQ`mDrd z8Ikp-gB1#a1b1Nfvis%_vJuE%MjA9-h&%ZMjb*v z1tIo@p!RNqhDmyLaAT9b~v4gJHtLQ5DG?rdm5crruC5Jb-V# z4~AJ}s=K4oSU9VxhvB;dPByMDG(%lDd22q^0zKRB{++9=y zJJnPK`{EnJM*jUgWuchp=@u;k@QOeWf;4uwsi}Z-#JBHWkYZKUYaulIWua%!e}SMU zzYY_(8=*nr)|gw#uY3gU;YgG0qWCt;E|E2g1D7vA88!JCQckJz^eDN9P-o3lyESd6 zH9qIXX;HF^Asssy+g$Pb3i|d#0vq zg|(Ph7ZW;%8r;L_Jag9I)^w)}MR-()P23o(xcIs?{ltoXSpla~@!8oA8nUOXD&mdq zio<+h>B^}w&Lvd0CY{)4aKoH}dp|(B9zQ`>QjV1i!=CMV3CvsdOJ&K%XhZMM&vLCi zzV^msH0_Csdq*$Zc-rwFs2Fj>8l{0`KtM{&U>D~YWYR$gp~NDZwxH_-eBSE z7m4P>x@t6hb10yc1tDZ`T2Fn0bG8?0Ql^zf#bJ(-@hH81z*|zMx%;AV%5s8+e*w*U z=a!tV7Tn7n(yQil+;0=u2S02U(wc@By^)FM-=BYJ*?#Y2USoi?-*q7hIW_^MKOoHY z*f+J_?zMZGM>Ddo@s?Ok+NLOQ1fLc06vm8By(lY=CzZ(=i|4U6J0zHdLZZN>3f~Qv z!pU^;mXViCCA6_-^nUwJSyVvNX^ro_{Mst(`{kg;w|&sXHH+0Zr`+JdCFVnLK9#jr zU!?Ri@zsa*r)v)C+Pa@cQ;aD-Qa9HIq@LhqdiNsev7%?fzWotIWj$IgKfNMa`Zf-E zg~)K4_fEIG%?zP|NIn;*jIK(d(EZX%QNiV@e=nE(WGdw}t9$>rQC7jXv%=-F%c+6999`RH^2YN7LKK5#&Bb5&qQJy?CC=A_8nii%RPt z0fAXGDbESA(RzKWaP-z521@w$HSgo6WMcB*$a=+2`% zA-VhZ1!x!SDFXL5?*u*u!_shBU3h5^;W((^y>i?)l~7pM@^=c(Ih7M8-11wLFth#s zRv=EP%`XGqOQ#*PG33I-lLfixxPHw=0I|VO7?jEJ(r_w|n8q;JjX%I6fk8cGRoF4z z@BFfsd)kF@Z}$9xW$f0oZpDd6U{;;mzyG7;CKWqtr|axZI=hT3WNNZ$4_9yIS+sZ62$AHG#UsI@hAP z>1#xDSvWVZp?df- zOE3@?*KNfr^E-c@kY@I7etZQhQw6KL%>K zfqK;^30+5^KwAnd)ZpL;FP}5XwG5)|6qWJZzSlMVfy#ce?{G-W<5WTxMRhPY?vSYM zd=y>mS*?o)>VEb-S1_EvO(Lw?I&p* zzjFjdUs{k3Ls z4w7UZaKXubmc?_tTXemL z7J$R_#M*V7yEWq1AU!l@nPCHeg?X6X8a4ddoI`H|9*^2uZ-x>YJ|!#76Kt8SXk6Se zwdf~1ahN0uDAg==w5PV@9DHHK%)PaxcqtE4%Q`R5;YIa22!H zKM}Oeao4OeA4b0+`xC<~H2s6}V-qj#uR6c%U*3oqnoW9LlBm&q?qvA*+u*Od{OcQ= zT~B-l)AT(=RU&+~ko&lX-S*sJT&?rvyzZ>f_u=0t`M5*d4> z6(}p%x>i&zbtnQ60S5sOelE*9OA&&M^NLxmWtpSZURFjdIG?K_buCaDmKwL*LhbgY z0XLY*2ax=m2Y&&q44bKo0;+2rQnL$nXYmJ635WEMBcu4LR+(Wb|%w??thObXG2Cp2s_3gYl`{b9`#mDtm zF)v%x3q%yYkh)8j?X`(Q=I8guCDstjq~E#IY;SK9(S)>uUrwzz2d6_YA02$wD-X!| z#mp(kNzo$XqCC!$L+M=a3dE17tT>7)yByaxi}5O5OCB&&In}e7b!R<`4;2Yy^*}Xj z0h6&UskoxM%*2sVh+kle$(zW@O06g2F0;>kkpA9OIDrwrKN;>apK$T(VUdT_q6U{W ziP090XswlLUel!h|7%gL9H+>of!?hX3lj>B%|14<)Br&*xQ96@roFC0_C-`h*kyv&xOLRBLaoA{X1_6#Zj zfpY?^$gx#=tN9Lq4h@jw13ogv7?|L@q>_z4&PAc$9jDUYOYDI#=;UWrMn&c8Z&R2T65^U`E_;2Km9{PU}b9hl$H^{1T|f0dwKI+(f_hZecS{USk? zU5KQ^rUbn^{A*R2hAW&s0L=<^1z>OTn^;q=7-GuBUiIb`8gc(+lJ5L~!k=4E1^=4b zk4cvPN3C2Vu&1YIQBhGv^NGSIPrN^LrE&PSqNuuJi;t9g5)TyugKIZo4|^Y9ob8N@ zi(|WY^0MW%jyUh$!#{Nn#33A*58l)k@56&~u6(~u92#;<6<1Lq9UBRJx>9bVbj~&; zy3vbpn^PRrPN?JIqbRX)T(Da*%0<-?5}H7?YPck!QW;_%3ZsM zIsFKHw`XZBAwdPX z#N&24=gR7w?9O|)$?u{HPzNV7_VhcHaeyPfx^fzAR}lVXy^N(9D7M9MHQ?=8=R z>&`J8`m%3eyynan?&TC^=;_Y(<{0qYyom&gnSz{L=?(+a`YT?TsaMMF{8N})2TywU zei}JOny}{Q^`5C_4>?L@s6gcp;p&<;x2fT|LXOws<6T+FDYWZz?#&uW*O&6{8Kw|M z?bj$9g(Yso?RYVL^Z7llA2D+2cQ8&R4uT!F6@J*=BC+~`spbV64XZUB2BmOt<4@q> z-hf%M0ypJxbRyy=Ha71#-#kL+DEm6Jex;okSa<;%vR>O(23TN|9;X>pS5 zhkfz{HOZ|*dSCevs9tRG#pQRz8!^8i3c0?E1Vft~7K`84U+GS|F(2>Vz|Id1Bv=gH zf(Aegbd3XPfO|l(;$n1sJ{m1ojXM+^|%Dp!fC!_76 zKaV`^)9~3tFdmeU-g2`bpm^y_4;?`#dJH^&T>K*L{+{`Qi$z0zSHos4+=_~JZ&GDdd|r5&Oj39 z-*KOO0$Ur-mTkq97>{_)e)@rJ)?*9g(pz14X)fIj+ZZ{IFB%UIF3KFsvUFSuHdDA4D*(eIN ze?UdHm;=x+k0ov3V4tOc7TShmEa2pqd1eB_ewCNYGiD=2QAA~8xKQ}qM|D_!?}ZS# zxw!|nc~K8!qhiNrc5Kt{SgH6qs#-cd7^z%-AA7dXnHr`7>kwG>2n80nOQ*?IKd>=; z_mjCfLi<{n0$s7!W!QDG+Q<*I$;H-1X$-~f@7{KN&o)ffgV<@&$#{4Fz2>Inf#cGb z?(?IF?V#FymI_H+ohGXcOnX=KO%U+ zgm3&4Q6aRTB)}T)uerhnGcDuYV_Kcn7g`#9@AclSDCpfM?3rfgbQ%|@C^y!JZVHiH zoIFjeIhYJ+h^B@!aiEpvl_iTBL18x*i77&4PhPy>x27j zH&ldq`D7+zO1(WM6b@51^*;zMySpA85uw@`$G~`6^sz%s0g9AqLf}bbn-H)@0guh4 zm}@k;l;x1tWl#x3!SMXS5($-K=~ZXy+;gvfD6NgZF%xwkPUpO`-0(>Pz;=uhM*8VJS9Mm|0FD&X3)^ijjLQKWP|L=S0HT}fz{ zwVCk^-w!j*xdb8I3)ZJaczbOGfm2z3jhlZ@zhj9|8>U2f@r}wxJB;Hk0TJyWSl3eT zbnfN!3rjPzv-81%?dl#yy`AY~Qez`$yTWks(UKpR0g`4A@Xo&n3t0-NQ8xOM;-{aI zUli|d_S`+Qk;VH(;Jw3^08rd9KpA|0g8&<^z)a|GlO0PX^%9mDy5?nUkspg5 z9#&>oT9)g32dZ6FNGhhs@IOp+n0cGY^SME(t3Elriv4lV1Uk!IaU_?^Bb2 z>E;q9;^%P>4bs;wAQ9h6P|HHkeWs}S;9ZU$Ier#Fx=nI(Uj>>E6!xK_vp7^T~KvVmq{-($GNnjLFYpA*IhpY3n$U2s__K zzXlABxX43}*~C4+yBs!l%JE}SO+%v9$&yX@?am<1j+7%ieX~ix+v=n;f)8MCfF!i~ zUR`h=BfoSG#Etk~>)@v*iy?7(j%6XHP{NXzE=Hu-=U%~}px4+f*Pr?NA2<9-7DNsM zV7}I@Cj4*SK}+s|B0!9_pS3eJqpp{q`R>PGWJh2~WKZwA-v->3&2$X*X%KqNJ*RG7rnTLJIN%G=-`ntU7TClw_7L=}|2x_k7rOSBaz9i-_7e9S zbDc6^RUcp{j~01V6cayZKjs|8WPGQ#WYgILUBdd7&B|p#3G5ZS`MCD(CO#Gz2J!0Y z7Tl}#i0BAm=1khlQO+%YECJzQ&5m6e4*!MVu|l(&65LGBY~0k z*-(beyn4Q+n8-dDYttaJW+at$0)75$8_~d>>PL(VPpIUP(i5Dr^dSefDqq^LsE1ZP zzI>|w9@jTzsxC6Dfvd_Hn>gg7*aT_>3g#yRx7bcJiZPSZLGk_r5cF2<#zR)%{iL&L z=dT0K)fO`*O{u0m^vXJkqHYN)vgt@uRZpcJtpiwzeO-f-`iawjI4PGB0I;LS#d(>3 z!5tNb(~BfSQ~AI0APEJSzi3H`@{afQh&#a|9hSi+LJXi(^gdRUtr7S&VVgw zKLLCq?4oK%LQqCeqUXW=9Sv4pk+0eK5x*cy&Kev&>p+oP#66C;t zKvArZ2^1Hk<189?T}o1Xldw^n+CME-lM9lmNx!H6j1W*%QB>#a_3^)a85EinA2hQ* z9DP;{B`%Y^O*HdLxtT&RnwLc-H+7$i@ZG=QO5B?m3M)qnS0Ve_I;sC zyZ&;8@d4eA2x4zsozytm$Kk$(rRDG^wQogd8sWfMgKEkr8G95A|J*=xk#*=f z7z-niNRPV&Ytys9nX|=5(YHe6R&xdPVXomlM@3a_E-)zT;LC(vP0XERqsDsb?H)aQ zHVBlDe2GM5-!KC4SYhu{{XA;oc)bjap(-Ft+}6 zLTr?m@buXO8^!)H1-6eeqLbTIU+zhd5>9y&IMr5p)Kso*$plmAh*V@SO2dLAH)`;U z$76-~VR$335v-(VMNR1foK|)C)x;vwNqZt5^Cu)`cpD`zI1ARK-+8pC_7ZQKxqkl` zepoLw!nzs_lW(kTip9)MOT(oZq+g05ix<}N$?@!6>wGd@)(~S}ebb1CP-||GJf0kT zk4rB$K#LQcABvWc+TIbu#2xJR71!CR;#|taImkD=L z_e&maGx7{;cO$|j`S+GZT~$UCc8eC()YRBA$Q_ktNIkt|Y)b9xm%U&&h%T_{QQjVD^l&Es*s0e zbFz*^2ihWOkk6O9U5&N7<+Nt%U$6h(3j7Py?xI+iED4dKFU{9#Q&9rxd@L8o;2O-d3}0~B8XQ;b!vLG z)?sSWZG1F;?lS%jyiCcYI9I8Q_*w1p5Bd{bwNSO({V30ix+ZqZ5EC9k}VM0oN x0NqUTc1jkeJJzP&q9_7W7lr5?DwyWqzWV$F)Yz?l7)1yCAa#v(inSag{|5jHCAa_p literal 0 HcmV?d00001 diff --git a/docs/user/generated/model_deployment/01_deploying_a_model_version.md b/docs/user/generated/model_deployment/01_deploying_a_model_version.md index 26fd12b2d..24c7ed170 100644 --- a/docs/user/generated/model_deployment/01_deploying_a_model_version.md +++ b/docs/user/generated/model_deployment/01_deploying_a_model_version.md @@ -150,6 +150,30 @@ with merlin.new_model_version() as v: ![Autoscaling Policy](../../../images/autoscaling_policy.png) +## CPU Limits + +By default, Merlin determines the CPU limits of all model deployments using platform-level configured values. These CPU +limits can either be calculated as a factor of the user-defined CPU request value for each deployment (e.g. 2x of the +CPU request value) or as a constant value across all deployments. + +However, users can override this platform-level configured value by setting this value explicitly on the UI or on +the SDK. + +On the UI: + +![CPU Limits](../../../images/override_cpu_limits.png) + +On the SDK: + +```python +merlin.deploy( + v, + environment_name=some_name, + resource_request=ResourceRequest(cpu_limit="2"), + deployment_mode=some_deployment_mode, +) +``` + ## Liveness Probe When deploying a model version, the model container will be built with a livenes probe by default. The liveness probe will periodically check that your model is still alive, and restart the pod automatically if it is deemed to be dead. diff --git a/docs/user/templates/model_deployment/01_deploying_a_model_version.md b/docs/user/templates/model_deployment/01_deploying_a_model_version.md index c41c99a09..ec007ff1b 100644 --- a/docs/user/templates/model_deployment/01_deploying_a_model_version.md +++ b/docs/user/templates/model_deployment/01_deploying_a_model_version.md @@ -150,6 +150,30 @@ with merlin.new_model_version() as v: ![Autoscaling Policy](../../../images/autoscaling_policy.png) +## CPU Limits + +By default, Merlin determines the CPU limits of all model deployments using platform-level configured values. These CPU +limits can either be calculated as a factor of the user-defined CPU request value for each deployment (e.g. 2x of the +CPU request value) or as a constant value across all deployments. + +However, users can override this platform-level configured value by setting this value explicitly on the UI or on +the SDK. + +On the UI: + +![CPU Limits](../../../images/override_cpu_limits.png) + +On the SDK: + +```python +merlin.deploy( + v, + environment_name=some_name, + resource_request=ResourceRequest(cpu_limit="2"), + deployment_mode=some_deployment_mode, +) +``` + ## Liveness Probe When deploying a model version, the model container will be built with a livenes probe by default. The liveness probe will periodically check that your model is still alive, and restart the pod automatically if it is deemed to be dead. From eff5126a80786fd0c88d3de8dc0ed40f6cf1fb7c Mon Sep 17 00:00:00 2001 From: ewezy Date: Thu, 23 May 2024 16:47:43 +0800 Subject: [PATCH 19/26] Refactor default env vars to use a different struct --- api/cluster/resource/templater.go | 5 +- api/cluster/resource/templater_test.go | 63 ++++++++++++-------------- api/config/config.go | 9 ++-- api/config/config_test.go | 10 ++-- api/config/deployment.go | 3 +- 5 files changed, 38 insertions(+), 52 deletions(-) diff --git a/api/cluster/resource/templater.go b/api/cluster/resource/templater.go index d30ed9677..ec9c03540 100644 --- a/api/cluster/resource/templater.go +++ b/api/cluster/resource/templater.go @@ -21,7 +21,6 @@ import ( "strconv" "strings" - "github.com/caraml-dev/merlin/client" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" kserveconstant "github.com/kserve/kserve/pkg/constants" "github.com/mitchellh/copystructure" @@ -933,10 +932,10 @@ func (t *InferenceServiceTemplater) applyDefaults(service *models.Service) { } } -func ParseEnvVars(envVars []client.EnvVar) models.EnvVars { +func ParseEnvVars(envVars []corev1.EnvVar) models.EnvVars { var parsedEnvVars models.EnvVars for _, envVar := range envVars { - parsedEnvVars = append(parsedEnvVars, models.EnvVar{Name: *envVar.Name, Value: *envVar.Value}) + parsedEnvVars = append(parsedEnvVars, models.EnvVar{Name: envVar.Name, Value: envVar.Value}) } return parsedEnvVars } diff --git a/api/cluster/resource/templater_test.go b/api/cluster/resource/templater_test.go index 6b8a31d27..aed0e2b65 100644 --- a/api/cluster/resource/templater_test.go +++ b/api/cluster/resource/templater_test.go @@ -23,7 +23,6 @@ import ( "testing" "time" - "github.com/caraml-dev/merlin/client" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" kserveconstant "github.com/kserve/kserve/pkg/constants" "github.com/stretchr/testify/assert" @@ -196,16 +195,10 @@ var ( userContainerCPULimitRequestFactor = float64(0) userContainerMemoryLimitRequestFactor = float64(2) - defaultWorkersEnvVarName = "WORKERS" - defaultWorkersEnvVarValue = "2" - defaultEnvVarsWithoutCPULimits = []client.EnvVar{ - { - Name: &defaultWorkersEnvVarName, - Value: &defaultWorkersEnvVarValue, - }, - } + defaultWorkersEnvVarName = "WORKERS" + defaultWorkersEnvVarValue = "2" - expDefaultEnvVarWithoutCPULimits = corev1.EnvVar{ + defaultEnvVarWithoutCPULimits = corev1.EnvVar{ Name: defaultWorkersEnvVarName, Value: defaultWorkersEnvVarValue, } @@ -346,7 +339,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, + Env: []corev1.EnvVar{defaultEnvVarWithoutCPULimits}, }, }, }, @@ -410,7 +403,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, Env: []corev1.EnvVar{ - expDefaultEnvVarWithoutCPULimits, + defaultEnvVarWithoutCPULimits, { Name: "env1", Value: "env1Value", }, @@ -472,7 +465,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, + Env: []corev1.EnvVar{defaultEnvVarWithoutCPULimits}, }, }, }, @@ -528,7 +521,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, + Env: []corev1.EnvVar{defaultEnvVarWithoutCPULimits}, }, }, }, @@ -581,7 +574,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, + Env: []corev1.EnvVar{defaultEnvVarWithoutCPULimits}, }, }, }, @@ -636,7 +629,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, + Env: []corev1.EnvVar{defaultEnvVarWithoutCPULimits}, }, }, }, @@ -691,7 +684,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, + Env: []corev1.EnvVar{defaultEnvVarWithoutCPULimits}, }, }, }, @@ -745,7 +738,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, + Env: []corev1.EnvVar{defaultEnvVarWithoutCPULimits}, }, }, }, @@ -1236,7 +1229,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expUserResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, + Env: []corev1.EnvVar{defaultEnvVarWithoutCPULimits}, }, }, }, @@ -1441,7 +1434,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, + Env: []corev1.EnvVar{defaultEnvVarWithoutCPULimits}, }, }, }, @@ -1525,7 +1518,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, + Env: []corev1.EnvVar{defaultEnvVarWithoutCPULimits}, }, }, }, @@ -1588,7 +1581,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, + Env: []corev1.EnvVar{defaultEnvVarWithoutCPULimits}, }, }, }, @@ -1652,7 +1645,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expUserResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, + Env: []corev1.EnvVar{defaultEnvVarWithoutCPULimits}, }, }, }, @@ -1715,7 +1708,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, + Env: []corev1.EnvVar{defaultEnvVarWithoutCPULimits}, }, }, }, @@ -1800,7 +1793,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, + Env: []corev1.EnvVar{defaultEnvVarWithoutCPULimits}, }, }, }, @@ -1855,7 +1848,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, Ports: grpcServerlessContainerPorts, - Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, + Env: []corev1.EnvVar{defaultEnvVarWithoutCPULimits}, LivenessProbe: probeConfigUPI, }, }, @@ -1977,7 +1970,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, Ports: grpcServerlessContainerPorts, - Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, + Env: []corev1.EnvVar{defaultEnvVarWithoutCPULimits}, LivenessProbe: probeConfigUPI, }, }, @@ -2075,7 +2068,7 @@ func TestCreateInferenceServiceSpec(t *testing.T) { UserContainerCPUDefaultLimit: userContainerCPUDefaultLimit, UserContainerCPULimitRequestFactor: userContainerCPULimitRequestFactor, UserContainerMemoryLimitRequestFactor: userContainerMemoryLimitRequestFactor, - DefaultEnvVarsWithoutCPULimits: defaultEnvVarsWithoutCPULimits, + DefaultEnvVarsWithoutCPULimits: []corev1.EnvVar{defaultEnvVarWithoutCPULimits}, } tpl := NewInferenceServiceTemplater(*deployConfig) @@ -2219,7 +2212,7 @@ func TestCreateInferenceServiceSpecWithTransformer(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, + Env: []corev1.EnvVar{defaultEnvVarWithoutCPULimits}, }, }, }, @@ -2311,7 +2304,7 @@ func TestCreateInferenceServiceSpecWithTransformer(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, + Env: []corev1.EnvVar{defaultEnvVarWithoutCPULimits}, }, }, }, @@ -2403,7 +2396,7 @@ func TestCreateInferenceServiceSpecWithTransformer(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, + Env: []corev1.EnvVar{defaultEnvVarWithoutCPULimits}, }, }, }, @@ -2486,7 +2479,7 @@ func TestCreateInferenceServiceSpecWithTransformer(t *testing.T) { Container: corev1.Container{ Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, - Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, + Env: []corev1.EnvVar{defaultEnvVarWithoutCPULimits}, Ports: grpcServerlessContainerPorts, LivenessProbe: probeConfigUPI, }, @@ -2584,7 +2577,7 @@ func TestCreateInferenceServiceSpecWithTransformer(t *testing.T) { Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, LivenessProbe: probeConfig, - Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, + Env: []corev1.EnvVar{defaultEnvVarWithoutCPULimits}, }, }, }, @@ -2692,7 +2685,7 @@ func TestCreateInferenceServiceSpecWithTransformer(t *testing.T) { Container: corev1.Container{ Name: kserveconstant.InferenceServiceContainerName, Resources: expDefaultModelResourceRequests, - Env: []corev1.EnvVar{expDefaultEnvVarWithoutCPULimits}, + Env: []corev1.EnvVar{defaultEnvVarWithoutCPULimits}, LivenessProbe: probeConfigUPI, Ports: grpcRawContainerPorts, }, @@ -3036,7 +3029,7 @@ func TestCreateInferenceServiceSpecWithTransformer(t *testing.T) { UserContainerCPUDefaultLimit: userContainerCPUDefaultLimit, UserContainerCPULimitRequestFactor: userContainerCPULimitRequestFactor, UserContainerMemoryLimitRequestFactor: userContainerMemoryLimitRequestFactor, - DefaultEnvVarsWithoutCPULimits: defaultEnvVarsWithoutCPULimits, + DefaultEnvVarsWithoutCPULimits: []corev1.EnvVar{defaultEnvVarWithoutCPULimits}, } tpl := NewInferenceServiceTemplater(*deployConfig) diff --git a/api/config/config.go b/api/config/config.go index 02a520cea..6c6996c3b 100644 --- a/api/config/config.go +++ b/api/config/config.go @@ -21,7 +21,6 @@ import ( "strings" "time" - "github.com/caraml-dev/merlin/client" mlpcluster "github.com/caraml-dev/mlp/api/pkg/cluster" "github.com/caraml-dev/mlp/api/pkg/instrumentation/newrelic" "github.com/caraml-dev/mlp/api/pkg/instrumentation/sentry" @@ -361,10 +360,10 @@ type PyFuncPublisherConfig struct { // KServe inference services type InferenceServiceDefaults struct { // TODO: Remove UserContainerCPUDefaultLimit when KServe finally allows default CPU limits to be removed - UserContainerCPUDefaultLimit string `json:"userContainerCPUDefaultLimit" default:"8"` - UserContainerCPULimitRequestFactor float64 `json:"userContainerLimitCPURequestFactor" default:"0"` - UserContainerMemoryLimitRequestFactor float64 `json:"userContainerLimitMemoryRequestFactor" default:"2"` - DefaultEnvVarsWithoutCPULimits []client.EnvVar `json:"defaultEnvVarsWithoutCPULimits"` + UserContainerCPUDefaultLimit string `json:"userContainerCPUDefaultLimit" default:"8"` + UserContainerCPULimitRequestFactor float64 `json:"userContainerLimitCPURequestFactor" default:"0"` + UserContainerMemoryLimitRequestFactor float64 `json:"userContainerLimitMemoryRequestFactor" default:"2"` + DefaultEnvVarsWithoutCPULimits []v1.EnvVar `json:"defaultEnvVarsWithoutCPULimits"` } // SimulationFeastConfig feast config that aimed to be used only for simulation of standard transformer diff --git a/api/config/config_test.go b/api/config/config_test.go index f49a9fdb8..cf494faa4 100644 --- a/api/config/config_test.go +++ b/api/config/config_test.go @@ -20,7 +20,6 @@ import ( "testing" "time" - "github.com/caraml-dev/merlin/client" "github.com/caraml-dev/merlin/pkg/transformer/feast" "github.com/caraml-dev/merlin/pkg/transformer/spec" mlpcluster "github.com/caraml-dev/mlp/api/pkg/cluster" @@ -338,9 +337,6 @@ func TestLoad(t *testing.T) { twoMinutes := 2 * time.Minute oneMinute := 1 * time.Minute - envVarNameFoo := "foo" - envVarValueBar := "bar" - tests := map[string]struct { filepaths []string env map[string]string @@ -589,10 +585,10 @@ func TestLoad(t *testing.T) { UserContainerCPUDefaultLimit: "100", UserContainerCPULimitRequestFactor: 0, UserContainerMemoryLimitRequestFactor: 2, - DefaultEnvVarsWithoutCPULimits: []client.EnvVar{ + DefaultEnvVarsWithoutCPULimits: []v1.EnvVar{ { - Name: &envVarNameFoo, - Value: &envVarValueBar, + Name: "foo", + Value: "bar", }, }, }, diff --git a/api/config/deployment.go b/api/config/deployment.go index 727890dda..209d209d2 100644 --- a/api/config/deployment.go +++ b/api/config/deployment.go @@ -17,7 +17,6 @@ package config import ( "time" - "github.com/caraml-dev/merlin/client" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" ) @@ -56,7 +55,7 @@ type DeploymentConfig struct { UserContainerCPUDefaultLimit string UserContainerCPULimitRequestFactor float64 UserContainerMemoryLimitRequestFactor float64 - DefaultEnvVarsWithoutCPULimits []client.EnvVar + DefaultEnvVarsWithoutCPULimits []corev1.EnvVar } type ResourceRequests struct { From a255f43bd27efa455e419350f95f3b2f551dcfbb Mon Sep 17 00:00:00 2001 From: ewezy Date: Thu, 23 May 2024 18:01:46 +0800 Subject: [PATCH 20/26] Add cpu limit regex check --- ui/src/pages/version/components/forms/validation/schema.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ui/src/pages/version/components/forms/validation/schema.js b/ui/src/pages/version/components/forms/validation/schema.js index 12b14bbb0..ae9b37cc4 100644 --- a/ui/src/pages/version/components/forms/validation/schema.js +++ b/ui/src/pages/version/components/forms/validation/schema.js @@ -10,6 +10,9 @@ const resourceRequestSchema = (maxAllowedReplica) => yup.object().shape({ cpu_request: yup .string() .matches(cpuRequestRegex, 'Valid CPU value is required, e.g "2" or "500m"'), + cpu_limit: yup + .string() + .matches(cpuRequestRegex, { message: 'Valid CPU value is required, e.g "2" or "500m"', excludeEmptyString: true }), memory_request: yup .string() .matches(memRequestRegex, "Valid RAM value is required, e.g. 512Mi"), From ac9cb631f379a02d45ae0f50e20efd1530beb5cd Mon Sep 17 00:00:00 2001 From: ewezy Date: Fri, 24 May 2024 11:07:00 +0800 Subject: [PATCH 21/26] Refactor cpu limit as nullable field --- api/cluster/resource/templater.go | 8 ++++---- api/cluster/resource/templater_test.go | 7 ++++--- api/models/resource_request.go | 2 +- api/queue/work/model_service_deployment_test.go | 3 --- api/service/environment_service_test.go | 14 -------------- 5 files changed, 9 insertions(+), 25 deletions(-) diff --git a/api/cluster/resource/templater.go b/api/cluster/resource/templater.go index ec9c03540..e730c5850 100644 --- a/api/cluster/resource/templater.go +++ b/api/cluster/resource/templater.go @@ -198,7 +198,7 @@ func (t *InferenceServiceTemplater) createPredictorSpec(modelService *models.Ser limits := map[corev1.ResourceName]resource.Quantity{} // Set cpu resource limits automatically if they have not been set - if modelService.ResourceRequest.CPULimit.IsZero() { + if modelService.ResourceRequest.CPULimit == nil || modelService.ResourceRequest.CPULimit.IsZero() { if t.deploymentConfig.UserContainerCPULimitRequestFactor != 0 { limits[corev1.ResourceCPU] = ScaleQuantity( modelService.ResourceRequest.CPURequest, t.deploymentConfig.UserContainerCPULimitRequestFactor, @@ -214,7 +214,7 @@ func (t *InferenceServiceTemplater) createPredictorSpec(modelService *models.Ser envVars = models.MergeEnvVars(ParseEnvVars(t.deploymentConfig.DefaultEnvVarsWithoutCPULimits), envVars) } } else { - limits[corev1.ResourceCPU] = modelService.ResourceRequest.CPULimit + limits[corev1.ResourceCPU] = *modelService.ResourceRequest.CPULimit } if t.deploymentConfig.UserContainerMemoryLimitRequestFactor != 0 { @@ -406,7 +406,7 @@ func (t *InferenceServiceTemplater) createTransformerSpec( // Set resource limits to request * userContainerCPULimitRequestFactor or UserContainerMemoryLimitRequestFactor limits := map[corev1.ResourceName]resource.Quantity{} // Set cpu resource limits automatically if they have not been set - if transformer.ResourceRequest.CPULimit.IsZero() { + if transformer.ResourceRequest.CPULimit == nil || transformer.ResourceRequest.CPULimit.IsZero() { if t.deploymentConfig.UserContainerCPULimitRequestFactor != 0 { limits[corev1.ResourceCPU] = ScaleQuantity( transformer.ResourceRequest.CPURequest, t.deploymentConfig.UserContainerCPULimitRequestFactor, @@ -422,7 +422,7 @@ func (t *InferenceServiceTemplater) createTransformerSpec( envVars = models.MergeEnvVars(ParseEnvVars(t.deploymentConfig.DefaultEnvVarsWithoutCPULimits), envVars) } } else { - limits[corev1.ResourceCPU] = transformer.ResourceRequest.CPULimit + limits[corev1.ResourceCPU] = *transformer.ResourceRequest.CPULimit } if t.deploymentConfig.UserContainerMemoryLimitRequestFactor != 0 { diff --git a/api/cluster/resource/templater_test.go b/api/cluster/resource/templater_test.go index aed0e2b65..3fa170c2e 100644 --- a/api/cluster/resource/templater_test.go +++ b/api/cluster/resource/templater_test.go @@ -93,12 +93,13 @@ var ( MemoryRequest: resource.MustParse("1Gi"), } + cpuLimit = resource.MustParse("8") userResourceRequestsWithCPULimits = &models.ResourceRequest{ MinReplica: 1, MaxReplica: 10, CPURequest: resource.MustParse("1"), MemoryRequest: resource.MustParse("1Gi"), - CPULimit: resource.MustParse("8"), + CPULimit: &cpuLimit, } expUserResourceRequests = corev1.ResourceRequirements{ @@ -118,7 +119,7 @@ var ( corev1.ResourceMemory: userResourceRequestsWithCPULimits.MemoryRequest, }, Limits: corev1.ResourceList{ - corev1.ResourceCPU: userResourceRequestsWithCPULimits.CPULimit, + corev1.ResourceCPU: *userResourceRequestsWithCPULimits.CPULimit, corev1.ResourceMemory: ScaleQuantity(userResourceRequestsWithCPULimits.MemoryRequest, 2), }, } @@ -4627,7 +4628,7 @@ func TestCreateTransformerSpec(t *testing.T) { MaxReplica: 1, CPURequest: cpuRequest, MemoryRequest: memoryRequest, - CPULimit: customCPULimit, + CPULimit: &customCPULimit, }, EnvVars: models.EnvVars{ {Name: transformerpkg.JaegerCollectorURL, Value: "NEW_HOST"}, // test user overwrite diff --git a/api/models/resource_request.go b/api/models/resource_request.go index 803301b2f..f92fae9f8 100644 --- a/api/models/resource_request.go +++ b/api/models/resource_request.go @@ -30,7 +30,7 @@ type ResourceRequest struct { // CPU request of inference service CPURequest resource.Quantity `json:"cpu_request"` // CPU limit of inference service - CPULimit resource.Quantity `json:"cpu_limit"` + CPULimit *resource.Quantity `json:"cpu_limit,omitempty"` // Memory request of inference service MemoryRequest resource.Quantity `json:"memory_request"` // GPU name diff --git a/api/queue/work/model_service_deployment_test.go b/api/queue/work/model_service_deployment_test.go index 1efd73980..403858322 100644 --- a/api/queue/work/model_service_deployment_test.go +++ b/api/queue/work/model_service_deployment_test.go @@ -34,7 +34,6 @@ func TestExecuteDeployment(t *testing.T) { MinReplica: 0, MaxReplica: 1, CPURequest: resource.MustParse("1"), - CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), GPURequest: resource.MustParse("0"), }, @@ -535,7 +534,6 @@ func TestExecuteDeployment(t *testing.T) { MinReplica: 0, MaxReplica: 1, CPURequest: resource.MustParse("1"), - CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), GPUName: "NVIDIA P4", GPURequest: resource.MustParse("1"), @@ -743,7 +741,6 @@ func TestExecuteRedeployment(t *testing.T) { MinReplica: 0, MaxReplica: 1, CPURequest: resource.MustParse("1"), - CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), GPURequest: resource.MustParse("0"), }, diff --git a/api/service/environment_service_test.go b/api/service/environment_service_test.go index 77615a624..46db6da95 100644 --- a/api/service/environment_service_test.go +++ b/api/service/environment_service_test.go @@ -197,7 +197,6 @@ func TestGetEnvironment(t *testing.T) { MinReplica: 1, MaxReplica: 4, CPURequest: resource.MustParse("1"), - CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), GPURequest: resource.MustParse("0"), }, @@ -224,7 +223,6 @@ func TestGetEnvironment(t *testing.T) { MinReplica: 1, MaxReplica: 4, CPURequest: resource.MustParse("1"), - CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), GPURequest: resource.MustParse("0"), }, @@ -255,7 +253,6 @@ func TestGetEnvironment(t *testing.T) { MinReplica: 1, MaxReplica: 4, CPURequest: resource.MustParse("1"), - CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), }, CreatedUpdated: models.CreatedUpdated{ @@ -323,7 +320,6 @@ func TestGetDefaultEnvironment(t *testing.T) { MinReplica: 1, MaxReplica: 4, CPURequest: resource.MustParse("1"), - CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), GPURequest: resource.MustParse("0"), }, @@ -350,7 +346,6 @@ func TestGetDefaultEnvironment(t *testing.T) { MinReplica: 1, MaxReplica: 4, CPURequest: resource.MustParse("1"), - CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), GPURequest: resource.MustParse("0"), }, @@ -381,7 +376,6 @@ func TestGetDefaultEnvironment(t *testing.T) { MinReplica: 1, MaxReplica: 4, CPURequest: resource.MustParse("1"), - CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), }, CreatedUpdated: models.CreatedUpdated{ @@ -449,7 +443,6 @@ func TestGetDefaultPredictionJobEnvironment(t *testing.T) { MinReplica: 1, MaxReplica: 4, CPURequest: resource.MustParse("1"), - CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), GPURequest: resource.MustParse("0"), }, @@ -476,7 +469,6 @@ func TestGetDefaultPredictionJobEnvironment(t *testing.T) { MinReplica: 1, MaxReplica: 4, CPURequest: resource.MustParse("1"), - CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), GPURequest: resource.MustParse("0"), }, @@ -507,7 +499,6 @@ func TestGetDefaultPredictionJobEnvironment(t *testing.T) { MinReplica: 1, MaxReplica: 4, CPURequest: resource.MustParse("1"), - CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), }, CreatedUpdated: models.CreatedUpdated{ @@ -576,7 +567,6 @@ func TestListEnvironment(t *testing.T) { MinReplica: 1, MaxReplica: 4, CPURequest: resource.MustParse("1"), - CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), GPURequest: resource.MustParse("0"), }, @@ -604,7 +594,6 @@ func TestListEnvironment(t *testing.T) { MinReplica: 1, MaxReplica: 4, CPURequest: resource.MustParse("1"), - CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), GPURequest: resource.MustParse("0"), }, @@ -638,7 +627,6 @@ func TestListEnvironment(t *testing.T) { MinReplica: 1, MaxReplica: 4, CPURequest: resource.MustParse("1"), - CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), GPURequest: resource.MustParse("0"), }, @@ -666,7 +654,6 @@ func TestListEnvironment(t *testing.T) { MinReplica: 1, MaxReplica: 4, CPURequest: resource.MustParse("1"), - CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), GPURequest: resource.MustParse("0"), }, @@ -700,7 +687,6 @@ func TestListEnvironment(t *testing.T) { MinReplica: 1, MaxReplica: 4, CPURequest: resource.MustParse("1"), - CPULimit: resource.MustParse("0"), MemoryRequest: resource.MustParse("1Gi"), }, CreatedUpdated: models.CreatedUpdated{ From 54b2a2d60c0e0d6607878b07290877da14c8b40b Mon Sep 17 00:00:00 2001 From: ewezy Date: Fri, 24 May 2024 13:40:21 +0800 Subject: [PATCH 22/26] Add cpu limits form group to transformer step and cleanup code --- ui/src/components/ResourcesConfigTable.js | 13 ++++---- .../forms/DeployModelVersionForm.js | 1 - .../forms/components/CPULimitsFormGroup.js | 2 +- .../forms/components/ImageBuilderSection.js | 3 +- .../components/forms/steps/ModelStep.js | 30 +++++++++---------- .../components/forms/steps/TransformerStep.js | 15 +++++++++- ui/src/services/transformer/Transformer.js | 4 +++ .../version_endpoint/VersionEndpoint.js | 2 +- 8 files changed, 43 insertions(+), 27 deletions(-) diff --git a/ui/src/components/ResourcesConfigTable.js b/ui/src/components/ResourcesConfigTable.js index d0bee6412..5332534bc 100644 --- a/ui/src/components/ResourcesConfigTable.js +++ b/ui/src/components/ResourcesConfigTable.js @@ -34,6 +34,12 @@ export const ResourcesConfigTable = ({ title: "CPU Request", description: cpu_request, }, + ...(cpu_limit !== undefined && cpu_limit !== "0" && cpu_limit !== "") ? [ + { + title: "CPU Limit", + description: cpu_limit, + } + ] : [], { title: "Memory Request", description: memory_request, @@ -48,13 +54,6 @@ export const ResourcesConfigTable = ({ }, ]; - if (cpu_limit !== "0") { - items.push({ - title: "CPU Limit", - description: cpu_limit, - }); - } - if (gpu_name !== undefined && gpu_name !== "") { items.push({ title: "GPU Name", diff --git a/ui/src/pages/version/components/forms/DeployModelVersionForm.js b/ui/src/pages/version/components/forms/DeployModelVersionForm.js index b2205db26..8baa0803c 100644 --- a/ui/src/pages/version/components/forms/DeployModelVersionForm.js +++ b/ui/src/pages/version/components/forms/DeployModelVersionForm.js @@ -54,7 +54,6 @@ export const DeployModelVersionForm = ({ }, [submissionResponse, onSuccess, model, version]); const onSubmit = () => { - // versionEndpoint toJSON() is not invoked, binding that causes many issues if (versionEndpoint?.resource_request?.cpu_limit === "") { delete versionEndpoint.resource_request.cpu_limit; diff --git a/ui/src/pages/version/components/forms/components/CPULimitsFormGroup.js b/ui/src/pages/version/components/forms/components/CPULimitsFormGroup.js index a4a0bd1ff..df9a12187 100644 --- a/ui/src/pages/version/components/forms/components/CPULimitsFormGroup.js +++ b/ui/src/pages/version/components/forms/components/CPULimitsFormGroup.js @@ -25,7 +25,7 @@ export const CPULimitsFormGroup = ({ } isInvalid={!!errors.cpu_limit} diff --git a/ui/src/pages/version/components/forms/components/ImageBuilderSection.js b/ui/src/pages/version/components/forms/components/ImageBuilderSection.js index 406ddf73a..6267ada7f 100644 --- a/ui/src/pages/version/components/forms/components/ImageBuilderSection.js +++ b/ui/src/pages/version/components/forms/components/ImageBuilderSection.js @@ -3,7 +3,8 @@ import { EuiFieldText, EuiFlexGroup, EuiFlexItem, - EuiFormRow, EuiTitle, + EuiFormRow, + EuiTitle, } from "@elastic/eui"; diff --git a/ui/src/pages/version/components/forms/steps/ModelStep.js b/ui/src/pages/version/components/forms/steps/ModelStep.js index e33971751..65fb0bf87 100644 --- a/ui/src/pages/version/components/forms/steps/ModelStep.js +++ b/ui/src/pages/version/components/forms/steps/ModelStep.js @@ -42,21 +42,21 @@ export const ModelStep = ({ version, isEnvironmentDisabled = false, maxAllowedRe maxAllowedReplica={maxAllowedReplica} errors={get(errors, "resource_request")} child={ - - - - - + + + + + } /> diff --git a/ui/src/pages/version/components/forms/steps/TransformerStep.js b/ui/src/pages/version/components/forms/steps/TransformerStep.js index 357e3bd3f..c46a300da 100644 --- a/ui/src/pages/version/components/forms/steps/TransformerStep.js +++ b/ui/src/pages/version/components/forms/steps/TransformerStep.js @@ -4,13 +4,14 @@ import { get, useOnChangeHandler, } from "@caraml-dev/ui-lib"; -import { EuiFlexGroup, EuiFlexItem } from "@elastic/eui"; +import { EuiAccordion, EuiFlexGroup, EuiFlexItem, EuiSpacer } from "@elastic/eui"; import React, { useContext } from "react"; import { PROTOCOL } from "../../../../../services/version_endpoint/VersionEndpoint"; import { EnvVariablesPanel } from "../components/EnvVariablesPanel"; import { LoggerPanel } from "../components/LoggerPanel"; import { ResourcesPanel } from "../components/ResourcesPanel"; import { SelectTransformerPanel } from "../components/SelectTransformerPanel"; +import { CPULimitsFormGroup } from "../components/CPULimitsFormGroup"; export const TransformerStep = ({ maxAllowedReplica }) => { const { @@ -40,6 +41,18 @@ export const TransformerStep = ({ maxAllowedReplica }) => { onChangeHandler={onChange("transformer.resource_request")} maxAllowedReplica={maxAllowedReplica} errors={get(errors, "transformer.resource_request")} + child={ + + + + + } /> {protocol !== PROTOCOL.UPI_V1 && ( diff --git a/ui/src/services/transformer/Transformer.js b/ui/src/services/transformer/Transformer.js index 03f441b53..6ba4eef62 100644 --- a/ui/src/services/transformer/Transformer.js +++ b/ui/src/services/transformer/Transformer.js @@ -22,6 +22,7 @@ export class Transformer { min_replica: process.env.REACT_APP_ENVIRONMENT === "production" ? 2 : 0, max_replica: process.env.REACT_APP_ENVIRONMENT === "production" ? 4 : 2, cpu_request: "500m", + cpu_limit: "", memory_request: "512Mi" }; @@ -89,6 +90,9 @@ export class Transformer { if (obj.config) { delete obj["config"]; } + if (obj.resource_request?.cpu_limit === "") { + delete obj.resource_request.cpu_limit; + } return obj; } diff --git a/ui/src/services/version_endpoint/VersionEndpoint.js b/ui/src/services/version_endpoint/VersionEndpoint.js index 4eccd3d1b..dc001a45f 100644 --- a/ui/src/services/version_endpoint/VersionEndpoint.js +++ b/ui/src/services/version_endpoint/VersionEndpoint.js @@ -26,7 +26,7 @@ export class VersionEndpoint { min_replica: process.env.REACT_APP_ENVIRONMENT === "production" ? 2 : 0, max_replica: process.env.REACT_APP_ENVIRONMENT === "production" ? 4 : 2, cpu_request: "500m", - cpu_limit: "0", + cpu_limit: "", memory_request: "512Mi", }; From 4f3e5692552a4836df7cb7ee427f1a7cf33b2d2f Mon Sep 17 00:00:00 2001 From: ewezy Date: Fri, 24 May 2024 15:51:17 +0800 Subject: [PATCH 23/26] Rename cpu limit section --- .../version/components/forms/components/CPULimitsFormGroup.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/src/pages/version/components/forms/components/CPULimitsFormGroup.js b/ui/src/pages/version/components/forms/components/CPULimitsFormGroup.js index df9a12187..47596e9b7 100644 --- a/ui/src/pages/version/components/forms/components/CPULimitsFormGroup.js +++ b/ui/src/pages/version/components/forms/components/CPULimitsFormGroup.js @@ -12,7 +12,7 @@ export const CPULimitsFormGroup = ({ return ( CPU Limits

} + title={

CPU Limit

} description={ Use this field to override the platform-level default CPU limit. @@ -23,7 +23,7 @@ export const CPULimitsFormGroup = ({ From 4d817ba0dc51c15b2734c720bca83e8348c6b50f Mon Sep 17 00:00:00 2001 From: ewezy Date: Mon, 27 May 2024 13:40:28 +0800 Subject: [PATCH 24/26] Add codecov config file that adds a threshold to allow random code coverage decreases --- .github/workflows/codecov-config/codecov.yml | 5 +++++ .github/workflows/merlin.yml | 2 ++ 2 files changed, 7 insertions(+) create mode 100644 .github/workflows/codecov-config/codecov.yml diff --git a/.github/workflows/codecov-config/codecov.yml b/.github/workflows/codecov-config/codecov.yml new file mode 100644 index 000000000..8901728c4 --- /dev/null +++ b/.github/workflows/codecov-config/codecov.yml @@ -0,0 +1,5 @@ +coverage: + status: + patch: + default: + threshold: 0.03% \ No newline at end of file diff --git a/.github/workflows/merlin.yml b/.github/workflows/merlin.yml index e943bceeb..15bce76b4 100644 --- a/.github/workflows/merlin.yml +++ b/.github/workflows/merlin.yml @@ -141,6 +141,7 @@ jobs: name: sdk-test-${{ matrix.python-version }} token: ${{ secrets.CODECOV_TOKEN }} working-directory: ./python/sdk + codecov_yml_path: ../../.github/workflows/codecov-config/codecov.yml lint-api: runs-on: ubuntu-latest @@ -197,6 +198,7 @@ jobs: name: api-test token: ${{ secrets.CODECOV_TOKEN }} working-directory: ./api + codecov_yml_path: ../.github/workflows/codecov-config/codecov.yml test-observation-publisher: runs-on: ubuntu-latest From 251cb67e23588deca7283b894a8dcf6692d961ae Mon Sep 17 00:00:00 2001 From: ewezy Date: Fri, 31 May 2024 18:10:41 +0800 Subject: [PATCH 25/26] Refactor code to determine cpu limit into single helper function --- api/cluster/resource/templater.go | 91 ++++++++++++++----------------- 1 file changed, 40 insertions(+), 51 deletions(-) diff --git a/api/cluster/resource/templater.go b/api/cluster/resource/templater.go index e730c5850..de4382b1f 100644 --- a/api/cluster/resource/templater.go +++ b/api/cluster/resource/templater.go @@ -192,29 +192,9 @@ func (t *InferenceServiceTemplater) CreateInferenceServiceSpec(modelService *mod } func (t *InferenceServiceTemplater) createPredictorSpec(modelService *models.Service) (kservev1beta1.PredictorSpec, error) { - envVars := modelService.EnvVars - - // Set resource limits to request * userContainerCPULimitRequestFactor or userContainerMemoryLimitRequestFactor - limits := map[corev1.ResourceName]resource.Quantity{} - - // Set cpu resource limits automatically if they have not been set - if modelService.ResourceRequest.CPULimit == nil || modelService.ResourceRequest.CPULimit.IsZero() { - if t.deploymentConfig.UserContainerCPULimitRequestFactor != 0 { - limits[corev1.ResourceCPU] = ScaleQuantity( - modelService.ResourceRequest.CPURequest, t.deploymentConfig.UserContainerCPULimitRequestFactor, - ) - } else { - // TODO: Remove this else-block when KServe finally allows default CPU limits to be removed - var err error - limits[corev1.ResourceCPU], err = resource.ParseQuantity(t.deploymentConfig.UserContainerCPUDefaultLimit) - if err != nil { - return kservev1beta1.PredictorSpec{}, err - } - // Set additional env vars to manage concurrency so model performance improves when no CPU limits are set - envVars = models.MergeEnvVars(ParseEnvVars(t.deploymentConfig.DefaultEnvVarsWithoutCPULimits), envVars) - } - } else { - limits[corev1.ResourceCPU] = *modelService.ResourceRequest.CPULimit + limits, envVars, err := t.getResourceLimitsAndEnvVars(modelService.ResourceRequest, modelService.EnvVars) + if err != nil { + return kservev1beta1.PredictorSpec{}, err } if t.deploymentConfig.UserContainerMemoryLimitRequestFactor != 0 { @@ -401,34 +381,9 @@ func (t *InferenceServiceTemplater) createTransformerSpec( modelService *models.Service, transformer *models.Transformer, ) (*kservev1beta1.TransformerSpec, error) { - envVars := transformer.EnvVars - - // Set resource limits to request * userContainerCPULimitRequestFactor or UserContainerMemoryLimitRequestFactor - limits := map[corev1.ResourceName]resource.Quantity{} - // Set cpu resource limits automatically if they have not been set - if transformer.ResourceRequest.CPULimit == nil || transformer.ResourceRequest.CPULimit.IsZero() { - if t.deploymentConfig.UserContainerCPULimitRequestFactor != 0 { - limits[corev1.ResourceCPU] = ScaleQuantity( - transformer.ResourceRequest.CPURequest, t.deploymentConfig.UserContainerCPULimitRequestFactor, - ) - } else { - // TODO: Remove this else-block when KServe finally allows default CPU limits to be removed - var err error - limits[corev1.ResourceCPU], err = resource.ParseQuantity(t.deploymentConfig.UserContainerCPUDefaultLimit) - if err != nil { - return nil, err - } - // Set additional env vars to manage concurrency so model performance improves when no CPU limits are set - envVars = models.MergeEnvVars(ParseEnvVars(t.deploymentConfig.DefaultEnvVarsWithoutCPULimits), envVars) - } - } else { - limits[corev1.ResourceCPU] = *transformer.ResourceRequest.CPULimit - } - - if t.deploymentConfig.UserContainerMemoryLimitRequestFactor != 0 { - limits[corev1.ResourceMemory] = ScaleQuantity( - transformer.ResourceRequest.MemoryRequest, t.deploymentConfig.UserContainerMemoryLimitRequestFactor, - ) + limits, envVars, err := t.getResourceLimitsAndEnvVars(transformer.ResourceRequest, transformer.EnvVars) + if err != nil { + return nil, err } // Put in defaults if not provided by users (user's input is used) @@ -932,6 +887,40 @@ func (t *InferenceServiceTemplater) applyDefaults(service *models.Service) { } } +func (t *InferenceServiceTemplater) getResourceLimitsAndEnvVars( + resourceRequest *models.ResourceRequest, + envVars models.EnvVars, +) (map[corev1.ResourceName]resource.Quantity, models.EnvVars, error) { + // Set resource limits to request * userContainerCPULimitRequestFactor or UserContainerMemoryLimitRequestFactor + limits := map[corev1.ResourceName]resource.Quantity{} + // Set cpu resource limits automatically if they have not been set + if resourceRequest.CPULimit == nil || resourceRequest.CPULimit.IsZero() { + if t.deploymentConfig.UserContainerCPULimitRequestFactor != 0 { + limits[corev1.ResourceCPU] = ScaleQuantity( + resourceRequest.CPURequest, t.deploymentConfig.UserContainerCPULimitRequestFactor, + ) + } else { + // TODO: Remove this else-block when KServe finally allows default CPU limits to be removed + var err error + limits[corev1.ResourceCPU], err = resource.ParseQuantity(t.deploymentConfig.UserContainerCPUDefaultLimit) + if err != nil { + return nil, nil, err + } + // Set additional env vars to manage concurrency so model performance improves when no CPU limits are set + envVars = models.MergeEnvVars(ParseEnvVars(t.deploymentConfig.DefaultEnvVarsWithoutCPULimits), envVars) + } + } else { + limits[corev1.ResourceCPU] = *resourceRequest.CPULimit + } + + if t.deploymentConfig.UserContainerMemoryLimitRequestFactor != 0 { + limits[corev1.ResourceMemory] = ScaleQuantity( + resourceRequest.MemoryRequest, t.deploymentConfig.UserContainerMemoryLimitRequestFactor, + ) + } + return limits, envVars, nil +} + func ParseEnvVars(envVars []corev1.EnvVar) models.EnvVars { var parsedEnvVars models.EnvVars for _, envVar := range envVars { From 6f1f13b3b9d128c8cc1277a24599a8769abab758 Mon Sep 17 00:00:00 2001 From: ewezy Date: Fri, 31 May 2024 18:18:15 +0800 Subject: [PATCH 26/26] Simplify cpu limit description --- .../version/components/forms/components/CPULimitsFormGroup.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/pages/version/components/forms/components/CPULimitsFormGroup.js b/ui/src/pages/version/components/forms/components/CPULimitsFormGroup.js index 47596e9b7..62e5f65fd 100644 --- a/ui/src/pages/version/components/forms/components/CPULimitsFormGroup.js +++ b/ui/src/pages/version/components/forms/components/CPULimitsFormGroup.js @@ -15,7 +15,7 @@ export const CPULimitsFormGroup = ({ title={

CPU Limit

} description={ - Use this field to override the platform-level default CPU limit. + Overrides the platform-level default CPU limit. } fullWidth