From 052a6468be228ac90a4faf7b8a4b11d1055f8812 Mon Sep 17 00:00:00 2001 From: shil Date: Sat, 21 Dec 2024 08:26:58 -0800 Subject: [PATCH] chore: temp work for datastorev2 --- api/apps/v1alpha1/nemo_datastorev2_types.go | 983 ++++++++ api/apps/v1alpha1/zz_generated.deepcopy.go | 189 ++ .../typed/apps/v1alpha1/apps_client.go | 5 + .../apps/v1alpha1/fake/fake_apps_client.go | 4 + .../v1alpha1/fake/fake_nemodatastorev2.go | 146 ++ .../apps/v1alpha1/generated_expansion.go | 2 + .../typed/apps/v1alpha1/nemodatastorev2.go | 68 + .../apps.nvidia.com_nemodatastorev2s.yaml | 2236 +++++++++++++++++ cmd/main.go | 11 + .../apps.nvidia.com_nemodatastorev2s.yaml | 2236 +++++++++++++++++ config/rbac/role.yaml | 3 + .../apps.nvidia.com_nemodatastorev2s.yaml | 2236 +++++++++++++++++ .../controller/nemo_datastorev2_controller.go | 642 +++++ manifests/deployment.yaml | 5 + 14 files changed, 8766 insertions(+) create mode 100644 api/apps/v1alpha1/nemo_datastorev2_types.go create mode 100644 api/versioned/typed/apps/v1alpha1/fake/fake_nemodatastorev2.go create mode 100644 api/versioned/typed/apps/v1alpha1/nemodatastorev2.go create mode 100644 bundle/manifests/apps.nvidia.com_nemodatastorev2s.yaml create mode 100644 config/crd/bases/apps.nvidia.com_nemodatastorev2s.yaml create mode 100644 deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemodatastorev2s.yaml create mode 100644 internal/controller/nemo_datastorev2_controller.go diff --git a/api/apps/v1alpha1/nemo_datastorev2_types.go b/api/apps/v1alpha1/nemo_datastorev2_types.go new file mode 100644 index 00000000..2d8ea736 --- /dev/null +++ b/api/apps/v1alpha1/nemo_datastorev2_types.go @@ -0,0 +1,983 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "fmt" + "maps" + "os" + + rendertypes "github.com/NVIDIA/k8s-nim-operator/internal/render/types" + utils "github.com/NVIDIA/k8s-nim-operator/internal/utils" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +const ( + // NemoDatastoreV2ConditionReady indicates that the NEMO datastore service is ready. + NemoDatastoreV2ConditionReady = "Ready" + // NemoDatastoreV2ConditionFailed indicates that the NEMO datastore service has failed. + NemoDatastoreV2ConditionFailed = "Failed" + + // NemoDatastoreV2StatusPending indicates that NEMO datastore service is in pending state + NemoDatastoreV2StatusPending = "Pending" + // NemoDatastoreV2StatusNotReady indicates that NEMO datastore service is not ready + NemoDatastoreV2StatusNotReady = "NotReady" + // NemoDatastoreV2StatusReady indicates that NEMO datastore service is ready + NemoDatastoreV2StatusReady = "Ready" + // NemoDatastoreV2StatusFailed indicates that NEMO datastore service has failed + NemoDatastoreV2StatusFailed = "Failed" +) + +// NemoDatastoreV2Spec defines the desired state of NemoDatastoreV2 +type NemoDatastoreV2Spec struct { + Image Image `json:"image,omitempty"` + Command []string `json:"command,omitempty"` + Args []string `json:"args,omitempty"` + Env []corev1.EnvVar `json:"env,omitempty"` + // The name of an secret that contains authn for the NGC NIM service API + AuthSecret string `json:"authSecret"` + Labels map[string]string `json:"labels,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + PodAffinity *corev1.PodAffinity `json:"podAffinity,omitempty"` + Resources *corev1.ResourceRequirements `json:"resources,omitempty"` + Expose Expose `json:"expose,omitempty"` + LivenessProbe Probe `json:"livenessProbe,omitempty"` + ReadinessProbe Probe `json:"readinessProbe,omitempty"` + StartupProbe Probe `json:"startupProbe,omitempty"` + Scale Autoscaling `json:"scale,omitempty"` + Metrics Metrics `json:"metrics,omitempty"` + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:default:=1 + Replicas int `json:"replicas,omitempty"` + UserID *int64 `json:"userID,omitempty"` + GroupID *int64 `json:"groupID,omitempty"` + RuntimeClass string `json:"runtimeClass,omitempty"` + + DataStoreParams NemoDatastoreV2Params `json:"dataStoreParams"` +} + +// NemoDatastoreV2Status defines the observed state of NemoDatastoreV2 +type NemoDatastoreV2Status struct { + Conditions []metav1.Condition `json:"conditions,omitempty"` + AvailableReplicas int32 `json:"availableReplicas,omitempty"` + State string `json:"state,omitempty"` +} + +type NemoDatastoreV2Params struct { + AppVersion string `json:"appVersion"` + DatabaseURL string `json:"databaseURL"` + DatabaseHost string `json:"databaseHost"` + DatabasePort string `json:"databasePort"` + DBSecret string `json:"dbSecret"` + GiteaAdminSecret string `json:"giteaAdminSecret"` + + ObjectStoreSecret string `json:"objStoreSecret"` + DataStoreSettingsSecret string `json:"datastoreSettingsSecret"` + LfsJwtSecret string `json:"lfsJwtSecret"` + + DataStoreInitSecret string `json:"datastoreInitSecret"` + DataStoreConfigSecret string `json:"datastoreConfigSecret"` + DataStoreInlineConfigSecret string `json:"datastoreInlineConfigSecret"` + + EnvConfigMap string `json:"envConfigmap"` + EnvSecret string `json:"envSecret"` + + InitContainerImage string `json:"initContainerImage,omitempty"` + InitContainerCommand []string `json:"initContainerCommand,omitempty"` + + SshEnabled bool `json:"sshEnabled"` + + PVCSharedData string `json:"pvcSharedData"` + StorageClass string `json:"storageClass"` + Size string `json:"size,omitempty"` +} + +// +genclient +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.state`,priority=0 +// +kubebuilder:printcolumn:name="Age",type="date",format="date-time",JSONPath=".metadata.creationTimestamp",priority=0 + +// NemoDatastoreV2 is the Schema for the NemoDatastoreV2 API +type NemoDatastoreV2 struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec NemoDatastoreV2Spec `json:"spec,omitempty"` + Status NemoDatastoreV2Status `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// NemoDatastoreV2List contains a list of NemoDatastoreV2 +type NemoDatastoreV2List struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []NemoDatastoreV2 `json:"items"` +} + +// GetPVCName returns the name to be used for the PVC based on the custom spec +// Prefers pvc.Name if explicitly set by the user in the NemoDatastoreV2 instance +func (n *NemoDatastoreV2) GetPVCName(pvc PersistentVolumeClaim) string { + pvcName := fmt.Sprintf("%s-pvc", n.GetName()) + if pvc.Name != "" { + pvcName = pvc.Name + } + return pvcName +} + +// GetStandardSelectorLabels returns the standard selector labels for the NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) GetStandardSelectorLabels() map[string]string { + return map[string]string{ + "app.kubernetes.io/name": n.Name, + "app.kubernetes.io/instance": n.Name, + } +} + +// GetStandardLabels returns the standard set of labels for NemoDatastoreV2 resources +func (n *NemoDatastoreV2) GetStandardLabels() map[string]string { + return map[string]string{ + "app.kubernetes.io/name": n.Name, + "app.kubernetes.io/instance": n.Name, + "app.kubernetes.io/operator-version": os.Getenv("OPERATOR_VERSION"), + "app.kubernetes.io/part-of": "nemo-datastore-service", + "app.kubernetes.io/managed-by": "k8s-nim-operator", + } +} + +// GetMainContainerEnv returns the standard set of env variables for the NemoDatastoreV2 main container +func (n *NemoDatastoreV2) GetStandardEnv() []corev1.EnvVar { + // add standard env required for NIM service + envVars := []corev1.EnvVar{ + { + Name: "SSH_LISTEN_PORT", + Value: "2222", + }, + { + Name: "SSH_PORT", + Value: "22", + }, + { + Name: "GITEA_APP_INI", + Value: "/data/gitea/conf/app.ini", + }, + { + Name: "GITEA_CUSTOM", + Value: "/data/gitea", + }, + { + Name: "GITEA_WORK_DIR", + Value: "/data", + }, + { + Name: "TMPDIR", + Value: "/tmp/gitea", + }, + { + Name: "GITEA_TEMP", + Value: "/tmp/gitea", + }, + { + Name: "HOME", + Value: "/data/gitea/git", + }, + { + Name: "GITEA__DATABASE__PASSWD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "password", + LocalObjectReference: corev1.LocalObjectReference{ + Name: n.Spec.DataStoreParams.DBSecret, + }, + }, + }, + }, + } + return envVars +} + +func (n *NemoDatastoreV2) GetInitContainerEnv() []corev1.EnvVar { + envVars := []corev1.EnvVar{ + { + Name: "GITEA_APP_INI", + Value: "/data/gitea/conf/app.ini", + }, + { + Name: "GITEA_CUSTOM", + Value: "/data/gitea", + }, + { + Name: "GITEA_WORK_DIR", + Value: "/data", + }, + { + Name: "TMPDIR", + Value: "/tmp/gitea", + }, + { + Name: "GITEA_TEMP", + Value: "/tmp/gitea", + }, + + { + Name: "HOME", + Value: "/data/gitea/git", + }, + { + Name: "GITEA__DATABASE__PASSWD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "password", + LocalObjectReference: corev1.LocalObjectReference{ + Name: n.Spec.DataStoreParams.DBSecret, + }, + }, + }, + }, + { + Name: "GITEA_ADMIN_USERNAME", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "username", + LocalObjectReference: corev1.LocalObjectReference{ + Name: n.Spec.DataStoreParams.GiteaAdminSecret, + }, + }, + }, + }, + { + Name: "GITEA_ADMIN_PASSWORD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "password", + LocalObjectReference: corev1.LocalObjectReference{ + Name: n.Spec.DataStoreParams.GiteaAdminSecret, + }, + }, + }, + }, + } + return envVars +} + +// GetVolumes returns volumes for the NemoDatastoreV2 container +func (n *NemoDatastoreV2) GetVolumes() []corev1.Volume { + /*volumes: + - name: init + secret: + defaultMode: 110 + secretName: datastore-v2-nemo-datastore-init + - name: config + secret: + defaultMode: 110 + secretName: datastore-v2-nemo-datastore + - name: inline-config-sources + secret: + defaultMode: 420 + secretName: datastore-v2-nemo-datastore-inline-config + - emptyDir: {} + name: temp + - name: data + persistentVolumeClaim: + claimName: datastore-shared-storage + */ + var initMode = int32(110) + var configMode = int32(420) + + volumes := []corev1.Volume{ + { + Name: "init", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: n.Spec.DataStoreParams.DataStoreInitSecret, + DefaultMode: &initMode, + }, + }, + }, + { + Name: "config", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: n.Spec.DataStoreParams.DataStoreConfigSecret, + DefaultMode: &initMode, + }, + }, + }, + { + Name: "inline-config-sources", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: n.Spec.DataStoreParams.DataStoreInlineConfigSecret, + DefaultMode: &configMode, + }, + }, + }, + { + Name: "temp", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "data", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: n.Spec.DataStoreParams.PVCSharedData, + }, + }, + }, + } + return volumes +} + +// GetStandardAnnotations returns default annotations to apply to the NemoDatastoreV2 instance +func (n *NemoDatastoreV2) GetEnvFrom() []corev1.EnvFromSource { + return []corev1.EnvFromSource{} +} + +// GetStandardAnnotations returns default annotations to apply to the NemoDatastoreV2 instance +func (n *NemoDatastoreV2) GetInitAppIniEnvFrom() []corev1.EnvFromSource { + return []corev1.EnvFromSource{ + { + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: n.Spec.DataStoreParams.DataStoreSettingsSecret, + }, + }, + }, + } +} + +// GetStandardAnnotations returns default annotations to apply to the NemoDatastoreV2 instance +func (n *NemoDatastoreV2) GetStandardAnnotations() map[string]string { + standardAnnotations := map[string]string{ + "openshift.io/scc": "nonroot", + } + return standardAnnotations +} + +// GetNemoDatastoreV2Annotations returns annotations to apply to the NemoDatastoreV2 instance +func (n *NemoDatastoreV2) GetNemoDatastoreV2Annotations() map[string]string { + standardAnnotations := n.GetStandardAnnotations() + + if n.Spec.Annotations != nil { + return utils.MergeMaps(standardAnnotations, n.Spec.Annotations) + } + + return standardAnnotations +} + +// GetServiceLabels returns merged labels to apply to the NemoDatastoreV2 instance +func (n *NemoDatastoreV2) GetServiceLabels() map[string]string { + standardLabels := n.GetStandardLabels() + + if n.Spec.Labels != nil { + return utils.MergeMaps(standardLabels, n.Spec.Labels) + } + return standardLabels +} + +// GetSelectorLabels returns standard selector labels to apply to the NemoDatastoreV2 instance +func (n *NemoDatastoreV2) GetSelectorLabels() map[string]string { + // TODO: add custom ones + return n.GetStandardSelectorLabels() +} + +// GetNodeSelector returns node selector labels for the NemoDatastoreV2 instance +func (n *NemoDatastoreV2) GetNodeSelector() map[string]string { + return n.Spec.NodeSelector +} + +// GetTolerations returns tolerations for the NemoDatastoreV2 instance +func (n *NemoDatastoreV2) GetTolerations() []corev1.Toleration { + return n.Spec.Tolerations +} + +// GetPodAffinity returns pod affinity for the NemoDatastoreV2 instance +func (n *NemoDatastoreV2) GetPodAffinity() *corev1.PodAffinity { + return n.Spec.PodAffinity +} + +// GetContainerName returns name of the container for NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) GetContainerName() string { + return fmt.Sprintf("%s-ctr", n.Name) +} + +// GetCommand return command to override for the NemoDatastoreV2 container +func (n *NemoDatastoreV2) GetCommand() []string { + return n.Spec.Command +} + +// GetArgs return arguments for the NemoDatastoreV2 container +func (n *NemoDatastoreV2) GetArgs() []string { + return n.Spec.Args +} + +// GetEnv returns merged slice of standard and user specified env variables +func (n *NemoDatastoreV2) GetEnv() []corev1.EnvVar { + return utils.MergeEnvVars(n.GetStandardEnv(), n.Spec.Env) +} + +// GetImage returns container image for the NemoDatastoreV2 +func (n *NemoDatastoreV2) GetImage() string { + return fmt.Sprintf("%s:%s", n.Spec.Image.Repository, n.Spec.Image.Tag) +} + +// GetImagePullSecrets returns the image pull secrets for the NIM container +func (n *NemoDatastoreV2) GetImagePullSecrets() []string { + return n.Spec.Image.PullSecrets +} + +// GetImagePullPolicy returns the image pull policy for the NIM container +func (n *NemoDatastoreV2) GetImagePullPolicy() string { + return n.Spec.Image.PullPolicy +} + +// GetResources returns resources to allocate to the NemoDatastoreV2 container +func (n *NemoDatastoreV2) GetResources() *corev1.ResourceRequirements { + return n.Spec.Resources +} + +// GetLivenessProbe returns liveness probe for the NemoDatastoreV2 container +func (n *NemoDatastoreV2) GetLivenessProbe() *corev1.Probe { + if n.Spec.LivenessProbe.Probe == nil { + return n.GetDefaultLivenessProbe() + } + return n.Spec.LivenessProbe.Probe +} + +// GetDefaultLivenessProbe returns the default liveness probe for the NemoDatastoreV2 container +func (n *NemoDatastoreV2) GetDefaultLivenessProbe() *corev1.Probe { + probe := corev1.Probe{ + InitialDelaySeconds: 15, + TimeoutSeconds: 1, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 3, + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/v1/health/live", + Port: intstr.IntOrString{ + Type: intstr.Type(0), + IntVal: n.Spec.Expose.Service.Port, + }, + }, + }, + } + + return &probe +} + +// GetReadinessProbe returns readiness probe for the NemoDatastoreV2 container +func (n *NemoDatastoreV2) GetReadinessProbe() *corev1.Probe { + if n.Spec.ReadinessProbe.Probe == nil { + return n.GetDefaultReadinessProbe() + } + return n.Spec.ReadinessProbe.Probe +} + +// GetDefaultReadinessProbe returns the default readiness probe for the NemoDatastoreV2 container +func (n *NemoDatastoreV2) GetDefaultReadinessProbe() *corev1.Probe { + probe := corev1.Probe{ + InitialDelaySeconds: 15, + TimeoutSeconds: 1, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 3, + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/v1/health/ready", + Port: intstr.IntOrString{ + Type: intstr.Type(0), + IntVal: n.Spec.Expose.Service.Port, + }, + }, + }, + } + + return &probe +} + +// GetStartupProbe returns startup probe for the NemoDatastoreV2 container +func (n *NemoDatastoreV2) GetStartupProbe() *corev1.Probe { + return n.Spec.StartupProbe.Probe +} + +// GetDefaultStartupProbe returns the default startup probe for the NemoDatastoreV2 container +func (n *NemoDatastoreV2) GetDefaultStartupProbe() *corev1.Probe { + probe := corev1.Probe{ + InitialDelaySeconds: 40, + TimeoutSeconds: 1, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 180, + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/v1/health/ready", + Port: intstr.IntOrString{ + Type: intstr.Type(0), + IntVal: n.Spec.Expose.Service.Port, + }, + }, + }, + } + + return &probe +} + +// GetVolumeMounts returns volumes for the NemoDatastoreV2 container +func (n *NemoDatastoreV2) GetVolumeMounts() []corev1.VolumeMount { + return []corev1.VolumeMount{ + { + MountPath: "/tmp", + Name: "temp", + }, + { + MountPath: "/data", + Name: "data", + }, + } +} + +func (n *NemoDatastoreV2) GetVolumeMountsInitContainer() []corev1.VolumeMount { + return []corev1.VolumeMount{ + { + MountPath: "/usr/sbin", + Name: "config", + }, + { + MountPath: "/tmp", + Name: "temp", + }, + { + MountPath: "/data", + Name: "data", + }, + { + MountPath: "/env-to-ini-mounts/inlines/", + Name: "inline-config-sources", + }, + { + MountPath: "/usr/sbin/init", + Name: "init", + }, + } +} + +func (n *NemoDatastoreV2) GetInitContainers() []corev1.Container { + user := int64(1000) + return []corev1.Container{ + { + Name: "init-datastore", + Image: n.GetImage(), + ImagePullPolicy: corev1.PullPolicy(n.GetImagePullPolicy()), + Command: []string{ + "/bin/sh", "-c", + }, + Args: []string{ + "/usr/sbin/init/init_directory_structure.sh && /usr/sbin/config_environment.sh", + }, + VolumeMounts: n.GetVolumeMountsInitContainer(), + Env: n.GetInitContainerEnv(), + EnvFrom: n.GetInitAppIniEnvFrom(), + }, + { + Name: "configure-datastore", + Image: n.GetImage(), + ImagePullPolicy: corev1.PullPolicy(n.GetImagePullPolicy()), + Command: []string{ + "/bin/sh", "-c", + }, + Args: []string{ + "/usr/sbin/init/configure_gitea.sh", + }, + VolumeMounts: n.GetVolumeMountsInitContainer(), + Env: n.GetInitContainerEnv(), + EnvFrom: n.GetInitAppIniEnvFrom(), + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &user, + }, + }, + } +} + +// GetServiceAccountName returns service account name for the NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) GetServiceAccountName() string { + return n.Name +} + +// GetRuntimeClass return the runtime class name for the NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) GetRuntimeClass() string { + return n.Spec.RuntimeClass +} + +// GetHPA returns the HPA spec for the NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) GetHPA() HorizontalPodAutoscalerSpec { + return n.Spec.Scale.HPA +} + +// GetServiceMonitor returns the Service Monitor details for the NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) GetServiceMonitor() ServiceMonitor { + return n.Spec.Metrics.ServiceMonitor +} + +// GetReplicas returns replicas for the NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) GetReplicas() int { + if n.IsAutoScalingEnabled() { + return 0 + } + return n.Spec.Replicas +} + +// GetDeploymentKind returns the kind of deployment for NemoDatastoreV2 +func (n *NemoDatastoreV2) GetDeploymentKind() string { + return "Deployment" +} + +// IsAutoScalingEnabled returns true if autoscaling is enabled for NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) IsAutoScalingEnabled() bool { + return n.Spec.Scale.Enabled != nil && *n.Spec.Scale.Enabled +} + +// IsIngressEnabled returns true if ingress is enabled for NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) IsIngressEnabled() bool { + return n.Spec.Expose.Ingress.Enabled != nil && *n.Spec.Expose.Ingress.Enabled +} + +// GetIngressSpec returns the Ingress spec NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) GetIngressSpec() networkingv1.IngressSpec { + return n.Spec.Expose.Ingress.Spec +} + +// IsServiceMonitorEnabled returns true if servicemonitor is enabled for NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) IsServiceMonitorEnabled() bool { + return n.Spec.Metrics.Enabled != nil && *n.Spec.Metrics.Enabled +} + +// GetServicePort returns the service port for the NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) GetServicePort() int32 { + return n.Spec.Expose.Service.Port +} + +// GetServiceType returns the service type for the NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) GetServiceType() string { + return string(n.Spec.Expose.Service.Type) +} + +// GetUserID returns the user ID for the NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) GetUserID() *int64 { + return n.Spec.UserID + +} + +// GetGroupID returns the group ID for the NemoDatastoreV2 deployment +func (n *NemoDatastoreV2) GetGroupID() *int64 { + return n.Spec.GroupID + +} + +// GetServiceAccountParams return params to render ServiceAccount from templates +func (n *NemoDatastoreV2) GetServiceAccountParams() *rendertypes.ServiceAccountParams { + params := &rendertypes.ServiceAccountParams{} + + // Set metadata + params.Name = n.GetName() + params.Namespace = n.GetNamespace() + params.Labels = n.GetServiceLabels() + params.Annotations = n.GetNemoDatastoreV2Annotations() + return params +} + +// GetDeploymentParams returns params to render Deployment from templates +func (n *NemoDatastoreV2) GetDeploymentParams() *rendertypes.DeploymentParams { + params := &rendertypes.DeploymentParams{} + + // Set metadata + params.Name = n.GetName() + params.Namespace = n.GetNamespace() + params.Labels = n.GetServiceLabels() + params.Annotations = n.GetNemoDatastoreV2Annotations() + + // Set template spec + if !n.IsAutoScalingEnabled() { + params.Replicas = n.GetReplicas() + } + params.NodeSelector = n.GetNodeSelector() + params.Tolerations = n.GetTolerations() + params.Affinity = n.GetPodAffinity() + params.ImagePullSecrets = n.GetImagePullSecrets() + params.ImagePullPolicy = n.GetImagePullPolicy() + + // Set labels and selectors + params.SelectorLabels = n.GetSelectorLabels() + + // Set container spec + params.ContainerName = n.GetContainerName() + params.Env = n.GetEnv() + params.Args = n.GetArgs() + params.Command = n.GetCommand() + params.Resources = n.GetResources() + params.Image = n.GetImage() + + // Set container probes + if IsProbeEnabled(n.Spec.LivenessProbe) { + params.LivenessProbe = n.GetLivenessProbe() + } + if IsProbeEnabled(n.Spec.ReadinessProbe) { + params.ReadinessProbe = n.GetReadinessProbe() + } + if IsProbeEnabled(n.Spec.StartupProbe) { + params.StartupProbe = n.GetStartupProbe() + } + params.UserID = n.GetUserID() + params.GroupID = n.GetGroupID() + + // Set service account + params.ServiceAccountName = n.GetServiceAccountName() + + // Set runtime class + params.RuntimeClassName = n.GetRuntimeClass() + return params +} + +// GetStatefulSetParams returns params to render StatefulSet from templates +func (n *NemoDatastoreV2) GetStatefulSetParams() *rendertypes.StatefulSetParams { + + params := &rendertypes.StatefulSetParams{} + + // Set metadata + params.Name = n.GetName() + params.Namespace = n.GetNamespace() + params.Labels = n.GetServiceLabels() + params.Annotations = n.GetNemoDatastoreV2Annotations() + + // Set template spec + if !n.IsAutoScalingEnabled() { + params.Replicas = n.GetReplicas() + } + params.ServiceName = n.GetName() + params.NodeSelector = n.GetNodeSelector() + params.Tolerations = n.GetTolerations() + params.Affinity = n.GetPodAffinity() + params.ImagePullSecrets = n.GetImagePullSecrets() + params.ImagePullPolicy = n.GetImagePullPolicy() + + // Set labels and selectors + params.SelectorLabels = n.GetSelectorLabels() + + // Set container spec + params.ContainerName = n.GetContainerName() + params.Env = n.GetEnv() + params.Args = n.GetArgs() + params.Command = n.GetCommand() + params.Resources = n.GetResources() + params.Image = n.GetImage() + + // Set container probes + params.LivenessProbe = n.GetLivenessProbe() + params.ReadinessProbe = n.GetReadinessProbe() + params.StartupProbe = n.GetStartupProbe() + + // Set service account + params.ServiceAccountName = n.GetServiceAccountName() + + // Set runtime class + params.RuntimeClassName = n.GetRuntimeClass() + return params +} + +// GetServiceParams returns params to render Service from templates +func (n *NemoDatastoreV2) GetServiceParams() *rendertypes.ServiceParams { + params := &rendertypes.ServiceParams{} + + // Set metadata + params.Name = n.GetName() + params.Namespace = n.GetNamespace() + params.Labels = n.GetServiceLabels() + params.Annotations = n.GetServiceAnnotations() + + // Set service selector labels + params.SelectorLabels = n.GetSelectorLabels() + + // Set service type + params.Type = n.GetServiceType() + + // Set service ports + params.Port = n.GetServicePort() + params.PortName = "service-port" + return params +} + +// GetIngressParams returns params to render Ingress from templates +func (n *NemoDatastoreV2) GetIngressParams() *rendertypes.IngressParams { + params := &rendertypes.IngressParams{} + + params.Enabled = n.IsIngressEnabled() + // Set metadata + params.Name = n.GetName() + params.Namespace = n.GetNamespace() + params.Labels = n.GetServiceLabels() + params.Annotations = n.GetIngressAnnotations() + params.Spec = n.GetIngressSpec() + return params +} + +// GetRoleParams returns params to render Role from templates +func (n *NemoDatastoreV2) GetRoleParams() *rendertypes.RoleParams { + params := &rendertypes.RoleParams{} + + // Set metadata + params.Name = n.GetName() + params.Namespace = n.GetNamespace() + + // Set rules to use SCC + params.Rules = []rbacv1.PolicyRule{ + { + APIGroups: []string{"security.openshift.io"}, + Resources: []string{"securitycontextconstraints"}, + ResourceNames: []string{"nonroot"}, + Verbs: []string{"use"}, + }, + } + + return params +} + +// GetRoleBindingParams returns params to render RoleBinding from templates +func (n *NemoDatastoreV2) GetRoleBindingParams() *rendertypes.RoleBindingParams { + params := &rendertypes.RoleBindingParams{} + + // Set metadata + params.Name = n.GetName() + params.Namespace = n.GetNamespace() + + params.ServiceAccountName = n.GetServiceAccountName() + params.RoleName = n.GetName() + return params +} + +// GetHPAParams returns params to render HPA from templates +func (n *NemoDatastoreV2) GetHPAParams() *rendertypes.HPAParams { + params := &rendertypes.HPAParams{} + + params.Enabled = n.IsAutoScalingEnabled() + + // Set metadata + params.Name = n.GetName() + params.Namespace = n.GetNamespace() + params.Labels = n.GetServiceLabels() + params.Annotations = n.GetHPAAnnotations() + + // Set HPA spec + hpa := n.GetHPA() + hpaSpec := autoscalingv2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + Kind: n.GetDeploymentKind(), + Name: n.GetName(), + APIVersion: "apps/v1", + }, + MinReplicas: hpa.MinReplicas, + MaxReplicas: hpa.MaxReplicas, + Metrics: hpa.Metrics, + Behavior: hpa.Behavior, + } + params.HPASpec = hpaSpec + return params +} + +// GetSCCParams return params to render SCC from templates +func (n *NemoDatastoreV2) GetSCCParams() *rendertypes.SCCParams { + params := &rendertypes.SCCParams{} + // Set metadata + params.Name = "nemo-datastore-scc" + + params.ServiceAccountName = n.GetServiceAccountName() + return params +} + +// GetServiceMonitorParams return params to render Service Monitor from templates +func (n *NemoDatastoreV2) GetServiceMonitorParams() *rendertypes.ServiceMonitorParams { + params := &rendertypes.ServiceMonitorParams{} + serviceMonitor := n.GetServiceMonitor() + params.Enabled = n.IsServiceMonitorEnabled() + params.Name = n.GetName() + params.Namespace = n.GetNamespace() + svcLabels := n.GetServiceLabels() + maps.Copy(svcLabels, serviceMonitor.AdditionalLabels) + params.Labels = svcLabels + params.Annotations = n.GetServiceMonitorAnnotations() + + // Set Service Monitor spec + smSpec := monitoringv1.ServiceMonitorSpec{ + NamespaceSelector: monitoringv1.NamespaceSelector{MatchNames: []string{n.Namespace}}, + Selector: metav1.LabelSelector{MatchLabels: n.GetServiceLabels()}, + Endpoints: []monitoringv1.Endpoint{{Port: "service-port", ScrapeTimeout: serviceMonitor.ScrapeTimeout, Interval: serviceMonitor.Interval}}, + } + params.SMSpec = smSpec + return params +} + +func (n *NemoDatastoreV2) GetIngressAnnotations() map[string]string { + NemoDatastoreV2Annotations := n.GetNemoDatastoreV2Annotations() + + if n.Spec.Expose.Ingress.Annotations != nil { + return utils.MergeMaps(NemoDatastoreV2Annotations, n.Spec.Expose.Ingress.Annotations) + } + return NemoDatastoreV2Annotations +} + +func (n *NemoDatastoreV2) GetServiceAnnotations() map[string]string { + NemoDatastoreV2Annotations := n.GetNemoDatastoreV2Annotations() + + if n.Spec.Expose.Service.Annotations != nil { + return utils.MergeMaps(NemoDatastoreV2Annotations, n.Spec.Expose.Service.Annotations) + } + return NemoDatastoreV2Annotations +} + +func (n *NemoDatastoreV2) GetHPAAnnotations() map[string]string { + NemoDatastoreV2Annotations := n.GetNemoDatastoreV2Annotations() + + if n.Spec.Scale.Annotations != nil { + return utils.MergeMaps(NemoDatastoreV2Annotations, n.Spec.Scale.Annotations) + } + return NemoDatastoreV2Annotations +} + +func (n *NemoDatastoreV2) GetServiceMonitorAnnotations() map[string]string { + NemoDatastoreV2Annotations := n.GetNemoDatastoreV2Annotations() + + if n.Spec.Metrics.ServiceMonitor.Annotations != nil { + return utils.MergeMaps(NemoDatastoreV2Annotations, n.Spec.Metrics.ServiceMonitor.Annotations) + } + return NemoDatastoreV2Annotations +} + +func init() { + SchemeBuilder.Register(&NemoDatastoreV2{}, &NemoDatastoreV2List{}) +} diff --git a/api/apps/v1alpha1/zz_generated.deepcopy.go b/api/apps/v1alpha1/zz_generated.deepcopy.go index fc779652..c8442fbe 100644 --- a/api/apps/v1alpha1/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/zz_generated.deepcopy.go @@ -1092,6 +1092,195 @@ func (in *NemoDatastoreStatus) DeepCopy() *NemoDatastoreStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NemoDatastoreV2) DeepCopyInto(out *NemoDatastoreV2) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoDatastoreV2. +func (in *NemoDatastoreV2) DeepCopy() *NemoDatastoreV2 { + if in == nil { + return nil + } + out := new(NemoDatastoreV2) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NemoDatastoreV2) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NemoDatastoreV2List) DeepCopyInto(out *NemoDatastoreV2List) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NemoDatastoreV2, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoDatastoreV2List. +func (in *NemoDatastoreV2List) DeepCopy() *NemoDatastoreV2List { + if in == nil { + return nil + } + out := new(NemoDatastoreV2List) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NemoDatastoreV2List) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NemoDatastoreV2Params) DeepCopyInto(out *NemoDatastoreV2Params) { + *out = *in + if in.InitContainerCommand != nil { + in, out := &in.InitContainerCommand, &out.InitContainerCommand + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoDatastoreV2Params. +func (in *NemoDatastoreV2Params) DeepCopy() *NemoDatastoreV2Params { + if in == nil { + return nil + } + out := new(NemoDatastoreV2Params) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NemoDatastoreV2Spec) DeepCopyInto(out *NemoDatastoreV2Spec) { + *out = *in + in.Image.DeepCopyInto(&out.Image) + if in.Command != nil { + in, out := &in.Command, &out.Command + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Args != nil { + in, out := &in.Args, &out.Args + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Env != nil { + in, out := &in.Env, &out.Env + *out = make([]v1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]v1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.PodAffinity != nil { + in, out := &in.PodAffinity, &out.PodAffinity + *out = new(v1.PodAffinity) + (*in).DeepCopyInto(*out) + } + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(v1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } + in.Expose.DeepCopyInto(&out.Expose) + in.LivenessProbe.DeepCopyInto(&out.LivenessProbe) + in.ReadinessProbe.DeepCopyInto(&out.ReadinessProbe) + in.StartupProbe.DeepCopyInto(&out.StartupProbe) + in.Scale.DeepCopyInto(&out.Scale) + in.Metrics.DeepCopyInto(&out.Metrics) + if in.UserID != nil { + in, out := &in.UserID, &out.UserID + *out = new(int64) + **out = **in + } + if in.GroupID != nil { + in, out := &in.GroupID, &out.GroupID + *out = new(int64) + **out = **in + } + in.DataStoreParams.DeepCopyInto(&out.DataStoreParams) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoDatastoreV2Spec. +func (in *NemoDatastoreV2Spec) DeepCopy() *NemoDatastoreV2Spec { + if in == nil { + return nil + } + out := new(NemoDatastoreV2Spec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NemoDatastoreV2Status) DeepCopyInto(out *NemoDatastoreV2Status) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoDatastoreV2Status. +func (in *NemoDatastoreV2Status) DeepCopy() *NemoDatastoreV2Status { + if in == nil { + return nil + } + out := new(NemoDatastoreV2Status) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NemoGuardrail) DeepCopyInto(out *NemoGuardrail) { *out = *in diff --git a/api/versioned/typed/apps/v1alpha1/apps_client.go b/api/versioned/typed/apps/v1alpha1/apps_client.go index debdc18f..092bcbb6 100644 --- a/api/versioned/typed/apps/v1alpha1/apps_client.go +++ b/api/versioned/typed/apps/v1alpha1/apps_client.go @@ -31,6 +31,7 @@ type AppsV1alpha1Interface interface { NIMPipelinesGetter NIMServicesGetter NemoDatastoresGetter + NemoDatastoreV2sGetter NemoGuardrailsGetter } @@ -55,6 +56,10 @@ func (c *AppsV1alpha1Client) NemoDatastores(namespace string) NemoDatastoreInter return newNemoDatastores(c, namespace) } +func (c *AppsV1alpha1Client) NemoDatastoreV2s(namespace string) NemoDatastoreV2Interface { + return newNemoDatastoreV2s(c, namespace) +} + func (c *AppsV1alpha1Client) NemoGuardrails(namespace string) NemoGuardrailInterface { return newNemoGuardrails(c, namespace) } diff --git a/api/versioned/typed/apps/v1alpha1/fake/fake_apps_client.go b/api/versioned/typed/apps/v1alpha1/fake/fake_apps_client.go index 21b14ff6..7784e2fe 100644 --- a/api/versioned/typed/apps/v1alpha1/fake/fake_apps_client.go +++ b/api/versioned/typed/apps/v1alpha1/fake/fake_apps_client.go @@ -43,6 +43,10 @@ func (c *FakeAppsV1alpha1) NemoDatastores(namespace string) v1alpha1.NemoDatasto return &FakeNemoDatastores{c, namespace} } +func (c *FakeAppsV1alpha1) NemoDatastoreV2s(namespace string) v1alpha1.NemoDatastoreV2Interface { + return &FakeNemoDatastoreV2s{c, namespace} +} + func (c *FakeAppsV1alpha1) NemoGuardrails(namespace string) v1alpha1.NemoGuardrailInterface { return &FakeNemoGuardrails{c, namespace} } diff --git a/api/versioned/typed/apps/v1alpha1/fake/fake_nemodatastorev2.go b/api/versioned/typed/apps/v1alpha1/fake/fake_nemodatastorev2.go new file mode 100644 index 00000000..fd957d51 --- /dev/null +++ b/api/versioned/typed/apps/v1alpha1/fake/fake_nemodatastorev2.go @@ -0,0 +1,146 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1alpha1 "github.com/NVIDIA/k8s-nim-operator/api/apps/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeNemoDatastoreV2s implements NemoDatastoreV2Interface +type FakeNemoDatastoreV2s struct { + Fake *FakeAppsV1alpha1 + ns string +} + +var nemodatastorev2sResource = v1alpha1.SchemeGroupVersion.WithResource("nemodatastorev2s") + +var nemodatastorev2sKind = v1alpha1.SchemeGroupVersion.WithKind("NemoDatastoreV2") + +// Get takes name of the nemoDatastoreV2, and returns the corresponding nemoDatastoreV2 object, and an error if there is any. +func (c *FakeNemoDatastoreV2s) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.NemoDatastoreV2, err error) { + emptyResult := &v1alpha1.NemoDatastoreV2{} + obj, err := c.Fake. + Invokes(testing.NewGetActionWithOptions(nemodatastorev2sResource, c.ns, name, options), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.NemoDatastoreV2), err +} + +// List takes label and field selectors, and returns the list of NemoDatastoreV2s that match those selectors. +func (c *FakeNemoDatastoreV2s) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.NemoDatastoreV2List, err error) { + emptyResult := &v1alpha1.NemoDatastoreV2List{} + obj, err := c.Fake. + Invokes(testing.NewListActionWithOptions(nemodatastorev2sResource, nemodatastorev2sKind, c.ns, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.NemoDatastoreV2List{ListMeta: obj.(*v1alpha1.NemoDatastoreV2List).ListMeta} + for _, item := range obj.(*v1alpha1.NemoDatastoreV2List).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested nemoDatastoreV2s. +func (c *FakeNemoDatastoreV2s) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchActionWithOptions(nemodatastorev2sResource, c.ns, opts)) + +} + +// Create takes the representation of a nemoDatastoreV2 and creates it. Returns the server's representation of the nemoDatastoreV2, and an error, if there is any. +func (c *FakeNemoDatastoreV2s) Create(ctx context.Context, nemoDatastoreV2 *v1alpha1.NemoDatastoreV2, opts v1.CreateOptions) (result *v1alpha1.NemoDatastoreV2, err error) { + emptyResult := &v1alpha1.NemoDatastoreV2{} + obj, err := c.Fake. + Invokes(testing.NewCreateActionWithOptions(nemodatastorev2sResource, c.ns, nemoDatastoreV2, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.NemoDatastoreV2), err +} + +// Update takes the representation of a nemoDatastoreV2 and updates it. Returns the server's representation of the nemoDatastoreV2, and an error, if there is any. +func (c *FakeNemoDatastoreV2s) Update(ctx context.Context, nemoDatastoreV2 *v1alpha1.NemoDatastoreV2, opts v1.UpdateOptions) (result *v1alpha1.NemoDatastoreV2, err error) { + emptyResult := &v1alpha1.NemoDatastoreV2{} + obj, err := c.Fake. + Invokes(testing.NewUpdateActionWithOptions(nemodatastorev2sResource, c.ns, nemoDatastoreV2, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.NemoDatastoreV2), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeNemoDatastoreV2s) UpdateStatus(ctx context.Context, nemoDatastoreV2 *v1alpha1.NemoDatastoreV2, opts v1.UpdateOptions) (result *v1alpha1.NemoDatastoreV2, err error) { + emptyResult := &v1alpha1.NemoDatastoreV2{} + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceActionWithOptions(nemodatastorev2sResource, "status", c.ns, nemoDatastoreV2, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.NemoDatastoreV2), err +} + +// Delete takes name of the nemoDatastoreV2 and deletes it. Returns an error if one occurs. +func (c *FakeNemoDatastoreV2s) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteActionWithOptions(nemodatastorev2sResource, c.ns, name, opts), &v1alpha1.NemoDatastoreV2{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeNemoDatastoreV2s) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionActionWithOptions(nemodatastorev2sResource, c.ns, opts, listOpts) + + _, err := c.Fake.Invokes(action, &v1alpha1.NemoDatastoreV2List{}) + return err +} + +// Patch applies the patch and returns the patched nemoDatastoreV2. +func (c *FakeNemoDatastoreV2s) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.NemoDatastoreV2, err error) { + emptyResult := &v1alpha1.NemoDatastoreV2{} + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceActionWithOptions(nemodatastorev2sResource, c.ns, name, pt, data, opts, subresources...), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.NemoDatastoreV2), err +} diff --git a/api/versioned/typed/apps/v1alpha1/generated_expansion.go b/api/versioned/typed/apps/v1alpha1/generated_expansion.go index 7e76a96e..18de8de7 100644 --- a/api/versioned/typed/apps/v1alpha1/generated_expansion.go +++ b/api/versioned/typed/apps/v1alpha1/generated_expansion.go @@ -25,4 +25,6 @@ type NIMServiceExpansion interface{} type NemoDatastoreExpansion interface{} +type NemoDatastoreV2Expansion interface{} + type NemoGuardrailExpansion interface{} diff --git a/api/versioned/typed/apps/v1alpha1/nemodatastorev2.go b/api/versioned/typed/apps/v1alpha1/nemodatastorev2.go new file mode 100644 index 00000000..d58def11 --- /dev/null +++ b/api/versioned/typed/apps/v1alpha1/nemodatastorev2.go @@ -0,0 +1,68 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + + v1alpha1 "github.com/NVIDIA/k8s-nim-operator/api/apps/v1alpha1" + scheme "github.com/NVIDIA/k8s-nim-operator/api/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" +) + +// NemoDatastoreV2sGetter has a method to return a NemoDatastoreV2Interface. +// A group's client should implement this interface. +type NemoDatastoreV2sGetter interface { + NemoDatastoreV2s(namespace string) NemoDatastoreV2Interface +} + +// NemoDatastoreV2Interface has methods to work with NemoDatastoreV2 resources. +type NemoDatastoreV2Interface interface { + Create(ctx context.Context, nemoDatastoreV2 *v1alpha1.NemoDatastoreV2, opts v1.CreateOptions) (*v1alpha1.NemoDatastoreV2, error) + Update(ctx context.Context, nemoDatastoreV2 *v1alpha1.NemoDatastoreV2, opts v1.UpdateOptions) (*v1alpha1.NemoDatastoreV2, error) + // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + UpdateStatus(ctx context.Context, nemoDatastoreV2 *v1alpha1.NemoDatastoreV2, opts v1.UpdateOptions) (*v1alpha1.NemoDatastoreV2, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.NemoDatastoreV2, error) + List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.NemoDatastoreV2List, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.NemoDatastoreV2, err error) + NemoDatastoreV2Expansion +} + +// nemoDatastoreV2s implements NemoDatastoreV2Interface +type nemoDatastoreV2s struct { + *gentype.ClientWithList[*v1alpha1.NemoDatastoreV2, *v1alpha1.NemoDatastoreV2List] +} + +// newNemoDatastoreV2s returns a NemoDatastoreV2s +func newNemoDatastoreV2s(c *AppsV1alpha1Client, namespace string) *nemoDatastoreV2s { + return &nemoDatastoreV2s{ + gentype.NewClientWithList[*v1alpha1.NemoDatastoreV2, *v1alpha1.NemoDatastoreV2List]( + "nemodatastorev2s", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *v1alpha1.NemoDatastoreV2 { return &v1alpha1.NemoDatastoreV2{} }, + func() *v1alpha1.NemoDatastoreV2List { return &v1alpha1.NemoDatastoreV2List{} }), + } +} diff --git a/bundle/manifests/apps.nvidia.com_nemodatastorev2s.yaml b/bundle/manifests/apps.nvidia.com_nemodatastorev2s.yaml new file mode 100644 index 00000000..36a7c940 --- /dev/null +++ b/bundle/manifests/apps.nvidia.com_nemodatastorev2s.yaml @@ -0,0 +1,2236 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.2 + name: nemodatastorev2s.apps.nvidia.com +spec: + group: apps.nvidia.com + names: + kind: NemoDatastoreV2 + listKind: NemoDatastoreV2List + plural: nemodatastorev2s + singular: nemodatastorev2 + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.state + name: Status + type: string + - format: date-time + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: NemoDatastoreV2 is the Schema for the NemoDatastoreV2 API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NemoDatastoreV2Spec defines the desired state of NemoDatastoreV2 + properties: + annotations: + additionalProperties: + type: string + type: object + args: + items: + type: string + type: array + authSecret: + description: The name of an secret that contains authn for the NGC + NIM service API + type: string + command: + items: + type: string + type: array + dataStoreParams: + properties: + appVersion: + type: string + databaseHost: + type: string + databasePort: + type: string + databaseURL: + type: string + datastoreConfigSecret: + type: string + datastoreInitSecret: + type: string + datastoreInlineConfigSecret: + type: string + datastoreSettingsSecret: + type: string + dbSecret: + type: string + envConfigmap: + type: string + envSecret: + type: string + giteaAdminSecret: + type: string + initContainerCommand: + items: + type: string + type: array + initContainerImage: + type: string + lfsJwtSecret: + type: string + objStoreSecret: + type: string + pvcSharedData: + type: string + size: + type: string + sshEnabled: + type: boolean + storageClass: + type: string + required: + - appVersion + - databaseHost + - databasePort + - databaseURL + - datastoreConfigSecret + - datastoreInitSecret + - datastoreInlineConfigSecret + - datastoreSettingsSecret + - dbSecret + - envConfigmap + - envSecret + - giteaAdminSecret + - lfsJwtSecret + - objStoreSecret + - pvcSharedData + - sshEnabled + - storageClass + type: object + env: + items: + description: EnvVar represents an environment variable present in + a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot + be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + expose: + description: Expose defines attributes to expose the service + properties: + ingress: + description: Ingress defines attributes to enable ingress for + the service + properties: + annotations: + additionalProperties: + type: string + type: object + enabled: + description: ingress, or virtualService - not both + type: boolean + spec: + description: IngressSpec describes the Ingress the user wishes + to exist. + properties: + defaultBackend: + description: |- + defaultBackend is the backend that should handle requests that don't + match any rule. If Rules are not specified, DefaultBackend must be specified. + If DefaultBackend is not set, the handling of requests that do not match any + of the rules will be up to the Ingress controller. + properties: + resource: + description: |- + resource is an ObjectRef to another Kubernetes resource in the namespace + of the Ingress object. If resource is specified, a service.Name and + service.Port must not be specified. + This is a mutually exclusive setting with "Service". + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + service: + description: |- + service references a service as a backend. + This is a mutually exclusive setting with "Resource". + properties: + name: + description: |- + name is the referenced service. The service must exist in + the same namespace as the Ingress object. + type: string + port: + description: |- + port of the referenced service. A port name or port number + is required for a IngressServiceBackend. + properties: + name: + description: |- + name is the name of the port on the Service. + This is a mutually exclusive setting with "Number". + type: string + number: + description: |- + number is the numerical port number (e.g. 80) on the Service. + This is a mutually exclusive setting with "Name". + format: int32 + type: integer + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + type: object + ingressClassName: + description: |- + ingressClassName is the name of an IngressClass cluster resource. Ingress + controller implementations use this field to know whether they should be + serving this Ingress resource, by a transitive connection + (controller -> IngressClass -> Ingress resource). Although the + `kubernetes.io/ingress.class` annotation (simple constant name) was never + formally defined, it was widely supported by Ingress controllers to create + a direct binding between Ingress controller and Ingress resources. Newly + created Ingress resources should prefer using the field. However, even + though the annotation is officially deprecated, for backwards compatibility + reasons, ingress controllers should still honor that annotation if present. + type: string + rules: + description: |- + rules is a list of host rules used to configure the Ingress. If unspecified, + or no rule matches, all traffic is sent to the default backend. + items: + description: |- + IngressRule represents the rules mapping the paths under a specified host to + the related backend services. Incoming requests are first evaluated for a host + match, then routed to the backend associated with the matching IngressRuleValue. + properties: + host: + description: "host is the fully qualified domain + name of a network host, as defined by RFC 3986.\nNote + the following deviations from the \"host\" part + of the\nURI as defined in RFC 3986:\n1. IPs are + not allowed. Currently an IngressRuleValue can + only apply to\n the IP in the Spec of the parent + Ingress.\n2. The `:` delimiter is not respected + because ports are not allowed.\n\t Currently + the port of an Ingress is implicitly :80 for http + and\n\t :443 for https.\nBoth these may change + in the future.\nIncoming requests are matched + against the host before the\nIngressRuleValue. + If the host is unspecified, the Ingress routes + all\ntraffic based on the specified IngressRuleValue.\n\nhost + can be \"precise\" which is a domain name without + the terminating dot of\na network host (e.g. \"foo.bar.com\") + or \"wildcard\", which is a domain name\nprefixed + with a single wildcard label (e.g. \"*.foo.com\").\nThe + wildcard character '*' must appear by itself as + the first DNS label and\nmatches only a single + label. You cannot have a wildcard label by itself + (e.g. Host == \"*\").\nRequests will be matched + against the Host field in the following way:\n1. + If host is precise, the request matches this rule + if the http host header is equal to Host.\n2. + If host is a wildcard, then the request matches + this rule if the http host header\nis to equal + to the suffix (removing the first label) of the + wildcard rule." + type: string + http: + description: |- + HTTPIngressRuleValue is a list of http selectors pointing to backends. + In the example: http:///? -> backend where + where parts of the url correspond to RFC 3986, this resource will be used + to match against everything after the last '/' and before the first '?' + or '#'. + properties: + paths: + description: paths is a collection of paths + that map requests to backends. + items: + description: |- + HTTPIngressPath associates a path with a backend. Incoming urls matching the + path are forwarded to the backend. + properties: + backend: + description: |- + backend defines the referenced service endpoint to which the traffic + will be forwarded to. + properties: + resource: + description: |- + resource is an ObjectRef to another Kubernetes resource in the namespace + of the Ingress object. If resource is specified, a service.Name and + service.Port must not be specified. + This is a mutually exclusive setting with "Service". + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type + of resource being referenced + type: string + name: + description: Name is the name + of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + service: + description: |- + service references a service as a backend. + This is a mutually exclusive setting with "Resource". + properties: + name: + description: |- + name is the referenced service. The service must exist in + the same namespace as the Ingress object. + type: string + port: + description: |- + port of the referenced service. A port name or port number + is required for a IngressServiceBackend. + properties: + name: + description: |- + name is the name of the port on the Service. + This is a mutually exclusive setting with "Number". + type: string + number: + description: |- + number is the numerical port number (e.g. 80) on the Service. + This is a mutually exclusive setting with "Name". + format: int32 + type: integer + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + type: object + path: + description: |- + path is matched against the path of an incoming request. Currently it can + contain characters disallowed from the conventional "path" part of a URL + as defined by RFC 3986. Paths must begin with a '/' and must be present + when using PathType with value "Exact" or "Prefix". + type: string + pathType: + description: |- + pathType determines the interpretation of the path matching. PathType can + be one of the following values: + * Exact: Matches the URL path exactly. + * Prefix: Matches based on a URL path prefix split by '/'. Matching is + done on a path element by element basis. A path element refers is the + list of labels in the path split by the '/' separator. A request is a + match for path p if every p is an element-wise prefix of p of the + request path. Note that if the last element of the path is a substring + of the last element in request path, it is not a match (e.g. /foo/bar + matches /foo/bar/baz, but does not match /foo/barbaz). + * ImplementationSpecific: Interpretation of the Path matching is up to + the IngressClass. Implementations can treat this as a separate PathType + or treat it identically to Prefix or Exact path types. + Implementations are required to support all path types. + type: string + required: + - backend + - pathType + type: object + type: array + x-kubernetes-list-type: atomic + required: + - paths + type: object + type: object + type: array + x-kubernetes-list-type: atomic + tls: + description: |- + tls represents the TLS configuration. Currently the Ingress only supports a + single TLS port, 443. If multiple members of this list specify different hosts, + they will be multiplexed on the same port according to the hostname specified + through the SNI TLS extension, if the ingress controller fulfilling the + ingress supports SNI. + items: + description: IngressTLS describes the transport layer + security associated with an ingress. + properties: + hosts: + description: |- + hosts is a list of hosts included in the TLS certificate. The values in + this list must match the name/s used in the tlsSecret. Defaults to the + wildcard host setting for the loadbalancer controller fulfilling this + Ingress, if left unspecified. + items: + type: string + type: array + x-kubernetes-list-type: atomic + secretName: + description: |- + secretName is the name of the secret used to terminate TLS traffic on + port 443. Field is left optional to allow TLS routing based on SNI + hostname alone. If the SNI host in a listener conflicts with the "Host" + header field used by an IngressRule, the SNI host is used for termination + and value of the "Host" header is used for routing. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + service: + description: Service defines attributes to create a service + properties: + annotations: + additionalProperties: + type: string + type: object + name: + description: override the default service name + type: string + port: + default: 8000 + format: int32 + type: integer + type: + description: Service Type string describes ingress methods + for a service + type: string + required: + - port + type: object + type: object + groupID: + format: int64 + type: integer + image: + description: Image defines image attributes + properties: + pullPolicy: + type: string + pullSecrets: + items: + type: string + type: array + repository: + type: string + tag: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + livenessProbe: + description: Probe defines attributes for startup/liveness/readiness + probes + properties: + enabled: + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP + port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + metrics: + description: Metrics defines attributes to setup metrics collection + properties: + enabled: + type: boolean + serviceMonitor: + description: for use with the Prometheus Operator and the primary + service object + properties: + additionalLabels: + additionalProperties: + type: string + type: object + annotations: + additionalProperties: + type: string + type: object + interval: + description: |- + Duration is a valid time duration that can be parsed by Prometheus model.ParseDuration() function. + Supported units: y, w, d, h, m, s, ms + Examples: `30s`, `1m`, `1h20m15s`, `15d` + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + scrapeTimeout: + description: |- + Duration is a valid time duration that can be parsed by Prometheus model.ParseDuration() function. + Supported units: y, w, d, h, m, s, ms + Examples: `30s`, `1m`, `1h20m15s`, `15d` + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + type: object + type: object + nodeSelector: + additionalProperties: + type: string + type: object + podAffinity: + description: Pod affinity is a group of inter pod affinity scheduling + rules. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with + the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + readinessProbe: + description: Probe defines attributes for startup/liveness/readiness + probes + properties: + enabled: + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP + port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + replicas: + default: 1 + minimum: 1 + type: integer + resources: + description: ResourceRequirements describes the compute resource requirements. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + runtimeClass: + type: string + scale: + description: Autoscaling defines attributes to automatically scale + the service based on metrics + properties: + annotations: + additionalProperties: + type: string + type: object + enabled: + type: boolean + hpa: + description: HorizontalPodAutoscalerSpec defines the parameters + required to setup HPA + properties: + behavior: + description: |- + HorizontalPodAutoscalerBehavior configures the scaling behavior of the target + in both Up and Down directions (scaleUp and scaleDown fields respectively). + properties: + scaleDown: + description: |- + scaleDown is scaling policy for scaling Down. + If not set, the default value is to allow to scale down to minReplicas pods, with a + 300 second stabilization window (i.e., the highest recommendation for + the last 300sec is used). + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the scaling + policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + type: object + scaleUp: + description: |- + scaleUp is scaling policy for scaling Up. + If not set, the default value is the higher of: + * increase no more than 4 pods per 60 seconds + * double the number of pods per 60 seconds + No stabilization is used. + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the scaling + policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + type: object + type: object + maxReplicas: + format: int32 + type: integer + metrics: + items: + description: |- + MetricSpec specifies how to scale based on a single metric + (only `type` and one other matching field should be set at once). + properties: + containerResource: + description: |- + containerResource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing a single container in + each pod of the current scale target (e.g. CPU or memory). Such metrics are + built in to Kubernetes, and have special scaling options on top of those + available to normal per-pod metrics using the "pods" source. + This is an alpha feature and can be enabled by the HPAContainerMetrics feature flag. + properties: + container: + description: container is the name of the container + in the pods of the scaling target + type: string + name: + description: name is the name of the resource in + question. + type: string + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: |- + external refers to a global metric that is not associated + with any Kubernetes object. It allows autoscaling based on information + coming from components running outside of cluster + (for example length of queue in cloud messaging service, or + QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: |- + object refers to a metric describing a single kubernetes object + (for example, hits-per-second on an Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions + of a object,such as kind,name apiVersion + properties: + apiVersion: + description: apiVersion is the API version of + the referent + type: string + kind: + description: 'kind is the kind of the referent; + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: |- + pods refers to a metric describing each pod in the current scale target + (for example, transactions-processed-per-second). The values will be + averaged together before being compared to the target value. + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: |- + resource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing each pod in the + current scale target (e.g. CPU or memory). Such metrics are built in to + Kubernetes, and have special scaling options on top of those available + to normal per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource in + question. + type: string + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: |- + type is the type of metric source. It should be one of "ContainerResource", "External", + "Object", "Pods" or "Resource", each mapping to a matching field in the object. + Note: "ContainerResource" type is available on when the feature-gate + HPAContainerMetrics is enabled + type: string + required: + - type + type: object + type: array + minReplicas: + format: int32 + type: integer + required: + - maxReplicas + type: object + type: object + startupProbe: + description: Probe defines attributes for startup/liveness/readiness + probes + properties: + enabled: + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP + port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + userID: + format: int64 + type: integer + required: + - authSecret + - dataStoreParams + type: object + status: + description: NemoDatastoreV2Status defines the observed state of NemoDatastoreV2 + properties: + availableReplicas: + format: int32 + type: integer + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + state: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/cmd/main.go b/cmd/main.go index 081e864d..499e7beb 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -199,6 +199,17 @@ func main() { os.Exit(1) } + if err = controller.NewNemoDatastoreV2Reconciler( + mgr.GetClient(), + mgr.GetScheme(), + updater, + render.NewRenderer("/manifests"), + ctrl.Log.WithName("controllers").WithName("NemoDatastoreV2"), + ).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "NemoDatastoreV2") + os.Exit(1) + } + // +kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { diff --git a/config/crd/bases/apps.nvidia.com_nemodatastorev2s.yaml b/config/crd/bases/apps.nvidia.com_nemodatastorev2s.yaml new file mode 100644 index 00000000..36a7c940 --- /dev/null +++ b/config/crd/bases/apps.nvidia.com_nemodatastorev2s.yaml @@ -0,0 +1,2236 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.2 + name: nemodatastorev2s.apps.nvidia.com +spec: + group: apps.nvidia.com + names: + kind: NemoDatastoreV2 + listKind: NemoDatastoreV2List + plural: nemodatastorev2s + singular: nemodatastorev2 + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.state + name: Status + type: string + - format: date-time + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: NemoDatastoreV2 is the Schema for the NemoDatastoreV2 API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NemoDatastoreV2Spec defines the desired state of NemoDatastoreV2 + properties: + annotations: + additionalProperties: + type: string + type: object + args: + items: + type: string + type: array + authSecret: + description: The name of an secret that contains authn for the NGC + NIM service API + type: string + command: + items: + type: string + type: array + dataStoreParams: + properties: + appVersion: + type: string + databaseHost: + type: string + databasePort: + type: string + databaseURL: + type: string + datastoreConfigSecret: + type: string + datastoreInitSecret: + type: string + datastoreInlineConfigSecret: + type: string + datastoreSettingsSecret: + type: string + dbSecret: + type: string + envConfigmap: + type: string + envSecret: + type: string + giteaAdminSecret: + type: string + initContainerCommand: + items: + type: string + type: array + initContainerImage: + type: string + lfsJwtSecret: + type: string + objStoreSecret: + type: string + pvcSharedData: + type: string + size: + type: string + sshEnabled: + type: boolean + storageClass: + type: string + required: + - appVersion + - databaseHost + - databasePort + - databaseURL + - datastoreConfigSecret + - datastoreInitSecret + - datastoreInlineConfigSecret + - datastoreSettingsSecret + - dbSecret + - envConfigmap + - envSecret + - giteaAdminSecret + - lfsJwtSecret + - objStoreSecret + - pvcSharedData + - sshEnabled + - storageClass + type: object + env: + items: + description: EnvVar represents an environment variable present in + a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot + be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + expose: + description: Expose defines attributes to expose the service + properties: + ingress: + description: Ingress defines attributes to enable ingress for + the service + properties: + annotations: + additionalProperties: + type: string + type: object + enabled: + description: ingress, or virtualService - not both + type: boolean + spec: + description: IngressSpec describes the Ingress the user wishes + to exist. + properties: + defaultBackend: + description: |- + defaultBackend is the backend that should handle requests that don't + match any rule. If Rules are not specified, DefaultBackend must be specified. + If DefaultBackend is not set, the handling of requests that do not match any + of the rules will be up to the Ingress controller. + properties: + resource: + description: |- + resource is an ObjectRef to another Kubernetes resource in the namespace + of the Ingress object. If resource is specified, a service.Name and + service.Port must not be specified. + This is a mutually exclusive setting with "Service". + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + service: + description: |- + service references a service as a backend. + This is a mutually exclusive setting with "Resource". + properties: + name: + description: |- + name is the referenced service. The service must exist in + the same namespace as the Ingress object. + type: string + port: + description: |- + port of the referenced service. A port name or port number + is required for a IngressServiceBackend. + properties: + name: + description: |- + name is the name of the port on the Service. + This is a mutually exclusive setting with "Number". + type: string + number: + description: |- + number is the numerical port number (e.g. 80) on the Service. + This is a mutually exclusive setting with "Name". + format: int32 + type: integer + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + type: object + ingressClassName: + description: |- + ingressClassName is the name of an IngressClass cluster resource. Ingress + controller implementations use this field to know whether they should be + serving this Ingress resource, by a transitive connection + (controller -> IngressClass -> Ingress resource). Although the + `kubernetes.io/ingress.class` annotation (simple constant name) was never + formally defined, it was widely supported by Ingress controllers to create + a direct binding between Ingress controller and Ingress resources. Newly + created Ingress resources should prefer using the field. However, even + though the annotation is officially deprecated, for backwards compatibility + reasons, ingress controllers should still honor that annotation if present. + type: string + rules: + description: |- + rules is a list of host rules used to configure the Ingress. If unspecified, + or no rule matches, all traffic is sent to the default backend. + items: + description: |- + IngressRule represents the rules mapping the paths under a specified host to + the related backend services. Incoming requests are first evaluated for a host + match, then routed to the backend associated with the matching IngressRuleValue. + properties: + host: + description: "host is the fully qualified domain + name of a network host, as defined by RFC 3986.\nNote + the following deviations from the \"host\" part + of the\nURI as defined in RFC 3986:\n1. IPs are + not allowed. Currently an IngressRuleValue can + only apply to\n the IP in the Spec of the parent + Ingress.\n2. The `:` delimiter is not respected + because ports are not allowed.\n\t Currently + the port of an Ingress is implicitly :80 for http + and\n\t :443 for https.\nBoth these may change + in the future.\nIncoming requests are matched + against the host before the\nIngressRuleValue. + If the host is unspecified, the Ingress routes + all\ntraffic based on the specified IngressRuleValue.\n\nhost + can be \"precise\" which is a domain name without + the terminating dot of\na network host (e.g. \"foo.bar.com\") + or \"wildcard\", which is a domain name\nprefixed + with a single wildcard label (e.g. \"*.foo.com\").\nThe + wildcard character '*' must appear by itself as + the first DNS label and\nmatches only a single + label. You cannot have a wildcard label by itself + (e.g. Host == \"*\").\nRequests will be matched + against the Host field in the following way:\n1. + If host is precise, the request matches this rule + if the http host header is equal to Host.\n2. + If host is a wildcard, then the request matches + this rule if the http host header\nis to equal + to the suffix (removing the first label) of the + wildcard rule." + type: string + http: + description: |- + HTTPIngressRuleValue is a list of http selectors pointing to backends. + In the example: http:///? -> backend where + where parts of the url correspond to RFC 3986, this resource will be used + to match against everything after the last '/' and before the first '?' + or '#'. + properties: + paths: + description: paths is a collection of paths + that map requests to backends. + items: + description: |- + HTTPIngressPath associates a path with a backend. Incoming urls matching the + path are forwarded to the backend. + properties: + backend: + description: |- + backend defines the referenced service endpoint to which the traffic + will be forwarded to. + properties: + resource: + description: |- + resource is an ObjectRef to another Kubernetes resource in the namespace + of the Ingress object. If resource is specified, a service.Name and + service.Port must not be specified. + This is a mutually exclusive setting with "Service". + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type + of resource being referenced + type: string + name: + description: Name is the name + of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + service: + description: |- + service references a service as a backend. + This is a mutually exclusive setting with "Resource". + properties: + name: + description: |- + name is the referenced service. The service must exist in + the same namespace as the Ingress object. + type: string + port: + description: |- + port of the referenced service. A port name or port number + is required for a IngressServiceBackend. + properties: + name: + description: |- + name is the name of the port on the Service. + This is a mutually exclusive setting with "Number". + type: string + number: + description: |- + number is the numerical port number (e.g. 80) on the Service. + This is a mutually exclusive setting with "Name". + format: int32 + type: integer + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + type: object + path: + description: |- + path is matched against the path of an incoming request. Currently it can + contain characters disallowed from the conventional "path" part of a URL + as defined by RFC 3986. Paths must begin with a '/' and must be present + when using PathType with value "Exact" or "Prefix". + type: string + pathType: + description: |- + pathType determines the interpretation of the path matching. PathType can + be one of the following values: + * Exact: Matches the URL path exactly. + * Prefix: Matches based on a URL path prefix split by '/'. Matching is + done on a path element by element basis. A path element refers is the + list of labels in the path split by the '/' separator. A request is a + match for path p if every p is an element-wise prefix of p of the + request path. Note that if the last element of the path is a substring + of the last element in request path, it is not a match (e.g. /foo/bar + matches /foo/bar/baz, but does not match /foo/barbaz). + * ImplementationSpecific: Interpretation of the Path matching is up to + the IngressClass. Implementations can treat this as a separate PathType + or treat it identically to Prefix or Exact path types. + Implementations are required to support all path types. + type: string + required: + - backend + - pathType + type: object + type: array + x-kubernetes-list-type: atomic + required: + - paths + type: object + type: object + type: array + x-kubernetes-list-type: atomic + tls: + description: |- + tls represents the TLS configuration. Currently the Ingress only supports a + single TLS port, 443. If multiple members of this list specify different hosts, + they will be multiplexed on the same port according to the hostname specified + through the SNI TLS extension, if the ingress controller fulfilling the + ingress supports SNI. + items: + description: IngressTLS describes the transport layer + security associated with an ingress. + properties: + hosts: + description: |- + hosts is a list of hosts included in the TLS certificate. The values in + this list must match the name/s used in the tlsSecret. Defaults to the + wildcard host setting for the loadbalancer controller fulfilling this + Ingress, if left unspecified. + items: + type: string + type: array + x-kubernetes-list-type: atomic + secretName: + description: |- + secretName is the name of the secret used to terminate TLS traffic on + port 443. Field is left optional to allow TLS routing based on SNI + hostname alone. If the SNI host in a listener conflicts with the "Host" + header field used by an IngressRule, the SNI host is used for termination + and value of the "Host" header is used for routing. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + service: + description: Service defines attributes to create a service + properties: + annotations: + additionalProperties: + type: string + type: object + name: + description: override the default service name + type: string + port: + default: 8000 + format: int32 + type: integer + type: + description: Service Type string describes ingress methods + for a service + type: string + required: + - port + type: object + type: object + groupID: + format: int64 + type: integer + image: + description: Image defines image attributes + properties: + pullPolicy: + type: string + pullSecrets: + items: + type: string + type: array + repository: + type: string + tag: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + livenessProbe: + description: Probe defines attributes for startup/liveness/readiness + probes + properties: + enabled: + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP + port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + metrics: + description: Metrics defines attributes to setup metrics collection + properties: + enabled: + type: boolean + serviceMonitor: + description: for use with the Prometheus Operator and the primary + service object + properties: + additionalLabels: + additionalProperties: + type: string + type: object + annotations: + additionalProperties: + type: string + type: object + interval: + description: |- + Duration is a valid time duration that can be parsed by Prometheus model.ParseDuration() function. + Supported units: y, w, d, h, m, s, ms + Examples: `30s`, `1m`, `1h20m15s`, `15d` + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + scrapeTimeout: + description: |- + Duration is a valid time duration that can be parsed by Prometheus model.ParseDuration() function. + Supported units: y, w, d, h, m, s, ms + Examples: `30s`, `1m`, `1h20m15s`, `15d` + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + type: object + type: object + nodeSelector: + additionalProperties: + type: string + type: object + podAffinity: + description: Pod affinity is a group of inter pod affinity scheduling + rules. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with + the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + readinessProbe: + description: Probe defines attributes for startup/liveness/readiness + probes + properties: + enabled: + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP + port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + replicas: + default: 1 + minimum: 1 + type: integer + resources: + description: ResourceRequirements describes the compute resource requirements. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + runtimeClass: + type: string + scale: + description: Autoscaling defines attributes to automatically scale + the service based on metrics + properties: + annotations: + additionalProperties: + type: string + type: object + enabled: + type: boolean + hpa: + description: HorizontalPodAutoscalerSpec defines the parameters + required to setup HPA + properties: + behavior: + description: |- + HorizontalPodAutoscalerBehavior configures the scaling behavior of the target + in both Up and Down directions (scaleUp and scaleDown fields respectively). + properties: + scaleDown: + description: |- + scaleDown is scaling policy for scaling Down. + If not set, the default value is to allow to scale down to minReplicas pods, with a + 300 second stabilization window (i.e., the highest recommendation for + the last 300sec is used). + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the scaling + policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + type: object + scaleUp: + description: |- + scaleUp is scaling policy for scaling Up. + If not set, the default value is the higher of: + * increase no more than 4 pods per 60 seconds + * double the number of pods per 60 seconds + No stabilization is used. + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the scaling + policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + type: object + type: object + maxReplicas: + format: int32 + type: integer + metrics: + items: + description: |- + MetricSpec specifies how to scale based on a single metric + (only `type` and one other matching field should be set at once). + properties: + containerResource: + description: |- + containerResource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing a single container in + each pod of the current scale target (e.g. CPU or memory). Such metrics are + built in to Kubernetes, and have special scaling options on top of those + available to normal per-pod metrics using the "pods" source. + This is an alpha feature and can be enabled by the HPAContainerMetrics feature flag. + properties: + container: + description: container is the name of the container + in the pods of the scaling target + type: string + name: + description: name is the name of the resource in + question. + type: string + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: |- + external refers to a global metric that is not associated + with any Kubernetes object. It allows autoscaling based on information + coming from components running outside of cluster + (for example length of queue in cloud messaging service, or + QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: |- + object refers to a metric describing a single kubernetes object + (for example, hits-per-second on an Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions + of a object,such as kind,name apiVersion + properties: + apiVersion: + description: apiVersion is the API version of + the referent + type: string + kind: + description: 'kind is the kind of the referent; + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: |- + pods refers to a metric describing each pod in the current scale target + (for example, transactions-processed-per-second). The values will be + averaged together before being compared to the target value. + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: |- + resource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing each pod in the + current scale target (e.g. CPU or memory). Such metrics are built in to + Kubernetes, and have special scaling options on top of those available + to normal per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource in + question. + type: string + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: |- + type is the type of metric source. It should be one of "ContainerResource", "External", + "Object", "Pods" or "Resource", each mapping to a matching field in the object. + Note: "ContainerResource" type is available on when the feature-gate + HPAContainerMetrics is enabled + type: string + required: + - type + type: object + type: array + minReplicas: + format: int32 + type: integer + required: + - maxReplicas + type: object + type: object + startupProbe: + description: Probe defines attributes for startup/liveness/readiness + probes + properties: + enabled: + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP + port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + userID: + format: int64 + type: integer + required: + - authSecret + - dataStoreParams + type: object + status: + description: NemoDatastoreV2Status defines the observed state of NemoDatastoreV2 + properties: + availableReplicas: + format: int32 + type: integer + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + state: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index f923e1fd..3143c1f1 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -63,6 +63,7 @@ rules: - apps.nvidia.com resources: - nemodatastores + - nemodatastorev2s - nemoguardrails - nimcaches - nimpipelines @@ -79,6 +80,7 @@ rules: - apps.nvidia.com resources: - nemodatastores/finalizers + - nemodatastorev2s/finalizers - nemoguardrails/finalizers - nimcaches/finalizers - nimpipelines/finalizers @@ -89,6 +91,7 @@ rules: - apps.nvidia.com resources: - nemodatastores/status + - nemodatastorev2s/status - nemoguardrails/status - nimcaches/status - nimpipelines/status diff --git a/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemodatastorev2s.yaml b/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemodatastorev2s.yaml new file mode 100644 index 00000000..36a7c940 --- /dev/null +++ b/deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nemodatastorev2s.yaml @@ -0,0 +1,2236 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.2 + name: nemodatastorev2s.apps.nvidia.com +spec: + group: apps.nvidia.com + names: + kind: NemoDatastoreV2 + listKind: NemoDatastoreV2List + plural: nemodatastorev2s + singular: nemodatastorev2 + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.state + name: Status + type: string + - format: date-time + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: NemoDatastoreV2 is the Schema for the NemoDatastoreV2 API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NemoDatastoreV2Spec defines the desired state of NemoDatastoreV2 + properties: + annotations: + additionalProperties: + type: string + type: object + args: + items: + type: string + type: array + authSecret: + description: The name of an secret that contains authn for the NGC + NIM service API + type: string + command: + items: + type: string + type: array + dataStoreParams: + properties: + appVersion: + type: string + databaseHost: + type: string + databasePort: + type: string + databaseURL: + type: string + datastoreConfigSecret: + type: string + datastoreInitSecret: + type: string + datastoreInlineConfigSecret: + type: string + datastoreSettingsSecret: + type: string + dbSecret: + type: string + envConfigmap: + type: string + envSecret: + type: string + giteaAdminSecret: + type: string + initContainerCommand: + items: + type: string + type: array + initContainerImage: + type: string + lfsJwtSecret: + type: string + objStoreSecret: + type: string + pvcSharedData: + type: string + size: + type: string + sshEnabled: + type: boolean + storageClass: + type: string + required: + - appVersion + - databaseHost + - databasePort + - databaseURL + - datastoreConfigSecret + - datastoreInitSecret + - datastoreInlineConfigSecret + - datastoreSettingsSecret + - dbSecret + - envConfigmap + - envSecret + - giteaAdminSecret + - lfsJwtSecret + - objStoreSecret + - pvcSharedData + - sshEnabled + - storageClass + type: object + env: + items: + description: EnvVar represents an environment variable present in + a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot + be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + expose: + description: Expose defines attributes to expose the service + properties: + ingress: + description: Ingress defines attributes to enable ingress for + the service + properties: + annotations: + additionalProperties: + type: string + type: object + enabled: + description: ingress, or virtualService - not both + type: boolean + spec: + description: IngressSpec describes the Ingress the user wishes + to exist. + properties: + defaultBackend: + description: |- + defaultBackend is the backend that should handle requests that don't + match any rule. If Rules are not specified, DefaultBackend must be specified. + If DefaultBackend is not set, the handling of requests that do not match any + of the rules will be up to the Ingress controller. + properties: + resource: + description: |- + resource is an ObjectRef to another Kubernetes resource in the namespace + of the Ingress object. If resource is specified, a service.Name and + service.Port must not be specified. + This is a mutually exclusive setting with "Service". + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + service: + description: |- + service references a service as a backend. + This is a mutually exclusive setting with "Resource". + properties: + name: + description: |- + name is the referenced service. The service must exist in + the same namespace as the Ingress object. + type: string + port: + description: |- + port of the referenced service. A port name or port number + is required for a IngressServiceBackend. + properties: + name: + description: |- + name is the name of the port on the Service. + This is a mutually exclusive setting with "Number". + type: string + number: + description: |- + number is the numerical port number (e.g. 80) on the Service. + This is a mutually exclusive setting with "Name". + format: int32 + type: integer + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + type: object + ingressClassName: + description: |- + ingressClassName is the name of an IngressClass cluster resource. Ingress + controller implementations use this field to know whether they should be + serving this Ingress resource, by a transitive connection + (controller -> IngressClass -> Ingress resource). Although the + `kubernetes.io/ingress.class` annotation (simple constant name) was never + formally defined, it was widely supported by Ingress controllers to create + a direct binding between Ingress controller and Ingress resources. Newly + created Ingress resources should prefer using the field. However, even + though the annotation is officially deprecated, for backwards compatibility + reasons, ingress controllers should still honor that annotation if present. + type: string + rules: + description: |- + rules is a list of host rules used to configure the Ingress. If unspecified, + or no rule matches, all traffic is sent to the default backend. + items: + description: |- + IngressRule represents the rules mapping the paths under a specified host to + the related backend services. Incoming requests are first evaluated for a host + match, then routed to the backend associated with the matching IngressRuleValue. + properties: + host: + description: "host is the fully qualified domain + name of a network host, as defined by RFC 3986.\nNote + the following deviations from the \"host\" part + of the\nURI as defined in RFC 3986:\n1. IPs are + not allowed. Currently an IngressRuleValue can + only apply to\n the IP in the Spec of the parent + Ingress.\n2. The `:` delimiter is not respected + because ports are not allowed.\n\t Currently + the port of an Ingress is implicitly :80 for http + and\n\t :443 for https.\nBoth these may change + in the future.\nIncoming requests are matched + against the host before the\nIngressRuleValue. + If the host is unspecified, the Ingress routes + all\ntraffic based on the specified IngressRuleValue.\n\nhost + can be \"precise\" which is a domain name without + the terminating dot of\na network host (e.g. \"foo.bar.com\") + or \"wildcard\", which is a domain name\nprefixed + with a single wildcard label (e.g. \"*.foo.com\").\nThe + wildcard character '*' must appear by itself as + the first DNS label and\nmatches only a single + label. You cannot have a wildcard label by itself + (e.g. Host == \"*\").\nRequests will be matched + against the Host field in the following way:\n1. + If host is precise, the request matches this rule + if the http host header is equal to Host.\n2. + If host is a wildcard, then the request matches + this rule if the http host header\nis to equal + to the suffix (removing the first label) of the + wildcard rule." + type: string + http: + description: |- + HTTPIngressRuleValue is a list of http selectors pointing to backends. + In the example: http:///? -> backend where + where parts of the url correspond to RFC 3986, this resource will be used + to match against everything after the last '/' and before the first '?' + or '#'. + properties: + paths: + description: paths is a collection of paths + that map requests to backends. + items: + description: |- + HTTPIngressPath associates a path with a backend. Incoming urls matching the + path are forwarded to the backend. + properties: + backend: + description: |- + backend defines the referenced service endpoint to which the traffic + will be forwarded to. + properties: + resource: + description: |- + resource is an ObjectRef to another Kubernetes resource in the namespace + of the Ingress object. If resource is specified, a service.Name and + service.Port must not be specified. + This is a mutually exclusive setting with "Service". + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type + of resource being referenced + type: string + name: + description: Name is the name + of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + service: + description: |- + service references a service as a backend. + This is a mutually exclusive setting with "Resource". + properties: + name: + description: |- + name is the referenced service. The service must exist in + the same namespace as the Ingress object. + type: string + port: + description: |- + port of the referenced service. A port name or port number + is required for a IngressServiceBackend. + properties: + name: + description: |- + name is the name of the port on the Service. + This is a mutually exclusive setting with "Number". + type: string + number: + description: |- + number is the numerical port number (e.g. 80) on the Service. + This is a mutually exclusive setting with "Name". + format: int32 + type: integer + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + type: object + path: + description: |- + path is matched against the path of an incoming request. Currently it can + contain characters disallowed from the conventional "path" part of a URL + as defined by RFC 3986. Paths must begin with a '/' and must be present + when using PathType with value "Exact" or "Prefix". + type: string + pathType: + description: |- + pathType determines the interpretation of the path matching. PathType can + be one of the following values: + * Exact: Matches the URL path exactly. + * Prefix: Matches based on a URL path prefix split by '/'. Matching is + done on a path element by element basis. A path element refers is the + list of labels in the path split by the '/' separator. A request is a + match for path p if every p is an element-wise prefix of p of the + request path. Note that if the last element of the path is a substring + of the last element in request path, it is not a match (e.g. /foo/bar + matches /foo/bar/baz, but does not match /foo/barbaz). + * ImplementationSpecific: Interpretation of the Path matching is up to + the IngressClass. Implementations can treat this as a separate PathType + or treat it identically to Prefix or Exact path types. + Implementations are required to support all path types. + type: string + required: + - backend + - pathType + type: object + type: array + x-kubernetes-list-type: atomic + required: + - paths + type: object + type: object + type: array + x-kubernetes-list-type: atomic + tls: + description: |- + tls represents the TLS configuration. Currently the Ingress only supports a + single TLS port, 443. If multiple members of this list specify different hosts, + they will be multiplexed on the same port according to the hostname specified + through the SNI TLS extension, if the ingress controller fulfilling the + ingress supports SNI. + items: + description: IngressTLS describes the transport layer + security associated with an ingress. + properties: + hosts: + description: |- + hosts is a list of hosts included in the TLS certificate. The values in + this list must match the name/s used in the tlsSecret. Defaults to the + wildcard host setting for the loadbalancer controller fulfilling this + Ingress, if left unspecified. + items: + type: string + type: array + x-kubernetes-list-type: atomic + secretName: + description: |- + secretName is the name of the secret used to terminate TLS traffic on + port 443. Field is left optional to allow TLS routing based on SNI + hostname alone. If the SNI host in a listener conflicts with the "Host" + header field used by an IngressRule, the SNI host is used for termination + and value of the "Host" header is used for routing. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + service: + description: Service defines attributes to create a service + properties: + annotations: + additionalProperties: + type: string + type: object + name: + description: override the default service name + type: string + port: + default: 8000 + format: int32 + type: integer + type: + description: Service Type string describes ingress methods + for a service + type: string + required: + - port + type: object + type: object + groupID: + format: int64 + type: integer + image: + description: Image defines image attributes + properties: + pullPolicy: + type: string + pullSecrets: + items: + type: string + type: array + repository: + type: string + tag: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + livenessProbe: + description: Probe defines attributes for startup/liveness/readiness + probes + properties: + enabled: + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP + port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + metrics: + description: Metrics defines attributes to setup metrics collection + properties: + enabled: + type: boolean + serviceMonitor: + description: for use with the Prometheus Operator and the primary + service object + properties: + additionalLabels: + additionalProperties: + type: string + type: object + annotations: + additionalProperties: + type: string + type: object + interval: + description: |- + Duration is a valid time duration that can be parsed by Prometheus model.ParseDuration() function. + Supported units: y, w, d, h, m, s, ms + Examples: `30s`, `1m`, `1h20m15s`, `15d` + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + scrapeTimeout: + description: |- + Duration is a valid time duration that can be parsed by Prometheus model.ParseDuration() function. + Supported units: y, w, d, h, m, s, ms + Examples: `30s`, `1m`, `1h20m15s`, `15d` + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + type: object + type: object + nodeSelector: + additionalProperties: + type: string + type: object + podAffinity: + description: Pod affinity is a group of inter pod affinity scheduling + rules. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with + the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + readinessProbe: + description: Probe defines attributes for startup/liveness/readiness + probes + properties: + enabled: + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP + port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + replicas: + default: 1 + minimum: 1 + type: integer + resources: + description: ResourceRequirements describes the compute resource requirements. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + runtimeClass: + type: string + scale: + description: Autoscaling defines attributes to automatically scale + the service based on metrics + properties: + annotations: + additionalProperties: + type: string + type: object + enabled: + type: boolean + hpa: + description: HorizontalPodAutoscalerSpec defines the parameters + required to setup HPA + properties: + behavior: + description: |- + HorizontalPodAutoscalerBehavior configures the scaling behavior of the target + in both Up and Down directions (scaleUp and scaleDown fields respectively). + properties: + scaleDown: + description: |- + scaleDown is scaling policy for scaling Down. + If not set, the default value is to allow to scale down to minReplicas pods, with a + 300 second stabilization window (i.e., the highest recommendation for + the last 300sec is used). + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the scaling + policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + type: object + scaleUp: + description: |- + scaleUp is scaling policy for scaling Up. + If not set, the default value is the higher of: + * increase no more than 4 pods per 60 seconds + * double the number of pods per 60 seconds + No stabilization is used. + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the scaling + policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + type: object + type: object + maxReplicas: + format: int32 + type: integer + metrics: + items: + description: |- + MetricSpec specifies how to scale based on a single metric + (only `type` and one other matching field should be set at once). + properties: + containerResource: + description: |- + containerResource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing a single container in + each pod of the current scale target (e.g. CPU or memory). Such metrics are + built in to Kubernetes, and have special scaling options on top of those + available to normal per-pod metrics using the "pods" source. + This is an alpha feature and can be enabled by the HPAContainerMetrics feature flag. + properties: + container: + description: container is the name of the container + in the pods of the scaling target + type: string + name: + description: name is the name of the resource in + question. + type: string + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: |- + external refers to a global metric that is not associated + with any Kubernetes object. It allows autoscaling based on information + coming from components running outside of cluster + (for example length of queue in cloud messaging service, or + QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: |- + object refers to a metric describing a single kubernetes object + (for example, hits-per-second on an Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions + of a object,such as kind,name apiVersion + properties: + apiVersion: + description: apiVersion is the API version of + the referent + type: string + kind: + description: 'kind is the kind of the referent; + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: |- + pods refers to a metric describing each pod in the current scale target + (for example, transactions-processed-per-second). The values will be + averaged together before being compared to the target value. + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: |- + resource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing each pod in the + current scale target (e.g. CPU or memory). Such metrics are built in to + Kubernetes, and have special scaling options on top of those available + to normal per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource in + question. + type: string + target: + description: target specifies the target value for + the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the + metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: |- + type is the type of metric source. It should be one of "ContainerResource", "External", + "Object", "Pods" or "Resource", each mapping to a matching field in the object. + Note: "ContainerResource" type is available on when the feature-gate + HPAContainerMetrics is enabled + type: string + required: + - type + type: object + type: array + minReplicas: + format: int32 + type: integer + required: + - maxReplicas + type: object + type: object + startupProbe: + description: Probe defines attributes for startup/liveness/readiness + probes + properties: + enabled: + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP + port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + userID: + format: int64 + type: integer + required: + - authSecret + - dataStoreParams + type: object + status: + description: NemoDatastoreV2Status defines the observed state of NemoDatastoreV2 + properties: + availableReplicas: + format: int32 + type: integer + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + state: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/internal/controller/nemo_datastorev2_controller.go b/internal/controller/nemo_datastorev2_controller.go new file mode 100644 index 00000000..3f2ead39 --- /dev/null +++ b/internal/controller/nemo_datastorev2_controller.go @@ -0,0 +1,642 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "reflect" + + "k8s.io/apimachinery/pkg/api/resource" + + appsv1alpha1 "github.com/NVIDIA/k8s-nim-operator/api/apps/v1alpha1" + "github.com/NVIDIA/k8s-nim-operator/internal/conditions" + "github.com/NVIDIA/k8s-nim-operator/internal/k8sutil" + "github.com/NVIDIA/k8s-nim-operator/internal/render" + "github.com/NVIDIA/k8s-nim-operator/internal/shared" + "github.com/NVIDIA/k8s-nim-operator/internal/utils" + "github.com/go-logr/logr" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +// NemoDatastoreV2Finalizer is the finalizer annotation +const NemoDatastoreV2Finalizer = "finalizer.NemoDatastoreV2.apps.nvidia.com" + +// NemoDatastoreV2Reconciler reconciles a NemoDatastoreV2 object +type NemoDatastoreV2Reconciler struct { + client.Client + scheme *runtime.Scheme + log logr.Logger + updater conditions.Updater + renderer render.Renderer + Config *rest.Config + recorder record.EventRecorder + orchestratorType k8sutil.OrchestratorType +} + +// Ensure NemoDatastoreV2Reconciler implements the Reconciler interface +var _ shared.Reconciler = &NemoDatastoreV2Reconciler{} + +// NewNemoDatastoreV2Reconciler creates a new reconciler for NemoDatastoreV2 with the given platform +func NewNemoDatastoreV2Reconciler(client client.Client, scheme *runtime.Scheme, updater conditions.Updater, renderer render.Renderer, log logr.Logger) *NemoDatastoreV2Reconciler { + return &NemoDatastoreV2Reconciler{ + Client: client, + scheme: scheme, + updater: updater, + renderer: renderer, + log: log, + } +} + +// +kubebuilder:rbac:groups=apps.nvidia.com,resources=nemodatastorev2s,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps.nvidia.com,resources=nemodatastorev2s/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=apps.nvidia.com,resources=nemodatastorev2s/finalizers,verbs=update +// +kubebuilder:rbac:groups=config.openshift.io,resources=clusterversions;proxies,verbs=get;list;watch +// +kubebuilder:rbac:groups=security.openshift.io,resources=securitycontextconstraints,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=security.openshift.io,resources=securitycontextconstraints,verbs=use,resourceNames=nonroot +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=serviceaccounts;pods;pods/eviction;services;services/finalizers;endpoints,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=persistentvolumeclaims;configmaps;secrets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps,resources=deployments;statefulsets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors;prometheusrules,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=scheduling.k8s.io,resources=priorityclasses,verbs=get;list;watch;create +// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalars,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=events,verbs=create;update;patch + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the NemoDatastoreV2 object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.2/pkg/reconcile +func (r *NemoDatastoreV2Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + defer r.refreshMetrics(ctx) + + // Fetch the NemoDatastoreV2 instance + NemoDatastoreV2 := &appsv1alpha1.NemoDatastoreV2{} + if err := r.Get(ctx, req.NamespacedName, NemoDatastoreV2); err != nil { + if client.IgnoreNotFound(err) != nil { + logger.Error(err, "unable to fetch NemoDatastoreV2", "name", req.NamespacedName) + } + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + logger.Info("Reconciling", "NemoDatastoreV2", NemoDatastoreV2.Name) + + // Check if the instance is marked for deletion + if NemoDatastoreV2.ObjectMeta.DeletionTimestamp.IsZero() { + // Add finalizer if not present + if !controllerutil.ContainsFinalizer(NemoDatastoreV2, NemoDatastoreV2Finalizer) { + controllerutil.AddFinalizer(NemoDatastoreV2, NemoDatastoreV2Finalizer) + if err := r.Update(ctx, NemoDatastoreV2); err != nil { + return ctrl.Result{}, err + } + } + } else { + // The instance is being deleted + if controllerutil.ContainsFinalizer(NemoDatastoreV2, NemoDatastoreV2Finalizer) { + // Perform platform specific cleanup of resources + if err := r.cleanupNemoDatastoreV2(ctx, NemoDatastoreV2); err != nil { + r.GetEventRecorder().Eventf(NemoDatastoreV2, corev1.EventTypeNormal, "Delete", + "NemoDatastoreV2 %s in deleted", NemoDatastoreV2.Name) + return ctrl.Result{}, err + } + + // Remove finalizer to allow for deletion + controllerutil.RemoveFinalizer(NemoDatastoreV2, NemoDatastoreV2Finalizer) + if err := r.Update(ctx, NemoDatastoreV2); err != nil { + r.GetEventRecorder().Eventf(NemoDatastoreV2, corev1.EventTypeNormal, "Delete", + "NemoDatastoreV2 %s finalizer removed", NemoDatastoreV2.Name) + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } + } + + // Fetch container orchestrator type + _, err := r.GetOrchestratorType() + if err != nil { + return ctrl.Result{}, fmt.Errorf("Unable to get container orchestrator type, %v", err) + } + + // Handle platform-specific reconciliation + if result, err := r.reconcileNemoDatastoreV2(ctx, NemoDatastoreV2); err != nil { + logger.Error(err, "error reconciling NemoDatastoreV2", "name", NemoDatastoreV2.Name) + return result, err + } + + return ctrl.Result{}, nil +} + +func (r *NemoDatastoreV2Reconciler) cleanupNemoDatastoreV2(ctx context.Context, nimService *appsv1alpha1.NemoDatastoreV2) error { + // All dependent (owned) objects will be automatically garbage collected. + // TODO: Handle any custom cleanup logic for the NIM microservice + return nil +} + +// GetScheme returns the scheme of the reconciler +func (r *NemoDatastoreV2Reconciler) GetScheme() *runtime.Scheme { + return r.scheme +} + +// GetLogger returns the logger of the reconciler +func (r *NemoDatastoreV2Reconciler) GetLogger() logr.Logger { + return r.log +} + +// GetClient returns the client instance +func (r *NemoDatastoreV2Reconciler) GetClient() client.Client { + return r.Client +} + +// GetConfig returns the rest config +func (r *NemoDatastoreV2Reconciler) GetConfig() *rest.Config { + return r.Config +} + +// GetUpdater returns the conditions updater instance +func (r *NemoDatastoreV2Reconciler) GetUpdater() conditions.Updater { + return r.updater +} + +// GetRenderer returns the renderer instance +func (r *NemoDatastoreV2Reconciler) GetRenderer() render.Renderer { + return r.renderer +} + +// GetEventRecorder returns the event recorder +func (r *NemoDatastoreV2Reconciler) GetEventRecorder() record.EventRecorder { + return r.recorder +} + +// GetOrchestratorType returns the container platform type +func (r *NemoDatastoreV2Reconciler) GetOrchestratorType() (k8sutil.OrchestratorType, error) { + if r.orchestratorType == "" { + orchestratorType, err := k8sutil.GetOrchestratorType(r.GetClient()) + if err != nil { + return k8sutil.Unknown, fmt.Errorf("Unable to get container orchestrator type, %v", err) + } + r.orchestratorType = orchestratorType + r.GetLogger().Info("Container orchestrator is successfully set", "type", orchestratorType) + } + return r.orchestratorType, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *NemoDatastoreV2Reconciler) SetupWithManager(mgr ctrl.Manager) error { + r.recorder = mgr.GetEventRecorderFor("nemo-datastore-service-controller") + return ctrl.NewControllerManagedBy(mgr). + For(&appsv1alpha1.NemoDatastoreV2{}). + Owns(&appsv1.Deployment{}). + Owns(&appsv1.StatefulSet{}). + Owns(&corev1.Service{}). + Owns(&corev1.ServiceAccount{}). + Owns(&rbacv1.Role{}). + Owns(&rbacv1.RoleBinding{}). + Owns(&networkingv1.Ingress{}). + Owns(&autoscalingv2.HorizontalPodAutoscaler{}). + WithEventFilter(predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + // Type assert to NemoDatastoreV2 + if oldNemoDatastoreV2, ok := e.ObjectOld.(*appsv1alpha1.NemoDatastoreV2); ok { + newNemoDatastoreV2 := e.ObjectNew.(*appsv1alpha1.NemoDatastoreV2) + + // Handle case where object is marked for deletion + if !newNemoDatastoreV2.ObjectMeta.DeletionTimestamp.IsZero() { + return true + } + + // Handle only spec updates + return !reflect.DeepEqual(oldNemoDatastoreV2.Spec, newNemoDatastoreV2.Spec) + } + // For other types we watch, reconcile them + return true + }, + }). + Complete(r) +} + +func (r *NemoDatastoreV2Reconciler) refreshMetrics(ctx context.Context) { + logger := log.FromContext(ctx) + // List all nodes + NemoDatastoreV2List := &appsv1alpha1.NemoDatastoreV2List{} + err := r.Client.List(ctx, NemoDatastoreV2List, &client.ListOptions{}) + if err != nil { + logger.Error(err, "unable to list NemoDatastoreV2s in the cluster") + return + } + //refreshNemoDatastoreV2Metrics(NemoDatastoreV2List) +} + +func (r *NemoDatastoreV2Reconciler) reconcileNemoDatastoreV2(ctx context.Context, nemoDatastore *appsv1alpha1.NemoDatastoreV2) (ctrl.Result, error) { + logger := log.FromContext(ctx) + var err error + defer func() { + if err != nil { + r.GetEventRecorder().Eventf(nemoDatastore, corev1.EventTypeWarning, conditions.Failed, + "NemoDatastoreV2 %s failed, msg: %s", nemoDatastore.Name, err.Error()) + } + }() + + pvc := corev1.PersistentVolumeClaim{} + pvc.Name = nemoDatastore.Spec.DataStoreParams.PVCSharedData + pvc.Spec.StorageClassName = &nemoDatastore.Spec.DataStoreParams.StorageClass + size, err := resource.ParseQuantity(nemoDatastore.Spec.DataStoreParams.Size) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to parse size for pvc creation %s, err %v", nemoDatastore.Name, err) + } + pvc.Spec.Resources = corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: size, + }, + } + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + pvc.Namespace = nemoDatastore.Namespace + + err = r.Create(ctx, &pvc) + if err != nil && !errors.IsAlreadyExists(err) { + logger.Error(err, "Failed to create pvc", "name", pvc.Name) + } + + // Generate annotation for the current operator-version and apply to all resources + // Get generic name for all resources + namespacedName := types.NamespacedName{Name: nemoDatastore.GetName(), Namespace: nemoDatastore.GetNamespace()} + + renderer := r.GetRenderer() + + // Sync serviceaccount + err = r.renderAndSyncResource(ctx, nemoDatastore, &renderer, &corev1.ServiceAccount{}, func() (client.Object, error) { + return renderer.ServiceAccount(nemoDatastore.GetServiceAccountParams()) + }, "serviceaccount", conditions.ReasonServiceAccountFailed) + if err != nil { + return ctrl.Result{}, err + } + + // Sync role + err = r.renderAndSyncResource(ctx, nemoDatastore, &renderer, &rbacv1.Role{}, func() (client.Object, error) { + return renderer.Role(nemoDatastore.GetRoleParams()) + }, "role", conditions.ReasonRoleFailed) + if err != nil { + return ctrl.Result{}, err + } + + // Sync rolebinding + err = r.renderAndSyncResource(ctx, nemoDatastore, &renderer, &rbacv1.RoleBinding{}, func() (client.Object, error) { + return renderer.RoleBinding(nemoDatastore.GetRoleBindingParams()) + }, "rolebinding", conditions.ReasonRoleBindingFailed) + if err != nil { + return ctrl.Result{}, err + } + + // Sync service + err = r.renderAndSyncResource(ctx, nemoDatastore, &renderer, &corev1.Service{}, func() (client.Object, error) { + return renderer.Service(nemoDatastore.GetServiceParams()) + }, "service", conditions.ReasonServiceFailed) + if err != nil { + return ctrl.Result{}, err + } + + // Sync ingress + if nemoDatastore.IsIngressEnabled() { + err = r.renderAndSyncResource(ctx, nemoDatastore, &renderer, &networkingv1.Ingress{}, func() (client.Object, error) { + return renderer.Ingress(nemoDatastore.GetIngressParams()) + }, "ingress", conditions.ReasonIngressFailed) + if err != nil { + return ctrl.Result{}, err + } + } else { + err = r.cleanupResource(ctx, &networkingv1.Ingress{}, namespacedName) + if err != nil && !errors.IsNotFound(err) { + return ctrl.Result{}, err + } + } + + // Sync HPA + if nemoDatastore.IsAutoScalingEnabled() { + err = r.renderAndSyncResource(ctx, nemoDatastore, &renderer, &autoscalingv2.HorizontalPodAutoscaler{}, func() (client.Object, error) { + return renderer.HPA(nemoDatastore.GetHPAParams()) + }, "hpa", conditions.ReasonHPAFailed) + if err != nil { + return ctrl.Result{}, err + } + } else { + // If autoscaling is disabled, ensure the HPA is deleted + err = r.cleanupResource(ctx, &autoscalingv2.HorizontalPodAutoscaler{}, namespacedName) + if err != nil { + return ctrl.Result{}, err + } + } + + // Sync Service Monitor + if nemoDatastore.IsServiceMonitorEnabled() { + err = r.renderAndSyncResource(ctx, nemoDatastore, &renderer, &monitoringv1.ServiceMonitor{}, func() (client.Object, error) { + return renderer.ServiceMonitor(nemoDatastore.GetServiceMonitorParams()) + }, "servicemonitor", conditions.ReasonServiceMonitorFailed) + if err != nil { + return ctrl.Result{}, err + } + } + + deploymentParams := nemoDatastore.GetDeploymentParams() + + // Setup volume mounts with model store + deploymentParams.Volumes = nemoDatastore.GetVolumes() + deploymentParams.VolumeMounts = nemoDatastore.GetVolumeMounts() + + logger.Info("Reconciling", "volumes", nemoDatastore.GetVolumes()) + + // Sync deployment + err = r.renderAndSyncResource(ctx, nemoDatastore, &renderer, &appsv1.Deployment{}, func() (client.Object, error) { + result, err := renderer.Deployment(deploymentParams) + if err != nil { + return nil, err + } + initContainers := nemoDatastore.GetInitContainers() + if len(initContainers) > 0 { + result.Spec.Template.Spec.InitContainers = initContainers + } + envFrom := nemoDatastore.GetEnvFrom() + if len(envFrom) > 0 { + result.Spec.Template.Spec.Containers[0].EnvFrom = envFrom + } + fsGroup := int64(1000) + result.Spec.Template.Spec.SecurityContext = &corev1.PodSecurityContext{ + FSGroup: &fsGroup, + } + return result, nil + }, "deployment", conditions.ReasonDeploymentFailed) + if err != nil { + return ctrl.Result{}, err + } + + // Wait for deployment + msg, ready, err := r.isDeploymentReady(ctx, &namespacedName) + if err != nil { + return ctrl.Result{}, err + } + + if !ready { + // Update status as NotReady + err = r.SetConditionsNotReady(ctx, nemoDatastore, conditions.NotReady, msg) + r.GetEventRecorder().Eventf(nemoDatastore, corev1.EventTypeNormal, conditions.NotReady, + "NemoDatastoreV2 %s not ready yet, msg: %s", nemoDatastore.Name, msg) + } else { + // Update status as ready + err = r.SetConditionsReady(ctx, nemoDatastore, conditions.Ready, msg) + r.GetEventRecorder().Eventf(nemoDatastore, corev1.EventTypeNormal, conditions.Ready, + "NemoDatastoreV2 %s ready, msg: %s", nemoDatastore.Name, msg) + } + + if err != nil { + logger.Error(err, "Unable to update status") + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil +} + +func (r *NemoDatastoreV2Reconciler) renderAndSyncResource(ctx context.Context, NemoDatastoreV2 client.Object, renderer *render.Renderer, obj client.Object, renderFunc func() (client.Object, error), conditionType string, reason string) error { + logger := log.FromContext(ctx) + + namespacedName := types.NamespacedName{Name: NemoDatastoreV2.GetName(), Namespace: NemoDatastoreV2.GetNamespace()} + + resource, err := renderFunc() + if err != nil { + logger.Error(err, "failed to render", conditionType, namespacedName) + statusError := r.updater.SetConditionsFailed(ctx, NemoDatastoreV2, reason, err.Error()) + if statusError != nil { + logger.Error(statusError, "failed to update status", "NemoDatastoreV2", NemoDatastoreV2.GetName()) + } + return err + } + + // Check if the resource is nil + if resource == nil { + logger.V(2).Info("rendered nil resource") + return nil + } + + metaAccessor, ok := resource.(metav1.Object) + if !ok { + logger.V(2).Info("rendered un-initialized resource") + return nil + } + + if metaAccessor == nil || metaAccessor.GetName() == "" || metaAccessor.GetNamespace() == "" { + logger.V(2).Info("rendered un-initialized resource") + return nil + } + + if err = controllerutil.SetControllerReference(NemoDatastoreV2, resource, r.GetScheme()); err != nil { + logger.Error(err, "failed to set owner", conditionType, namespacedName) + statusError := r.updater.SetConditionsFailed(ctx, NemoDatastoreV2, reason, err.Error()) + if statusError != nil { + logger.Error(statusError, "failed to update status", "NemoDatastoreV2", NemoDatastoreV2.GetName()) + } + return err + } + + err = r.syncResource(ctx, obj, resource, namespacedName) + if err != nil { + logger.Error(err, "failed to sync", conditionType, namespacedName) + statusError := r.updater.SetConditionsFailed(ctx, NemoDatastoreV2, reason, err.Error()) + if statusError != nil { + logger.Error(statusError, "failed to update status", "NemoDatastoreV2", NemoDatastoreV2.GetName()) + } + return err + } + return nil +} + +// CheckDeploymentReadiness checks if the Deployment is ready +func (r *NemoDatastoreV2Reconciler) isDeploymentReady(ctx context.Context, namespacedName *types.NamespacedName) (string, bool, error) { + deployment := &appsv1.Deployment{} + err := r.Get(ctx, client.ObjectKey{Name: namespacedName.Name, Namespace: namespacedName.Namespace}, deployment) + if err != nil { + if errors.IsNotFound(err) { + return "", false, nil + } + return "", false, err + } + + cond := getDeploymentCondition(deployment.Status, appsv1.DeploymentProgressing) + if cond != nil && cond.Reason == "ProgressDeadlineExceeded" { + return fmt.Sprintf("deployment %q exceeded its progress deadline", deployment.Name), false, nil + } + if deployment.Spec.Replicas != nil && deployment.Status.UpdatedReplicas < *deployment.Spec.Replicas { + return fmt.Sprintf("Waiting for deployment %q rollout to finish: %d out of %d new replicas have been updated...\n", deployment.Name, deployment.Status.UpdatedReplicas, *deployment.Spec.Replicas), false, nil + } + if deployment.Status.Replicas > deployment.Status.UpdatedReplicas { + return fmt.Sprintf("Waiting for deployment %q rollout to finish: %d old replicas are pending termination...\n", deployment.Name, deployment.Status.Replicas-deployment.Status.UpdatedReplicas), false, nil + } + if deployment.Status.AvailableReplicas < deployment.Status.UpdatedReplicas { + return fmt.Sprintf("Waiting for deployment %q rollout to finish: %d of %d updated replicas are available...\n", deployment.Name, deployment.Status.AvailableReplicas, deployment.Status.UpdatedReplicas), false, nil + } + return fmt.Sprintf("deployment %q successfully rolled out\n", deployment.Name), true, nil +} + +func (r *NemoDatastoreV2Reconciler) syncResource(ctx context.Context, obj client.Object, desired client.Object, namespacedName types.NamespacedName) error { + logger := log.FromContext(ctx) + + err := r.Get(ctx, namespacedName, obj) + if err != nil && !errors.IsNotFound(err) { + return err + } + + if !utils.IsSpecChanged(obj, desired) { + logger.V(2).Info("Object spec has not changed, skipping update", "obj", obj) + return nil + } + logger.V(2).Info("Object spec has changed, updating") + + if errors.IsNotFound(err) { + err = r.Create(ctx, desired) + if err != nil { + return err + } + } else { + err = r.Update(ctx, desired) + if err != nil { + return err + } + } + return nil +} + +// cleanupResource deletes the given Kubernetes resource if it exists. +// If the resource does not exist or an error occurs during deletion, the function returns nil or the error. +// +// Parameters: +// ctx (context.Context): The context for the operation. +// obj (client.Object): The Kubernetes resource to delete. +// namespacedName (types.NamespacedName): The namespaced name of the resource. +// +// Returns: +// error: An error if the resource deletion fails, or nil if the resource is not found or deletion is successful. +func (r *NemoDatastoreV2Reconciler) cleanupResource(ctx context.Context, obj client.Object, namespacedName types.NamespacedName) error { + + logger := log.FromContext(ctx) + + err := r.Get(ctx, namespacedName, obj) + if err != nil && !errors.IsNotFound(err) { + return err + } + + if errors.IsNotFound(err) { + return nil + } + + err = r.Delete(ctx, obj) + if err != nil { + return err + } + logger.V(2).Info("NIM Service object changed, deleting ", "obj", obj) + return nil +} + +func (r *NemoDatastoreV2Reconciler) SetConditionsReady(ctx context.Context, cr *appsv1alpha1.NemoDatastoreV2, reason, message string) error { + meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ + Type: conditions.Ready, + Status: metav1.ConditionTrue, + Reason: reason, + Message: message, + }) + + meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ + Type: conditions.Failed, + Status: metav1.ConditionFalse, + Reason: conditions.Ready, + }) + + cr.Status.State = appsv1alpha1.NemoDatastoreV2StatusReady + return r.updateNemoDatastoreV2Status(ctx, cr) +} + +func (r *NemoDatastoreV2Reconciler) SetConditionsNotReady(ctx context.Context, cr *appsv1alpha1.NemoDatastoreV2, reason, message string) error { + meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ + Type: conditions.Ready, + Status: metav1.ConditionFalse, + Reason: reason, + Message: message, + }) + + meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ + Type: conditions.Failed, + Status: metav1.ConditionFalse, + Reason: conditions.Ready, + Message: message, + }) + + cr.Status.State = appsv1alpha1.NemoDatastoreV2StatusNotReady + return r.updateNemoDatastoreV2Status(ctx, cr) +} + +func (r *NemoDatastoreV2Reconciler) SetConditionsFailed(ctx context.Context, cr *appsv1alpha1.NemoDatastoreV2, reason, message string) error { + meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ + Type: conditions.Ready, + Status: metav1.ConditionFalse, + Reason: conditions.Failed, + }) + + meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ + Type: conditions.Failed, + Status: metav1.ConditionTrue, + Reason: reason, + Message: message, + }) + cr.Status.State = appsv1alpha1.NemoDatastoreV2StatusFailed + return r.updateNemoDatastoreV2Status(ctx, cr) +} + +func (r *NemoDatastoreV2Reconciler) updateNemoDatastoreV2Status(ctx context.Context, cr *appsv1alpha1.NemoDatastoreV2) error { + + obj := &appsv1alpha1.NemoDatastoreV2{} + errGet := r.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: cr.GetNamespace()}, obj) + if errGet != nil { + return errGet + } + obj.Status = cr.Status + if err := r.Status().Update(ctx, obj); err != nil { + return err + } + return nil +} diff --git a/manifests/deployment.yaml b/manifests/deployment.yaml index 9094e9e9..670269fc 100644 --- a/manifests/deployment.yaml +++ b/manifests/deployment.yaml @@ -81,6 +81,11 @@ spec: configMap: name: {{ .ConfigMap.Name }} {{- end }} + {{- if .Secret }} + secret: + secretName: {{ .Secret.SecretName }} + defaultMode: {{ .Secret.DefaultMode }} + {{- end }} {{- if .EmptyDir }} emptyDir: medium: {{ .EmptyDir.Medium }}