From 51ef2189e11b7916750f58d62d1a9af3569b2bd8 Mon Sep 17 00:00:00 2001 From: David Grove Date: Mon, 16 Dec 2024 15:33:59 -0500 Subject: [PATCH] Instructions for RHOAI 2.16 (#112) --- SETUP.md | 6 + setup.RHOAI-v2.16/CLUSTER-SETUP.md | 149 +++++++++ setup.RHOAI-v2.16/TEAM-SETUP.md | 91 +++++ setup.RHOAI-v2.16/UNINSTALL.md | 23 ++ setup.RHOAI-v2.16/UPGRADE-FAST.md | 34 ++ setup.RHOAI-v2.16/UPGRADE-STABLE.md | 31 ++ .../coscheduler-priority-patch.yaml | 3 + setup.RHOAI-v2.16/default-flavor.yaml | 4 + setup.RHOAI-v2.16/mlbatch-dsc.yaml | 32 ++ setup.RHOAI-v2.16/mlbatch-dsci.yaml | 14 + setup.RHOAI-v2.16/mlbatch-edit-role.yaml | 151 +++++++++ setup.RHOAI-v2.16/mlbatch-priorities.yaml | 26 ++ setup.RHOAI-v2.16/mlbatch-subscription.yaml | 310 ++++++++++++++++++ .../mlbatch-upgrade-configmaps.yaml | 125 +++++++ setup.k8s-v1.27/CLUSTER-SETUP.md | 1 + .../kueue/controller_manager_config.yaml | 6 +- setup.k8s-v1.30/CLUSTER-SETUP.md | 2 + .../kueue/controller_manager_config.yaml | 6 +- setup.tmpl/CLUSTER-SETUP.md.tmpl | 10 +- setup.tmpl/Kubernetes-v1.27.yaml | 1 + setup.tmpl/Kubernetes-v1.30.yaml | 1 + setup.tmpl/Makefile | 2 + setup.tmpl/RHOAI-v2.10.yaml | 1 + setup.tmpl/RHOAI-v2.13.yaml | 3 +- setup.tmpl/RHOAI-v2.15.yaml | 3 +- setup.tmpl/RHOAI-v2.16.yaml | 7 + 26 files changed, 1032 insertions(+), 10 deletions(-) create mode 100644 setup.RHOAI-v2.16/CLUSTER-SETUP.md create mode 100644 setup.RHOAI-v2.16/TEAM-SETUP.md create mode 100644 setup.RHOAI-v2.16/UNINSTALL.md create mode 100644 setup.RHOAI-v2.16/UPGRADE-FAST.md create mode 100644 setup.RHOAI-v2.16/UPGRADE-STABLE.md create mode 100644 setup.RHOAI-v2.16/coscheduler-priority-patch.yaml create mode 100644 setup.RHOAI-v2.16/default-flavor.yaml create mode 100644 setup.RHOAI-v2.16/mlbatch-dsc.yaml create mode 100644 setup.RHOAI-v2.16/mlbatch-dsci.yaml create mode 100644 setup.RHOAI-v2.16/mlbatch-edit-role.yaml create mode 100644 setup.RHOAI-v2.16/mlbatch-priorities.yaml create mode 100644 setup.RHOAI-v2.16/mlbatch-subscription.yaml create mode 100644 setup.RHOAI-v2.16/mlbatch-upgrade-configmaps.yaml create mode 100644 setup.tmpl/RHOAI-v2.16.yaml diff --git a/SETUP.md b/SETUP.md index 9d06384..0003773 100644 --- a/SETUP.md +++ b/SETUP.md @@ -30,6 +30,12 @@ Red Hat OpenShift AI as the base platform for MLBatch. Please see for the life cycle dates of currently supported ***stable*** and ***fast*** releases. Instructions are provided for the following Red Hat OpenShift AI ***stable*** releases: ++ Red Hat OpenShift AI 2.16 + + [RHOAI 2.16 Cluster Setup](./setup.RHOAI-v2.16/CLUSTER-SETUP.md) + + [RHOAI 2.16 Team Setup](./setup.RHOAI-v2.16/TEAM-SETUP.md) + + [UPGRADING from RHOAI 2.13](./setup.RHOAI-v2.16/UPGRADE-STABLE.md) + + [UPGRADING from RHOAI 2.15](./setup.RHOAI-v2.16/UPGRADE-FAST.md) + + [RHOAI 2.16 Uninstall](./setup.RHOAI-v2.15/UNINSTALL.md) + Red Hat OpenShift AI 2.13 + [RHOAI 2.13 Cluster Setup](./setup.RHOAI-v2.13/CLUSTER-SETUP.md) + [RHOAI 2.13 Team Setup](./setup.RHOAI-v2.13/TEAM-SETUP.md) diff --git a/setup.RHOAI-v2.16/CLUSTER-SETUP.md b/setup.RHOAI-v2.16/CLUSTER-SETUP.md new file mode 100644 index 0000000..166c66c --- /dev/null +++ b/setup.RHOAI-v2.16/CLUSTER-SETUP.md @@ -0,0 +1,149 @@ +# Cluster Setup + +The cluster setup installs Red Hat OpenShift AI and Coscheduler, configures Kueue, +cluster roles, and priority classes. + +If MLBatch is deployed on a cluster that used to run earlier versions of ODH, +[MCAD](https://github.com/project-codeflare/mcad), Red Hat OpenShift AI, or Coscheduler, +make sure to scrub traces of these installations. In particular, make sure to +delete the following custom resource definitions (CRD) if present on the +cluster. Make sure to delete all instances prior to deleting the CRDs: +```sh +# Delete old appwrappers and crd +oc delete appwrappers --all -A +oc delete crd appwrappers.workload.codeflare.dev + +# Delete old noderesourcetopologies and crd +oc delete noderesourcetopologies --all -A +oc delete crd noderesourcetopologies.topology.node.k8s.io +``` + +## Priorities + +Create `default-priority`, `high-priority`, and `low-priority` priority classes: +```sh +oc apply -f setup.RHOAI-v2.16/mlbatch-priorities.yaml +``` + +## Coscheduler + +Install Coscheduler v0.28.9 as a secondary scheduler and configure packing: +```sh +helm install scheduler-plugins --namespace scheduler-plugins --create-namespace \ + scheduler-plugins/manifests/install/charts/as-a-second-scheduler/ \ + --set-json pluginConfig='[{"args":{"scoringStrategy":{"resources":[{"name":"nvidia.com/gpu","weight":1}],"requestedToCapacityRatio":{"shape":[{"utilization":0,"score":0},{"utilization":100,"score":10}]},"type":"RequestedToCapacityRatio"}},"name":"NodeResourcesFit"}]' +``` +Patch Coscheduler pod priorities: +```sh +oc patch deployment -n scheduler-plugins --type=json --patch-file setup.RHOAI-v2.16/coscheduler-priority-patch.yaml scheduler-plugins-controller +oc patch deployment -n scheduler-plugins --type=json --patch-file setup.RHOAI-v2.16/coscheduler-priority-patch.yaml scheduler-plugins-scheduler +``` + +## Red Hat OpenShift AI + +Create the Red Hat OpenShift AI subscription: +```sh +oc apply -f setup.RHOAI-v2.16/mlbatch-subscription.yaml +```` +Identify install plan: +```sh +oc get ip -n redhat-ods-operator +``` +``` +NAMESPACE NAME CSV APPROVAL APPROVED +redhat-ods-operator install-kmh8w rhods-operator.2.10.0 Manual false +``` +Approve install plan replacing the generated plan name below with the actual +value: +```sh +oc patch ip -n redhat-ods-operator --type merge --patch '{"spec":{"approved":true}}' install-kmh8w +``` +Create DSC Initialization: +```sh +oc apply -f setup.RHOAI-v2.16/mlbatch-dsci.yaml +``` +Create Data Science Cluster: +```sh +oc apply -f setup.RHOAI-v2.16/mlbatch-dsc.yaml +``` +The provided DSCI and DSC are intended to install a minimal set of Red Hat OpenShift +AI managed components: `codeflare`, `kueue`, `ray`, and `trainingoperator`. The +remaining components such as `dashboard` can be optionally enabled. + +The configuration of the managed components differs from the default Red Hat OpenShift +AI configuration as follows: +- Kubeflow Training Operator: + - `gang-scheduler-name` is set to `scheduler-plugins-scheduler`, +- Kueue: + - `manageJobsWithoutQueueName` is enabled, + - `batch/job` integration is disabled, + - `waitForPodsReady` is disabled, + - `LendingLimit` feature gate is enabled, + - `fairSharing` is enabled, + - `enableClusterQueueResources` metrics is enabled, +- Codeflare operator: + - the AppWrapper controller is enabled and configured as follows: + - `userRBACAdmissionCheck` is disabled, + - `schedulerName` is set to `scheduler-plugins-scheduler`, + - `queueName` is set to `default-queue`, + - `slackQueueName` is set to `slack-cluster-queue` +- pod priorities, resource requests and limits have been adjusted. + + + +## Kueue Configuration + +Create Kueue's default flavor: +```sh +oc apply -f setup.RHOAI-v2.16/default-flavor.yaml +``` + +## Cluster Role + +Create `mlbatch-edit` role: +```sh +oc apply -f setup.RHOAI-v2.16/mlbatch-edit-role.yaml +``` + +## Slack Cluster Queue + +Create the designated slack `ClusterQueue` which will be used to automate +minor adjustments to cluster capacity caused by node failures and +scheduler maintanence. +```sh +oc apply -f- << EOF +apiVersion: kueue.x-k8s.io/v1beta1 +kind: ClusterQueue +metadata: + name: slack-cluster-queue +spec: + namespaceSelector: {} + cohort: default-cohort + preemption: + withinClusterQueue: LowerOrNewerEqualPriority + reclaimWithinCohort: Any + borrowWithinCohort: + policy: Never + resourceGroups: + - coveredResources: ["cpu", "memory", "nvidia.com/gpu", "nvidia.com/roce_gdr", "pods"] + flavors: + - name: default-flavor + resources: + - name: "cpu" + nominalQuota: 8000m + - name: "memory" + nominalQuota: 128Gi + - name: "nvidia.com/gpu" + nominalQuota: 8 + - name: "nvidia.com/roce_gdr" + nominalQuota: 1 + - name: "pods" + nominalQuota: 100 +EOF +``` +Edit the above quantities to adjust the quota to the desired +values. Pod counts are optional and can be omitted from the list of +covered resources. The `lendingLimit` for each resource will be +dynamically adjusted by the MLBatch system to reflect reduced cluster +capacity. See [QUOTA_MAINTENANCE.md](../QUOTA_MAINTENANCE.md) for a +detailed discussion of the role of the slack `ClusterQueue`. diff --git a/setup.RHOAI-v2.16/TEAM-SETUP.md b/setup.RHOAI-v2.16/TEAM-SETUP.md new file mode 100644 index 0000000..85c9429 --- /dev/null +++ b/setup.RHOAI-v2.16/TEAM-SETUP.md @@ -0,0 +1,91 @@ +# Team Setup + +A *team* in MLBatch is a group of users that share a resource quota. + +Before setting up your teams and quotas, please read [QUOTA_MAINTENANCE.md](../QUOTA_MAINTENANCE.md) +for a discussion of our recommended best practices. + + +Setting up a new team requires the cluster admin to create a project, +a user group, a quota, a queue, and the required role bindings as described below. + +Create project: +```sh +oc new-project team1 +``` +Create user group: +```sh +oc adm groups new team1-edit-group +``` +Add users to group for example: +```sh +oc adm groups add-users team1-edit-group user1 +``` +Bind cluster role to group in namespace: +```sh +oc adm policy add-role-to-group mlbatch-edit team1-edit-group --role-namespace="" --namespace team1 +``` + +Specify the intended quota for the namespace by creating a `ClusterQueue`: +```sh +oc apply -f- << EOF +apiVersion: kueue.x-k8s.io/v1beta1 +kind: ClusterQueue +metadata: + name: team1-cluster-queue +spec: + namespaceSelector: {} + cohort: default-cohort + preemption: + withinClusterQueue: LowerOrNewerEqualPriority + reclaimWithinCohort: Any + borrowWithinCohort: + policy: Never + resourceGroups: + - coveredResources: ["cpu", "memory", "nvidia.com/gpu", "nvidia.com/roce_gdr", "pods"] + flavors: + - name: default-flavor + resources: + - name: "cpu" + nominalQuota: 8000m + # borrowingLimit: 0 + # lendingLimit: 0 + - name: "memory" + nominalQuota: 128Gi + # borrowingLimit: 0 + # lendingLimit: 0 + - name: "nvidia.com/gpu" + nominalQuota: 16 + # borrowingLimit: 0 + # lendingLimit: 0 + - name: "nvidia.com/roce_gdr" + nominalQuota: 4 + # borrowingLimit: 0 + # lendingLimit: 0 + - name: "pods" + nominalQuota: 100 + # borrowingLimit: 0 + # lendingLimit: 0 +EOF +``` +Edit the above quantities to adjust the quota to the desired values. Pod counts +are optional and can be omitted from the list of covered resources. + +Uncomment all `borrowingLimit` lines to prevent this namespace from borrowing +quota from other namespaces. Uncomment all `lendingLimit` lines to prevent other +namespaces from borrowing quota from this namespace. + +Create a `LocalQueue` to bind the `ClusterQueue` to the namespace: +```sh +oc apply -n team1 -f- << EOF +apiVersion: kueue.x-k8s.io/v1beta1 +kind: LocalQueue +metadata: + name: default-queue +spec: + clusterQueue: team1-cluster-queue +EOF +``` +We recommend naming the local queue `default-queue` as `AppWrappers` will +default to this queue name. + diff --git a/setup.RHOAI-v2.16/UNINSTALL.md b/setup.RHOAI-v2.16/UNINSTALL.md new file mode 100644 index 0000000..776045d --- /dev/null +++ b/setup.RHOAI-v2.16/UNINSTALL.md @@ -0,0 +1,23 @@ +# Uninstall + +***First, remove all team projects and corresponding cluster queues.*** + +Then to uninstall the MLBatch controllers and reclaim the corresponding +namespaces, run: +```sh +# OpenShift AI uninstall +oc delete dsc mlbatch-dsc +oc delete dsci mlbatch-dsci +oc delete subscription -n redhat-ods-operator rhods-operator +oc delete csv -n redhat-ods-operator -l operators.coreos.com/rhods-operator.redhat-ods-operator +oc delete crd featuretrackers.features.opendatahub.io \ + dscinitializations.dscinitialization.opendatahub.io \ + datascienceclusters.datasciencecluster.opendatahub.io +oc delete operators rhods-operator.redhat-ods-operator +oc delete operatorgroup -n redhat-ods-operator rhods-operator +oc delete namespace redhat-ods-applications redhat-ods-monitoring redhat-ods-operator + +# Coscheduler uninstall +helm uninstall -n scheduler-plugins scheduler-plugins +oc delete namespace scheduler-plugins +``` diff --git a/setup.RHOAI-v2.16/UPGRADE-FAST.md b/setup.RHOAI-v2.16/UPGRADE-FAST.md new file mode 100644 index 0000000..eeb9bb3 --- /dev/null +++ b/setup.RHOAI-v2.16/UPGRADE-FAST.md @@ -0,0 +1,34 @@ +# Upgrading from RHOAI 2.15 + +These instructions assume you installed and configured RHOAI 2.15 following +the MLBatch [install instructions for RHOAI-v2.15](../setup.RHOAI-v2.15/CLUSTER-SETUP.md) +or the [upgrade instructions for RHOAI-V2.15](../setup.RHOAI-v2.15/UPGRADE.md) + +Your subscription will have automatically created an unapproved +install plan to upgrade to RHOAI 2.16. + +Before beginning, verify that the expected install plan exists: +```sh +oc get ip -n redhat-ods-operator +``` +Typical output would be: +```sh +NAME CSV APPROVAL APPROVED +install-kpzzl rhods-operator.2.16.0 Manual false +install-nqrbp rhods-operator.2.15.0 Manual true +``` + +Assuming the install plan exists you can begin the upgrade process. + +First, update the MLBatch modifications to the default RHOAI configuration maps. +```sh +oc apply -f setup.RHOAI-v2.16/mlbatch-upgrade-configmaps.yaml +``` + +There are no MLBatch modifications to the default RHOAI configuration maps +beyond those already made in previous installs. Therefore, you can simply +approve the install plan replacing the example plan name below with the actual +value on your cluster: +```sh +oc patch ip -n redhat-ods-operator --type merge --patch '{"spec":{"approved":true}}' install-kpzzl +``` diff --git a/setup.RHOAI-v2.16/UPGRADE-STABLE.md b/setup.RHOAI-v2.16/UPGRADE-STABLE.md new file mode 100644 index 0000000..e17651e --- /dev/null +++ b/setup.RHOAI-v2.16/UPGRADE-STABLE.md @@ -0,0 +1,31 @@ +# Upgrading from RHOAI 2.13 + +These instructions assume you installed and configured RHOAI 2.13 following +the MLBatch [install instructions for RHOAI-v2.13](../setup.RHOAI-v2.13/CLUSTER-SETUP.md). + +Your subscription will have automatically created an unapproved +install plan to upgrade to RHOAI 2.16. + +Before beginning, verify that the expected install plan exists: +```sh +oc get ip -n redhat-ods-operator +``` +Typical output would be: +```sh +NAME CSV APPROVAL APPROVED +install-kpzzl rhods-operator.2.16.0 Manual false +install-nqrbp rhods-operator.2.13.0 Manual true +``` + +Assuming the install plan exists you can begin the upgrade process. + +First, update the MLBatch modifications to the default RHOAI configuration maps. +```sh +oc apply -f setup.RHOAI-v2.16/mlbatch-upgrade-configmaps.yaml +``` + +Second, approve the install plan replacing the example plan name below with the actual +value on your cluster: +```sh +oc patch ip -n redhat-ods-operator --type merge --patch '{"spec":{"approved":true}}' install-kpzzl +``` diff --git a/setup.RHOAI-v2.16/coscheduler-priority-patch.yaml b/setup.RHOAI-v2.16/coscheduler-priority-patch.yaml new file mode 100644 index 0000000..278802f --- /dev/null +++ b/setup.RHOAI-v2.16/coscheduler-priority-patch.yaml @@ -0,0 +1,3 @@ +- op: add + path: /spec/template/spec/priorityClassName + value: system-node-critical diff --git a/setup.RHOAI-v2.16/default-flavor.yaml b/setup.RHOAI-v2.16/default-flavor.yaml new file mode 100644 index 0000000..6cbccf3 --- /dev/null +++ b/setup.RHOAI-v2.16/default-flavor.yaml @@ -0,0 +1,4 @@ +apiVersion: kueue.x-k8s.io/v1beta1 +kind: ResourceFlavor +metadata: + name: default-flavor diff --git a/setup.RHOAI-v2.16/mlbatch-dsc.yaml b/setup.RHOAI-v2.16/mlbatch-dsc.yaml new file mode 100644 index 0000000..66336bc --- /dev/null +++ b/setup.RHOAI-v2.16/mlbatch-dsc.yaml @@ -0,0 +1,32 @@ +apiVersion: datasciencecluster.opendatahub.io/v1 +kind: DataScienceCluster +metadata: + name: mlbatch-dsc +spec: + components: + codeflare: + managementState: Managed + dashboard: + managementState: Removed + datasciencepipelines: + managementState: Removed + kserve: + managementState: Removed + serving: + ingressGateway: + certificate: + type: SelfSigned + managementState: Removed + name: knative-serving + kueue: + managementState: Managed + modelmeshserving: + managementState: Removed + ray: + managementState: Managed + trainingoperator: + managementState: Managed + trustyai: + managementState: Removed + workbenches: + managementState: Removed diff --git a/setup.RHOAI-v2.16/mlbatch-dsci.yaml b/setup.RHOAI-v2.16/mlbatch-dsci.yaml new file mode 100644 index 0000000..77785c3 --- /dev/null +++ b/setup.RHOAI-v2.16/mlbatch-dsci.yaml @@ -0,0 +1,14 @@ +apiVersion: dscinitialization.opendatahub.io/v1 +kind: DSCInitialization +metadata: + name: mlbatch-dsci +spec: + applicationsNamespace: redhat-ods-applications + monitoring: + managementState: Managed + namespace: redhat-ods-monitoring + serviceMesh: + managementState: Removed + trustedCABundle: + customCABundle: "" + managementState: Managed diff --git a/setup.RHOAI-v2.16/mlbatch-edit-role.yaml b/setup.RHOAI-v2.16/mlbatch-edit-role.yaml new file mode 100644 index 0000000..4bb9e45 --- /dev/null +++ b/setup.RHOAI-v2.16/mlbatch-edit-role.yaml @@ -0,0 +1,151 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: mlbatch-edit +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - delete + - get + - list + - watch +- apiGroups: + - apps + resources: + - deployments + - statefulsets + verbs: + - delete + - get + - list + - watch +- apiGroups: + - "" + resources: + - services + - secrets + - configmaps + - persistentvolumeclaims + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - "*" + verbs: + - get + - list + - watch +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ray.io + resources: + - rayjobs + - rayclusters + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - batch + resources: + - jobs + verbs: + - delete + - get + - list + - watch +- apiGroups: + - workload.codeflare.dev + resources: + - appwrappers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - scheduling.k8s.io + resources: + - priorityclasses + verbs: + - get + - list + - watch +- apiGroups: + - scheduling.x-k8s.io + resources: + - podgroups + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + - pods/logs + verbs: + - get +- apiGroups: + - "" + resources: + - pods/exec + - pods/portforward + verbs: + - create +- apiGroups: + - route.openshift.io + resources: + - routes + verbs: + - get + - list + - watch + - delete +- apiGroups: + - "" + - project.openshift.io + resources: + - projects + verbs: + - get diff --git a/setup.RHOAI-v2.16/mlbatch-priorities.yaml b/setup.RHOAI-v2.16/mlbatch-priorities.yaml new file mode 100644 index 0000000..77c8f3b --- /dev/null +++ b/setup.RHOAI-v2.16/mlbatch-priorities.yaml @@ -0,0 +1,26 @@ +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: low-priority +value: 1 +preemptionPolicy: PreemptLowerPriority +globalDefault: false +description: "This is the priority class for all lower priority jobs." +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: default-priority +value: 5 +preemptionPolicy: PreemptLowerPriority +globalDefault: true +description: "This is the priority class for all jobs (default priority)." +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: high-priority +value: 10 +preemptionPolicy: PreemptLowerPriority +globalDefault: false +description: "This is the priority class defined for highly important jobs that would evict lower and default priority jobs." diff --git a/setup.RHOAI-v2.16/mlbatch-subscription.yaml b/setup.RHOAI-v2.16/mlbatch-subscription.yaml new file mode 100644 index 0000000..0f5eec1 --- /dev/null +++ b/setup.RHOAI-v2.16/mlbatch-subscription.yaml @@ -0,0 +1,310 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: redhat-ods-operator +--- +apiVersion: v1 +kind: Namespace +metadata: + name: redhat-ods-applications +--- +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: rhods-operator + namespace: redhat-ods-operator +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mlbatch-codeflare + namespace: redhat-ods-operator +data: + manager.yaml: | + apiVersion: apps/v1 + kind: Deployment + metadata: + name: manager + namespace: system + spec: + selector: + matchLabels: + app.kubernetes.io/name: codeflare-operator + app.kubernetes.io/part-of: codeflare + replicas: 1 + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + app.kubernetes.io/name: codeflare-operator + app.kubernetes.io/part-of: codeflare + spec: + priorityClassName: system-node-critical + securityContext: + runAsNonRoot: true + # TODO(user): For common cases that do not require escalating privileges + # it is recommended to ensure that all your Pods/Containers are restrictive. + # More info: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted + # Please uncomment the following code if your project does NOT have to work on old Kubernetes + # versions < 1.20 or on vendors versions which do NOT support this field by default (i.e. Openshift < 4.11 ). + # seccompProfile: + # type: RuntimeDefault + containers: + - command: + - /manager + image: $(codeflare_operator_controller_image) + imagePullPolicy: Always + name: manager + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - containerPort: 8080 + protocol: TCP + name: metrics + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: "1" + memory: 1Gi + requests: + cpu: "1" + memory: 1Gi + serviceAccountName: controller-manager + terminationGracePeriodSeconds: 10 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: codeflare-operator-config + namespace: redhat-ods-applications +data: + config.yaml: | + appwrapper: + enabled: true + Config: + autopilot: + injectAntiAffinities: true + monitorNodes: true + resourceTaints: + nvidia.com/gpu: + - key: autopilot.ibm.com/gpuhealth + value: ERR + effect: NoSchedule + - key: autopilot.ibm.com/gpuhealth + value: TESTING + effect: NoSchedule + - key: autopilot.ibm.com/gpuhealth + value: EVICT + effect: NoExecute + defaultQueueName: default-queue + enableKueueIntegrations: true + kueueJobReconciller: + manageJobsWithoutQueueName: true + waitForPodsReady: + blockAdmission: false + enable: false + schedulerName: scheduler-plugins-scheduler + slackQueueName: slack-cluster-queue + userRBACAdmissionCheck: false +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mlbatch-kuberay + namespace: redhat-ods-operator +data: + kuberay-operator-image-patch.yaml: | + apiVersion: apps/v1 + kind: Deployment + metadata: + name: kuberay-operator + spec: + template: + spec: + priorityClassName: system-node-critical + containers: + - name: kuberay-operator + image: $(image) +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mlbatch-kueue + namespace: redhat-ods-operator +data: + controller_manager_config.yaml: | + apiVersion: config.kueue.x-k8s.io/v1beta1 + kind: Configuration + health: + healthProbeBindAddress: :8081 + metrics: + bindAddress: :8080 + enableClusterQueueResources: true + webhook: + port: 9443 + leaderElection: + leaderElect: true + resourceName: c1f6bfd2.kueue.x-k8s.io + controller: + groupKindConcurrency: + Job.batch: 5 + Pod: 5 + Workload.kueue.x-k8s.io: 5 + LocalQueue.kueue.x-k8s.io: 1 + ClusterQueue.kueue.x-k8s.io: 1 + ResourceFlavor.kueue.x-k8s.io: 1 + clientConnection: + qps: 50 + burst: 100 + #pprofBindAddress: :8082 + waitForPodsReady: + enable: false + blockAdmission: false + manageJobsWithoutQueueName: true + #internalCertManagement: + # enable: false + # webhookServiceName: "" + # webhookSecretName: "" + integrations: + frameworks: + # - "batch/job" + - "kubeflow.org/mpijob" + - "ray.io/rayjob" + - "ray.io/raycluster" + - "jobset.x-k8s.io/jobset" + - "kubeflow.org/mxjob" + - "kubeflow.org/paddlejob" + - "kubeflow.org/pytorchjob" + - "kubeflow.org/tfjob" + - "kubeflow.org/xgboostjob" + # - "pod" + externalFrameworks: + - "AppWrapper.v1beta2.workload.codeflare.dev" + # podOptions: + # namespaceSelector: + # matchExpressions: + # - key: kubernetes.io/metadata.name + # operator: NotIn + # values: [ kube-system, kueue-system ] + fairSharing: + enable: true + preemptionStrategies: [LessThanOrEqualToFinalShare, LessThanInitialShare] + manager_config_patch.yaml: | + apiVersion: apps/v1 + kind: Deployment + metadata: + name: controller-manager + namespace: system + spec: + template: + spec: + priorityClassName: system-node-critical + containers: + - name: manager + image: $(image) + args: + - "--config=/controller_manager_config.yaml" + - "--zap-log-level=2" + - "--feature-gates=LendingLimit=true" + volumeMounts: + - name: manager-config + mountPath: /controller_manager_config.yaml + subPath: controller_manager_config.yaml + volumes: + - name: manager-config + configMap: + name: manager-config +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mlbatch-training-operator + namespace: redhat-ods-operator +data: + manager_config_patch.yaml: | + apiVersion: apps/v1 + kind: Deployment + metadata: + name: training-operator + spec: + template: + spec: + priorityClassName: system-node-critical + containers: + - name: training-operator + image: $(image) + args: + - "--zap-log-level=2" + - "--gang-scheduler-name=scheduler-plugins-scheduler" + resources: + requests: + cpu: 100m + memory: 100Mi + limits: + cpu: 500m + memory: 1000Mi +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: rhods-operator + namespace: redhat-ods-operator +spec: + channel: stable + installPlanApproval: Manual + name: rhods-operator + source: redhat-operators + sourceNamespace: openshift-marketplace + startingCSV: rhods-operator.2.16.0 + config: + env: + - name: "DISABLE_DSC_CONFIG" + volumeMounts: + - name: mlbatch-codeflare + mountPath: /opt/manifests/codeflare/manager/manager.yaml + subPath: manager.yaml + - name: mlbatch-kuberay + mountPath: /opt/manifests/ray/openshift/kuberay-operator-image-patch.yaml + subPath: kuberay-operator-image-patch.yaml + - name: mlbatch-kueue + mountPath: /opt/manifests/kueue/components/manager/controller_manager_config.yaml + subPath: controller_manager_config.yaml + - name: mlbatch-kueue + mountPath: /opt/manifests/kueue/rhoai/manager_config_patch.yaml + subPath: manager_config_patch.yaml + - name: mlbatch-training-operator + mountPath: /opt/manifests/trainingoperator/rhoai/manager_config_patch.yaml + subPath: manager_config_patch.yaml + volumes: + - name: mlbatch-codeflare + configMap: + name: mlbatch-codeflare + - name: mlbatch-kuberay + configMap: + name: mlbatch-kuberay + - name: mlbatch-kueue + configMap: + name: mlbatch-kueue + - name: mlbatch-training-operator + configMap: + name: mlbatch-training-operator diff --git a/setup.RHOAI-v2.16/mlbatch-upgrade-configmaps.yaml b/setup.RHOAI-v2.16/mlbatch-upgrade-configmaps.yaml new file mode 100644 index 0000000..c111aa4 --- /dev/null +++ b/setup.RHOAI-v2.16/mlbatch-upgrade-configmaps.yaml @@ -0,0 +1,125 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: codeflare-operator-config + namespace: redhat-ods-applications +data: + config.yaml: | + appwrapper: + enabled: true + Config: + autopilot: + injectAntiAffinities: true + monitorNodes: true + resourceTaints: + nvidia.com/gpu: + - key: autopilot.ibm.com/gpuhealth + value: ERR + effect: NoSchedule + - key: autopilot.ibm.com/gpuhealth + value: TESTING + effect: NoSchedule + - key: autopilot.ibm.com/gpuhealth + value: EVICT + effect: NoExecute + defaultQueueName: default-queue + enableKueueIntegrations: true + kueueJobReconciller: + manageJobsWithoutQueueName: true + waitForPodsReady: + blockAdmission: false + enable: false + schedulerName: scheduler-plugins-scheduler + slackQueueName: slack-cluster-queue + userRBACAdmissionCheck: false +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mlbatch-kueue + namespace: redhat-ods-operator +data: + controller_manager_config.yaml: | + apiVersion: config.kueue.x-k8s.io/v1beta1 + kind: Configuration + health: + healthProbeBindAddress: :8081 + metrics: + bindAddress: :8080 + enableClusterQueueResources: true + webhook: + port: 9443 + leaderElection: + leaderElect: true + resourceName: c1f6bfd2.kueue.x-k8s.io + controller: + groupKindConcurrency: + Job.batch: 5 + Pod: 5 + Workload.kueue.x-k8s.io: 5 + LocalQueue.kueue.x-k8s.io: 1 + ClusterQueue.kueue.x-k8s.io: 1 + ResourceFlavor.kueue.x-k8s.io: 1 + clientConnection: + qps: 50 + burst: 100 + #pprofBindAddress: :8082 + waitForPodsReady: + enable: false + blockAdmission: false + manageJobsWithoutQueueName: true + #internalCertManagement: + # enable: false + # webhookServiceName: "" + # webhookSecretName: "" + integrations: + frameworks: + # - "batch/job" + - "kubeflow.org/mpijob" + - "ray.io/rayjob" + - "ray.io/raycluster" + - "jobset.x-k8s.io/jobset" + - "kubeflow.org/mxjob" + - "kubeflow.org/paddlejob" + - "kubeflow.org/pytorchjob" + - "kubeflow.org/tfjob" + - "kubeflow.org/xgboostjob" + # - "pod" + externalFrameworks: + - "AppWrapper.v1beta2.workload.codeflare.dev" + # podOptions: + # namespaceSelector: + # matchExpressions: + # - key: kubernetes.io/metadata.name + # operator: NotIn + # values: [ kube-system, kueue-system ] + fairSharing: + enable: true + preemptionStrategies: [LessThanOrEqualToFinalShare, LessThanInitialShare] + manager_config_patch.yaml: | + apiVersion: apps/v1 + kind: Deployment + metadata: + name: controller-manager + namespace: system + spec: + template: + spec: + priorityClassName: system-node-critical + containers: + - name: manager + image: $(image) + args: + - "--config=/controller_manager_config.yaml" + - "--zap-log-level=2" + - "--feature-gates=LendingLimit=true" + volumeMounts: + - name: manager-config + mountPath: /controller_manager_config.yaml + subPath: controller_manager_config.yaml + volumes: + - name: manager-config + configMap: + name: manager-config +--- diff --git a/setup.k8s-v1.27/CLUSTER-SETUP.md b/setup.k8s-v1.27/CLUSTER-SETUP.md index 7bbce99..05585ca 100644 --- a/setup.k8s-v1.27/CLUSTER-SETUP.md +++ b/setup.k8s-v1.27/CLUSTER-SETUP.md @@ -79,6 +79,7 @@ operators as follows: - `batch/job` integration is disabled, - `waitForPodsReady` is disabled, - `LendingLimit` feature gate is enabled, + - `fairSharing` is enabled, - `enableClusterQueueResources` metrics is enabled, - AppWrapper operator: - `userRBACAdmissionCheck` is disabled, diff --git a/setup.k8s-v1.27/kueue/controller_manager_config.yaml b/setup.k8s-v1.27/kueue/controller_manager_config.yaml index 5772abd..20c38eb 100644 --- a/setup.k8s-v1.27/kueue/controller_manager_config.yaml +++ b/setup.k8s-v1.27/kueue/controller_manager_config.yaml @@ -57,8 +57,8 @@ integrations: # - key: kubernetes.io/metadata.name # operator: NotIn # values: [ kube-system, kueue-system ] -#fairSharing: -# enable: true -# preemptionStrategies: [LessThanOrEqualToFinalShare, LessThanInitialShare] +fairSharing: + enable: true + preemptionStrategies: [LessThanOrEqualToFinalShare, LessThanInitialShare] #resources: # excludeResourcePrefixes: [] diff --git a/setup.k8s-v1.30/CLUSTER-SETUP.md b/setup.k8s-v1.30/CLUSTER-SETUP.md index 40134d1..ee20570 100644 --- a/setup.k8s-v1.30/CLUSTER-SETUP.md +++ b/setup.k8s-v1.30/CLUSTER-SETUP.md @@ -75,8 +75,10 @@ operators as follows: - Kubeflow Training Operator: - `gang-scheduler-name` is set to `scheduler-plugins-scheduler`, - Kueue: + - `batch/job` integration is disabled, - `waitForPodsReady` is disabled, - `LendingLimit` feature gate is enabled, + - `fairSharing` is enabled, - `enableClusterQueueResources` metrics is enabled, - AppWrapper operator: - `userRBACAdmissionCheck` is disabled, diff --git a/setup.k8s-v1.30/kueue/controller_manager_config.yaml b/setup.k8s-v1.30/kueue/controller_manager_config.yaml index b9bd737..4325927 100644 --- a/setup.k8s-v1.30/kueue/controller_manager_config.yaml +++ b/setup.k8s-v1.30/kueue/controller_manager_config.yaml @@ -57,8 +57,8 @@ integrations: # - key: kubernetes.io/metadata.name # operator: NotIn # values: [ kube-system, kueue-system ] -#fairSharing: -# enable: true -# preemptionStrategies: [LessThanOrEqualToFinalShare, LessThanInitialShare] +fairSharing: + enable: true + preemptionStrategies: [LessThanOrEqualToFinalShare, LessThanInitialShare] #resources: # excludeResourcePrefixes: [] diff --git a/setup.tmpl/CLUSTER-SETUP.md.tmpl b/setup.tmpl/CLUSTER-SETUP.md.tmpl index a0b0af1..e8e8679 100644 --- a/setup.tmpl/CLUSTER-SETUP.md.tmpl +++ b/setup.tmpl/CLUSTER-SETUP.md.tmpl @@ -105,6 +105,9 @@ AI configuration as follows: - `batch/job` integration is disabled, - `waitForPodsReady` is disabled, - `LendingLimit` feature gate is enabled, +{{- if .FAIRSHARE }} + - `fairSharing` is enabled, +{{- end }} - `enableClusterQueueResources` metrics is enabled, - Codeflare operator: - the AppWrapper controller is enabled and configured as follows: @@ -116,7 +119,7 @@ AI configuration as follows: {{- end }} - pod priorities, resource requests and limits have been adjusted. -{{ if or (eq .VERSION "RHOAI-v2.10") (eq .VERSION "RHOAI-v2.11") (eq .VERSION "RHOAI-v2.13") -}} +{{ if or (eq .VERSION "RHOAI-v2.10") (eq .VERSION "RHOAI-v2.13") -}} To work around https://issues.redhat.com/browse/RHOAIENG-7887 (a race condition in Red Hat OpenShift AI installation), do a rolling restart of the Kueue manager. ```sh @@ -166,10 +169,13 @@ operators as follows: - Kueue: {{- if not .VAP }} - `manageJobsWithoutQueueName` is enabled, - - `batch/job` integration is disabled, {{- end }} + - `batch/job` integration is disabled, - `waitForPodsReady` is disabled, - `LendingLimit` feature gate is enabled, +{{- if .FAIRSHARE }} + - `fairSharing` is enabled, +{{- end }} - `enableClusterQueueResources` metrics is enabled, - AppWrapper operator: - `userRBACAdmissionCheck` is disabled, diff --git a/setup.tmpl/Kubernetes-v1.27.yaml b/setup.tmpl/Kubernetes-v1.27.yaml index e632567..35682d9 100644 --- a/setup.tmpl/Kubernetes-v1.27.yaml +++ b/setup.tmpl/Kubernetes-v1.27.yaml @@ -5,3 +5,4 @@ VERSION: k8s-v1.27 KUBECTL: kubectl VAP: false SLACKCQ: true +FAIRSHARE: true diff --git a/setup.tmpl/Kubernetes-v1.30.yaml b/setup.tmpl/Kubernetes-v1.30.yaml index 45be21f..a755af6 100644 --- a/setup.tmpl/Kubernetes-v1.30.yaml +++ b/setup.tmpl/Kubernetes-v1.30.yaml @@ -5,3 +5,4 @@ VERSION: k8s-v1.30 KUBECTL: kubectl VAP: true SLACKCQ: true +FAIRSHARE: true diff --git a/setup.tmpl/Makefile b/setup.tmpl/Makefile index 1507774..25e7c78 100644 --- a/setup.tmpl/Makefile +++ b/setup.tmpl/Makefile @@ -27,6 +27,8 @@ docs: gotmpl ../tools/gotmpl/gotmpl -input ./TEAM-SETUP.md.tmpl -output ../setup.RHOAI-v2.13/TEAM-SETUP.md -values RHOAI-v2.13.yaml ../tools/gotmpl/gotmpl -input ./CLUSTER-SETUP.md.tmpl -output ../setup.RHOAI-v2.15/CLUSTER-SETUP.md -values RHOAI-v2.15.yaml ../tools/gotmpl/gotmpl -input ./TEAM-SETUP.md.tmpl -output ../setup.RHOAI-v2.15/TEAM-SETUP.md -values RHOAI-v2.15.yaml + ../tools/gotmpl/gotmpl -input ./CLUSTER-SETUP.md.tmpl -output ../setup.RHOAI-v2.16/CLUSTER-SETUP.md -values RHOAI-v2.16.yaml + ../tools/gotmpl/gotmpl -input ./TEAM-SETUP.md.tmpl -output ../setup.RHOAI-v2.16/TEAM-SETUP.md -values RHOAI-v2.16.yaml ../tools/gotmpl/gotmpl -input ./CLUSTER-SETUP.md.tmpl -output ../setup.k8s-v1.27/CLUSTER-SETUP.md -values Kubernetes-v1.27.yaml ../tools/gotmpl/gotmpl -input ./TEAM-SETUP.md.tmpl -output ../setup.k8s-v1.27/TEAM-SETUP.md -values Kubernetes-v1.27.yaml ../tools/gotmpl/gotmpl -input ./CLUSTER-SETUP.md.tmpl -output ../setup.k8s-v1.30/CLUSTER-SETUP.md -values Kubernetes-v1.30.yaml diff --git a/setup.tmpl/RHOAI-v2.10.yaml b/setup.tmpl/RHOAI-v2.10.yaml index 5cb2e17..141eadd 100644 --- a/setup.tmpl/RHOAI-v2.10.yaml +++ b/setup.tmpl/RHOAI-v2.10.yaml @@ -4,3 +4,4 @@ OPENSHIFT: true VERSION: RHOAI-v2.10 KUBECTL: oc SLACKCQ: false +FAIRSHARE: false \ No newline at end of file diff --git a/setup.tmpl/RHOAI-v2.13.yaml b/setup.tmpl/RHOAI-v2.13.yaml index 200b631..9ac6942 100644 --- a/setup.tmpl/RHOAI-v2.13.yaml +++ b/setup.tmpl/RHOAI-v2.13.yaml @@ -3,4 +3,5 @@ OPENSHIFT: true VERSION: RHOAI-v2.13 KUBECTL: oc -SLACKCQ: true \ No newline at end of file +SLACKCQ: true +FAIRSHARE: false diff --git a/setup.tmpl/RHOAI-v2.15.yaml b/setup.tmpl/RHOAI-v2.15.yaml index 8594b86..8999636 100644 --- a/setup.tmpl/RHOAI-v2.15.yaml +++ b/setup.tmpl/RHOAI-v2.15.yaml @@ -3,4 +3,5 @@ OPENSHIFT: true VERSION: RHOAI-v2.15 KUBECTL: oc -SLACKCQ: true \ No newline at end of file +SLACKCQ: true +FAIRSHARE: false diff --git a/setup.tmpl/RHOAI-v2.16.yaml b/setup.tmpl/RHOAI-v2.16.yaml new file mode 100644 index 0000000..1be5d8a --- /dev/null +++ b/setup.tmpl/RHOAI-v2.16.yaml @@ -0,0 +1,7 @@ +# Values for RHOAI 2.16 + +OPENSHIFT: true +VERSION: RHOAI-v2.16 +KUBECTL: oc +SLACKCQ: true +FAIRSHARE: true