From 8355856ef7acca8442da217f5595af6c1368c222 Mon Sep 17 00:00:00 2001 From: David Grove Date: Fri, 7 Jun 2024 14:10:02 -0400 Subject: [PATCH] Simplify yaml generation by starting from non-AppWrapper template The v1/beta2 AppWrapper can be wrapped around a RayCluster as a simple prefix to the "real" yaml. So we can simplify yaml generation/manipulation by starting with a vanilla RayCluster and just optionally put it inside an AppWrapper at the end. --- src/codeflare_sdk/cluster/awload.py | 2 +- src/codeflare_sdk/cluster/cluster.py | 8 +- .../templates/base-template.yaml | 383 +++++++++--------- src/codeflare_sdk/utils/generate_yaml.py | 182 +++------ tests/test-case-no-mcad.yamls | 1 - tests/unit_test.py | 2 - 6 files changed, 254 insertions(+), 324 deletions(-) diff --git a/src/codeflare_sdk/cluster/awload.py b/src/codeflare_sdk/cluster/awload.py index c622f8772..7455b2161 100644 --- a/src/codeflare_sdk/cluster/awload.py +++ b/src/codeflare_sdk/cluster/awload.py @@ -30,7 +30,7 @@ class AWManager: """ An object for submitting and removing existing AppWrapper yamls - to be added to the MCAD queue. + to be added to the Kueue localqueue. """ def __init__(self, filename: str) -> None: diff --git a/src/codeflare_sdk/cluster/cluster.py b/src/codeflare_sdk/cluster/cluster.py index 35c26b0a9..e5bbcd86a 100644 --- a/src/codeflare_sdk/cluster/cluster.py +++ b/src/codeflare_sdk/cluster/cluster.py @@ -147,11 +147,9 @@ def create_app_wrapper(self): template = self.config.template image = self.config.image appwrapper = self.config.appwrapper - instance_types = self.config.machine_types env = self.config.envs image_pull_secrets = self.config.image_pull_secrets write_to_file = self.config.write_to_file - verify_tls = self.config.verify_tls local_queue = self.config.local_queue labels = self.config.labels return generate_appwrapper( @@ -169,11 +167,9 @@ def create_app_wrapper(self): template=template, image=image, appwrapper=appwrapper, - instance_types=instance_types, env=env, image_pull_secrets=image_pull_secrets, write_to_file=write_to_file, - verify_tls=verify_tls, local_queue=local_queue, labels=labels, ) @@ -181,8 +177,8 @@ def create_app_wrapper(self): # creates a new cluster with the provided or default spec def up(self): """ - Applies the AppWrapper yaml, pushing the resource request onto - the MCAD queue. + Applies the Cluster yaml, pushing the resource request onto + the Kueue localqueue. """ # check if RayCluster CustomResourceDefinition exists if not throw RuntimeError diff --git a/src/codeflare_sdk/templates/base-template.yaml b/src/codeflare_sdk/templates/base-template.yaml index b6a70b2b6..0e0719610 100644 --- a/src/codeflare_sdk/templates/base-template.yaml +++ b/src/codeflare_sdk/templates/base-template.yaml @@ -1,207 +1,198 @@ -apiVersion: workload.codeflare.dev/v1beta2 -kind: AppWrapper +# This config demonstrates KubeRay's Ray autoscaler integration. +# The resource requests and limits in this config are too small for production! +# For an example with more realistic resource configuration, see +# ray-cluster.autoscaler.large.yaml. +apiVersion: ray.io/v1 +kind: RayCluster metadata: - name: aw-kuberay + labels: + controller-tools.k8s.io: "1.0" + # A unique identifier for the head node and workers of this cluster. + name: kuberay-cluster namespace: default spec: - components: - - template: - # This config demonstrates KubeRay's Ray autoscaler integration. - # The resource requests and limits in this config are too small for production! - # For an example with more realistic resource configuration, see - # ray-cluster.autoscaler.large.yaml. - apiVersion: ray.io/v1 - kind: RayCluster + # The version of Ray you are using. Make sure all Ray containers are running this version of Ray. + rayVersion: '2.7.0' + # If enableInTreeAutoscaling is true, the autoscaler sidecar will be added to the Ray head pod. + # Ray autoscaler integration is supported only for Ray versions >= 1.11.0 + # Ray autoscaler integration is Beta with KubeRay >= 0.3.0 and Ray >= 2.0.0. + enableInTreeAutoscaling: false + # autoscalerOptions is an OPTIONAL field specifying configuration overrides for the Ray autoscaler. + # The example configuration shown below below represents the DEFAULT values. + # (You may delete autoscalerOptions if the defaults are suitable.) + autoscalerOptions: + # upscalingMode is "Default" or "Aggressive." + # Conservative: Upscaling is rate-limited; the number of pending worker pods is at most the size of the Ray cluster. + # Default: Upscaling is not rate-limited. + # Aggressive: An alias for Default; upscaling is not rate-limited. + upscalingMode: Default + # idleTimeoutSeconds is the number of seconds to wait before scaling down a worker pod which is not using Ray resources. + idleTimeoutSeconds: 60 + # image optionally overrides the autoscaler's container image. + # If instance.spec.rayVersion is at least "2.0.0", the autoscaler will default to the same image as + # the ray container. For older Ray versions, the autoscaler will default to using the Ray 2.0.0 image. + ## image: "my-repo/my-custom-autoscaler-image:tag" + # imagePullPolicy optionally overrides the autoscaler container's image pull policy. + imagePullPolicy: Always + # resources specifies optional resource request and limit overrides for the autoscaler container. + # For large Ray clusters, we recommend monitoring container resource usage to determine if overriding the defaults is required. + resources: + limits: + cpu: "500m" + memory: "512Mi" + requests: + cpu: "500m" + memory: "512Mi" + ######################headGroupSpec################################# + # head group template and specs, (perhaps 'group' is not needed in the name) + headGroupSpec: + # Kubernetes Service Type, valid values are 'ClusterIP', 'NodePort' and 'LoadBalancer' + serviceType: ClusterIP + enableIngress: false + # logical group name, for this called head-group, also can be functional + # pod type head or worker + # rayNodeType: head # Not needed since it is under the headgroup + # the following params are used to complete the ray start: ray start --head --block ... + rayStartParams: + # Flag "no-monitor" will be automatically set when autoscaling is enabled. + dashboard-host: '0.0.0.0' + block: 'true' + # num-cpus: '1' # can be auto-completed from the limits + # Use `resources` to optionally specify custom resource annotations for the Ray node. + # The value of `resources` is a string-integer mapping. + # Currently, `resources` must be provided in the specific format demonstrated below: + # resources: '"{\"Custom1\": 1, \"Custom2\": 5}"' + num-gpus: '0' + #pod template + template: + spec: + containers: + # The Ray head pod + - name: ray-head + image: quay.io/project-codeflare/ray:latest-py39-cu118 + imagePullPolicy: Always + ports: + - containerPort: 6379 + name: gcs + - containerPort: 8265 + name: dashboard + - containerPort: 10001 + name: client + lifecycle: + preStop: + exec: + command: ["/bin/sh","-c","ray stop"] + resources: + limits: + cpu: 2 + memory: "8G" + nvidia.com/gpu: 0 + requests: + cpu: 2 + memory: "8G" + nvidia.com/gpu: 0 + volumeMounts: + - mountPath: /etc/pki/tls/certs/odh-trusted-ca-bundle.crt + name: odh-trusted-ca-cert + subPath: odh-trusted-ca-bundle.crt + - mountPath: /etc/ssl/certs/odh-trusted-ca-bundle.crt + name: odh-trusted-ca-cert + subPath: odh-trusted-ca-bundle.crt + - mountPath: /etc/pki/tls/certs/odh-ca-bundle.crt + name: odh-ca-cert + subPath: odh-ca-bundle.crt + - mountPath: /etc/ssl/certs/odh-ca-bundle.crt + name: odh-ca-cert + subPath: odh-ca-bundle.crt + volumes: + - name: odh-trusted-ca-cert + configMap: + name: odh-trusted-ca-bundle + items: + - key: ca-bundle.crt + path: odh-trusted-ca-bundle.crt + optional: true + - name: odh-ca-cert + configMap: + name: odh-trusted-ca-bundle + items: + - key: odh-ca-bundle.crt + path: odh-ca-bundle.crt + optional: true + workerGroupSpecs: + # the pod replicas in this group typed worker + - replicas: 3 + minReplicas: 3 + maxReplicas: 3 + # logical group name, for this called small-group, also can be functional + groupName: small-group + # if worker pods need to be added, we can simply increment the replicas + # if worker pods need to be removed, we decrement the replicas, and populate the podsToDelete list + # the operator will remove pods from the list until the number of replicas is satisfied + # when a pod is confirmed to be deleted, its name will be removed from the list below + #scaleStrategy: + # workersToDelete: + # - raycluster-complete-worker-small-group-bdtwh + # - raycluster-complete-worker-small-group-hv457 + # - raycluster-complete-worker-small-group-k8tj7 + # the following params are used to complete the ray start: ray start --block ... + rayStartParams: + block: 'true' + num-gpus: 1 + #pod template + template: metadata: labels: - controller-tools.k8s.io: "1.0" - # A unique identifier for the head node and workers of this cluster. - name: kuberay-cluster + key: value + # annotations for pod + annotations: + key: value # finalizers: # - kubernetes spec: - # The version of Ray you are using. Make sure all Ray containers are running this version of Ray. - rayVersion: '2.7.0' - # If enableInTreeAutoscaling is true, the autoscaler sidecar will be added to the Ray head pod. - # Ray autoscaler integration is supported only for Ray versions >= 1.11.0 - # Ray autoscaler integration is Beta with KubeRay >= 0.3.0 and Ray >= 2.0.0. - enableInTreeAutoscaling: false - # autoscalerOptions is an OPTIONAL field specifying configuration overrides for the Ray autoscaler. - # The example configuration shown below below represents the DEFAULT values. - # (You may delete autoscalerOptions if the defaults are suitable.) - autoscalerOptions: - # upscalingMode is "Default" or "Aggressive." - # Conservative: Upscaling is rate-limited; the number of pending worker pods is at most the size of the Ray cluster. - # Default: Upscaling is not rate-limited. - # Aggressive: An alias for Default; upscaling is not rate-limited. - upscalingMode: Default - # idleTimeoutSeconds is the number of seconds to wait before scaling down a worker pod which is not using Ray resources. - idleTimeoutSeconds: 60 - # image optionally overrides the autoscaler's container image. - # If instance.spec.rayVersion is at least "2.0.0", the autoscaler will default to the same image as - # the ray container. For older Ray versions, the autoscaler will default to using the Ray 2.0.0 image. - ## image: "my-repo/my-custom-autoscaler-image:tag" - # imagePullPolicy optionally overrides the autoscaler container's image pull policy. - imagePullPolicy: Always - # resources specifies optional resource request and limit overrides for the autoscaler container. - # For large Ray clusters, we recommend monitoring container resource usage to determine if overriding the defaults is required. + containers: + - name: machine-learning # must consist of lower case alphanumeric characters or '-', and must start and end with an alphanumeric character (e.g. 'my-name', or '123-abc' + image: quay.io/project-codeflare/ray:latest-py39-cu118 + # environment variables to set in the container.Optional. + # Refer to https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/ + lifecycle: + preStop: + exec: + command: ["/bin/sh","-c","ray stop"] resources: limits: - cpu: "500m" - memory: "512Mi" + cpu: "2" + memory: "12G" + nvidia.com/gpu: "1" requests: - cpu: "500m" - memory: "512Mi" - ######################headGroupSpec################################# - # head group template and specs, (perhaps 'group' is not needed in the name) - headGroupSpec: - # Kubernetes Service Type, valid values are 'ClusterIP', 'NodePort' and 'LoadBalancer' - serviceType: ClusterIP - enableIngress: false - # logical group name, for this called head-group, also can be functional - # pod type head or worker - # rayNodeType: head # Not needed since it is under the headgroup - # the following params are used to complete the ray start: ray start --head --block ... - rayStartParams: - # Flag "no-monitor" will be automatically set when autoscaling is enabled. - dashboard-host: '0.0.0.0' - block: 'true' - # num-cpus: '1' # can be auto-completed from the limits - # Use `resources` to optionally specify custom resource annotations for the Ray node. - # The value of `resources` is a string-integer mapping. - # Currently, `resources` must be provided in the specific format demonstrated below: - # resources: '"{\"Custom1\": 1, \"Custom2\": 5}"' - num-gpus: '0' - #pod template - template: - spec: - containers: - # The Ray head pod - - name: ray-head - image: quay.io/project-codeflare/ray:latest-py39-cu118 - imagePullPolicy: Always - ports: - - containerPort: 6379 - name: gcs - - containerPort: 8265 - name: dashboard - - containerPort: 10001 - name: client - lifecycle: - preStop: - exec: - command: ["/bin/sh","-c","ray stop"] - resources: - limits: - cpu: 2 - memory: "8G" - nvidia.com/gpu: 0 - requests: - cpu: 2 - memory: "8G" - nvidia.com/gpu: 0 - volumeMounts: - - mountPath: /etc/pki/tls/certs/odh-trusted-ca-bundle.crt - name: odh-trusted-ca-cert - subPath: odh-trusted-ca-bundle.crt - - mountPath: /etc/ssl/certs/odh-trusted-ca-bundle.crt - name: odh-trusted-ca-cert - subPath: odh-trusted-ca-bundle.crt - - mountPath: /etc/pki/tls/certs/odh-ca-bundle.crt - name: odh-ca-cert - subPath: odh-ca-bundle.crt - - mountPath: /etc/ssl/certs/odh-ca-bundle.crt - name: odh-ca-cert - subPath: odh-ca-bundle.crt - volumes: - - name: odh-trusted-ca-cert - configMap: - name: odh-trusted-ca-bundle - items: - - key: ca-bundle.crt - path: odh-trusted-ca-bundle.crt - optional: true - - name: odh-ca-cert - configMap: - name: odh-trusted-ca-bundle - items: - - key: odh-ca-bundle.crt - path: odh-ca-bundle.crt - optional: true - workerGroupSpecs: - # the pod replicas in this group typed worker - - replicas: 3 - minReplicas: 3 - maxReplicas: 3 - # logical group name, for this called small-group, also can be functional - groupName: small-group - # if worker pods need to be added, we can simply increment the replicas - # if worker pods need to be removed, we decrement the replicas, and populate the podsToDelete list - # the operator will remove pods from the list until the number of replicas is satisfied - # when a pod is confirmed to be deleted, its name will be removed from the list below - #scaleStrategy: - # workersToDelete: - # - raycluster-complete-worker-small-group-bdtwh - # - raycluster-complete-worker-small-group-hv457 - # - raycluster-complete-worker-small-group-k8tj7 - # the following params are used to complete the ray start: ray start --block ... - rayStartParams: - block: 'true' - num-gpus: 1 - #pod template - template: - metadata: - labels: - key: value - # annotations for pod - annotations: - key: value - # finalizers: - # - kubernetes - spec: - containers: - - name: machine-learning # must consist of lower case alphanumeric characters or '-', and must start and end with an alphanumeric character (e.g. 'my-name', or '123-abc' - image: quay.io/project-codeflare/ray:latest-py39-cu118 - # environment variables to set in the container.Optional. - # Refer to https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/ - lifecycle: - preStop: - exec: - command: ["/bin/sh","-c","ray stop"] - resources: - limits: - cpu: "2" - memory: "12G" - nvidia.com/gpu: "1" - requests: - cpu: "2" - memory: "12G" - nvidia.com/gpu: "1" - volumeMounts: - - mountPath: /etc/pki/tls/certs/odh-trusted-ca-bundle.crt - name: odh-trusted-ca-cert - subPath: odh-trusted-ca-bundle.crt - - mountPath: /etc/ssl/certs/odh-trusted-ca-bundle.crt - name: odh-trusted-ca-cert - subPath: odh-trusted-ca-bundle.crt - - mountPath: /etc/pki/tls/certs/odh-ca-bundle.crt - name: odh-ca-cert - subPath: odh-ca-bundle.crt - - mountPath: /etc/ssl/certs/odh-ca-bundle.crt - name: odh-ca-cert - subPath: odh-ca-bundle.crt - volumes: - - name: odh-trusted-ca-cert - configMap: - name: odh-trusted-ca-bundle - items: - - key: ca-bundle.crt - path: odh-trusted-ca-bundle.crt - optional: true - - name: odh-ca-cert - configMap: - name: odh-trusted-ca-bundle - items: - - key: odh-ca-bundle.crt - path: odh-ca-bundle.crt - optional: true + cpu: "2" + memory: "12G" + nvidia.com/gpu: "1" + volumeMounts: + - mountPath: /etc/pki/tls/certs/odh-trusted-ca-bundle.crt + name: odh-trusted-ca-cert + subPath: odh-trusted-ca-bundle.crt + - mountPath: /etc/ssl/certs/odh-trusted-ca-bundle.crt + name: odh-trusted-ca-cert + subPath: odh-trusted-ca-bundle.crt + - mountPath: /etc/pki/tls/certs/odh-ca-bundle.crt + name: odh-ca-cert + subPath: odh-ca-bundle.crt + - mountPath: /etc/ssl/certs/odh-ca-bundle.crt + name: odh-ca-cert + subPath: odh-ca-bundle.crt + volumes: + - name: odh-trusted-ca-cert + configMap: + name: odh-trusted-ca-bundle + items: + - key: ca-bundle.crt + path: odh-trusted-ca-bundle.crt + optional: true + - name: odh-ca-cert + configMap: + name: odh-trusted-ca-bundle + items: + - key: odh-ca-bundle.crt + path: odh-ca-bundle.crt + optional: true diff --git a/src/codeflare_sdk/utils/generate_yaml.py b/src/codeflare_sdk/utils/generate_yaml.py index d6290c5c3..370c7346e 100755 --- a/src/codeflare_sdk/utils/generate_yaml.py +++ b/src/codeflare_sdk/utils/generate_yaml.py @@ -77,13 +77,10 @@ def is_kind_cluster(): return False -def update_names(yaml, item, appwrapper_name, cluster_name, namespace): - metadata = yaml.get("metadata") - metadata["name"] = appwrapper_name - metadata["namespace"] = namespace - lower_meta = item.get("template", {}).get("metadata") - lower_meta["name"] = cluster_name - lower_meta["namespace"] = namespace +def update_names(cluster_yaml, cluster_name, namespace): + meta = cluster_yaml.get("metadata") + meta["name"] = cluster_name + meta["namespace"] = namespace def update_image(spec, image): @@ -125,7 +122,7 @@ def update_resources(spec, min_cpu, max_cpu, min_memory, max_memory, gpu): def update_nodes( - item, + cluster_yaml, appwrapper_name, min_cpu, max_cpu, @@ -140,48 +137,35 @@ def update_nodes( head_memory, head_gpus, ): - if "template" in item.keys(): - head = item.get("template").get("spec").get("headGroupSpec") - head["rayStartParams"]["num-gpus"] = str(int(head_gpus)) - - worker = item.get("template").get("spec").get("workerGroupSpecs")[0] - # Head counts as first worker - worker["replicas"] = workers - worker["minReplicas"] = workers - worker["maxReplicas"] = workers - worker["groupName"] = "small-group-" + appwrapper_name - worker["rayStartParams"]["num-gpus"] = str(int(gpu)) - - for comp in [head, worker]: - spec = comp.get("template").get("spec") - update_image_pull_secrets(spec, image_pull_secrets) - update_image(spec, image) - update_env(spec, env) - if comp == head: - # TODO: Eventually add head node configuration outside of template - update_resources( - spec, head_cpus, head_cpus, head_memory, head_memory, head_gpus - ) - else: - update_resources(spec, min_cpu, max_cpu, min_memory, max_memory, gpu) + head = cluster_yaml.get("spec").get("headGroupSpec") + head["rayStartParams"]["num-gpus"] = str(int(head_gpus)) + + worker = cluster_yaml.get("spec").get("workerGroupSpecs")[0] + # Head counts as first worker + worker["replicas"] = workers + worker["minReplicas"] = workers + worker["maxReplicas"] = workers + worker["groupName"] = "small-group-" + appwrapper_name + worker["rayStartParams"]["num-gpus"] = str(int(gpu)) + + for comp in [head, worker]: + spec = comp.get("template").get("spec") + update_image_pull_secrets(spec, image_pull_secrets) + update_image(spec, image) + update_env(spec, env) + if comp == head: + # TODO: Eventually add head node configuration outside of template + update_resources( + spec, head_cpus, head_cpus, head_memory, head_memory, head_gpus + ) + else: + update_resources(spec, min_cpu, max_cpu, min_memory, max_memory, gpu) def del_from_list_by_name(l: list, target: typing.List[str]) -> list: return [x for x in l if x["name"] not in target] -def write_user_appwrapper(user_yaml, output_file_name): - # Create the directory if it doesn't exist - directory_path = os.path.dirname(output_file_name) - if not os.path.exists(directory_path): - os.makedirs(directory_path) - - with open(output_file_name, "w") as outfile: - yaml.dump(user_yaml, outfile, default_flow_style=False) - - print(f"Written to: {output_file_name}") - - def get_default_kueue_name(namespace: str): # If the local queue is set, use it. Otherwise, try to use the default queue. try: @@ -240,64 +224,40 @@ def add_queue_label(item: dict, namespace: str, local_queue: Optional[str]): def augment_labels(item: dict, labels: dict): - if "template" in item: - if not "labels" in item["template"]["metadata"]: - item["template"]["metadata"]["labels"] = {} - item["template"]["metadata"]["labels"].update(labels) + if not "labels" in item["metadata"]: + item["metadata"]["labels"] = {} + item["metadata"]["labels"].update(labels) def notebook_annotations(item: dict): nb_prefix = os.environ.get("NB_PREFIX") if nb_prefix: - if "template" in item: - if not "annotations" in item["template"]["metadata"]: - item["template"]["metadata"]["annotations"] = {} - item["template"]["metadata"]["annotations"].update( + if not "annotations" in item["metadata"]: + item["metadata"]["annotations"] = {} + item["metadata"]["annotations"].update( {"app.kubernetes.io/managed-by": nb_prefix} - ) + ) -def write_components( - user_yaml: dict, - output_file_name: str, -): +def wrap_cluster(cluster_yaml: dict, appwrapper_name: str, namespace: str): + return { + "apiVersion": "workload.codeflare.dev/v1beta2", + "kind": "AppWrapper", + "metadata": {"name": appwrapper_name, "namespace": namespace}, + "spec": {"components": [{"template": cluster_yaml}]}, + } + + +def write_user_yaml(user_yaml, output_file_name): # Create the directory if it doesn't exist directory_path = os.path.dirname(output_file_name) if not os.path.exists(directory_path): os.makedirs(directory_path) - components = user_yaml.get("spec", "resources").get("components") - open(output_file_name, "w").close() - with open(output_file_name, "a") as outfile: - for component in components: - if "template" in component: - outfile.write("---\n") - yaml.dump(component["template"], outfile, default_flow_style=False) - print(f"Written to: {output_file_name}") - - -def load_components( - user_yaml: dict, - name: str, -): - component_list = [] - components = user_yaml.get("spec", "resources").get("components") - for component in components: - if "template" in component: - component_list.append(component["template"]) - - resources = "---\n" + "---\n".join( - [yaml.dump(component) for component in component_list] - ) - user_yaml = resources - print(f"Yaml resources loaded for {name}") - return user_yaml - + with open(output_file_name, "w") as outfile: + yaml.dump(user_yaml, outfile, default_flow_style=False) -def load_appwrapper(user_yaml: dict, name: str): - user_yaml = yaml.dump(user_yaml) - print(f"Yaml resources loaded for {name}") - return user_yaml + print(f"Written to: {output_file_name}") def generate_appwrapper( @@ -315,27 +275,17 @@ def generate_appwrapper( template: str, image: str, appwrapper: bool, - instance_types: list, env, image_pull_secrets: list, write_to_file: bool, - verify_tls: bool, local_queue: Optional[str], labels, ): - user_yaml = read_template(template) + cluster_yaml = read_template(template) appwrapper_name, cluster_name = gen_names(name) - resources = user_yaml.get("spec", "resources") - item = resources.get("components")[0] - update_names( - user_yaml, - item, - appwrapper_name, - cluster_name, - namespace, - ) + update_names(cluster_yaml, cluster_name, namespace) update_nodes( - item, + cluster_yaml, appwrapper_name, min_cpu, max_cpu, @@ -350,27 +300,23 @@ def generate_appwrapper( head_memory, head_gpus, ) + augment_labels(cluster_yaml, labels) + notebook_annotations(cluster_yaml) - augment_labels(item, labels) - notebook_annotations(item) - - if appwrapper: - add_queue_label(user_yaml, namespace, local_queue) - else: - add_queue_label(item["template"], namespace, local_queue) + user_yaml = ( + wrap_cluster(cluster_yaml, appwrapper_name, namespace) + if appwrapper + else cluster_yaml + ) - directory_path = os.path.expanduser("~/.codeflare/resources/") - outfile = os.path.join(directory_path, appwrapper_name + ".yaml") + add_queue_label(user_yaml, namespace, local_queue) if write_to_file: - if appwrapper: - write_user_appwrapper(user_yaml, outfile) - else: - write_components(user_yaml, outfile) + directory_path = os.path.expanduser("~/.codeflare/resources/") + outfile = os.path.join(directory_path, appwrapper_name + ".yaml") + write_user_yaml(user_yaml, outfile) return outfile else: - if appwrapper: - user_yaml = load_appwrapper(user_yaml, name) - else: - user_yaml = load_components(user_yaml, name) + user_yaml = yaml.dump(user_yaml) + print(f"Yaml resources loaded for {name}") return user_yaml diff --git a/tests/test-case-no-mcad.yamls b/tests/test-case-no-mcad.yamls index b69c0d60f..37d5d4dbb 100644 --- a/tests/test-case-no-mcad.yamls +++ b/tests/test-case-no-mcad.yamls @@ -1,4 +1,3 @@ ---- apiVersion: ray.io/v1 kind: RayCluster metadata: diff --git a/tests/unit_test.py b/tests/unit_test.py index b54d65490..bdd5ffadd 100644 --- a/tests/unit_test.py +++ b/tests/unit_test.py @@ -73,8 +73,6 @@ from codeflare_sdk.utils.generate_yaml import ( gen_names, is_openshift_cluster, - read_template, - write_components, ) import openshift