From ae55bff3e8c503bce8034ea8a6f40b9760d7ec66 Mon Sep 17 00:00:00 2001 From: Daniel Marzini <44803752+danielmarzini@users.noreply.github.com> Date: Tue, 29 Oct 2024 11:32:23 +0100 Subject: [PATCH 01/13] Slurm on GKE - Guide and code --- slurm-on-gke/.gitignore | 6 + slurm-on-gke/README.md | 448 ++++++++++++++++++ slurm-on-gke/image/Dockerfile | 121 +++++ slurm-on-gke/image/docker-entrypoint.sh | 146 ++++++ slurm-on-gke/infrastructure/main.tf | 196 ++++++++ slurm-on-gke/infrastructure/providers.tf | 23 + slurm-on-gke/infrastructure/slurm.tf | 111 +++++ slurm-on-gke/infrastructure/variables.tf | 46 ++ slurm-on-gke/main.tf | 49 ++ slurm-on-gke/modules/slurm-cluster/main.tf | 50 ++ .../00-configmap-slurm-config.yml | 122 +++++ .../00-configmap-slurmdb-config.yml | 50 ++ .../00-secret-database-auth.yml | 29 ++ .../00-secret-munge-key.yml | 29 ++ .../01-pvc-slurm-shared-storage.yml | 34 ++ .../01-pvc-var-lib-mysql.yml | 36 ++ .../01-pvc-var-spool-slurmctld.yml | 36 ++ .../02-svc-deployment-login.yml | 86 ++++ .../02-svc-deployment-mysql.yml | 70 +++ .../02-svc-deployment-slurmdbd.yml | 95 ++++ .../03-svc-statefulset-slurmctld.yml | 103 ++++ .../manifest-templates/04-svc-mysql.yml | 38 ++ .../manifest-templates/04-svc-slurmctld-0.yml | 71 +++ .../manifest-templates/04-svc-slurmdb.yml | 38 ++ slurm-on-gke/modules/slurm-cluster/outputs.tf | 25 + .../modules/slurm-cluster/variables.tf | 59 +++ slurm-on-gke/modules/slurm-nodeset/main.tf | 58 +++ .../03-svc-statefulset-slurmd.yml | 124 +++++ .../manifest-templates/04-svc-slurmd.yml | 39 ++ .../modules/slurm-nodeset/variables.tf | 91 ++++ slurm-on-gke/providers.tf | 44 ++ slurm-on-gke/variables.tf | 87 ++++ 32 files changed, 2560 insertions(+) create mode 100644 slurm-on-gke/.gitignore create mode 100644 slurm-on-gke/README.md create mode 100644 slurm-on-gke/image/Dockerfile create mode 100644 slurm-on-gke/image/docker-entrypoint.sh create mode 100644 slurm-on-gke/infrastructure/main.tf create mode 100644 slurm-on-gke/infrastructure/providers.tf create mode 100644 slurm-on-gke/infrastructure/slurm.tf create mode 100644 slurm-on-gke/infrastructure/variables.tf create mode 100644 slurm-on-gke/main.tf create mode 100644 slurm-on-gke/modules/slurm-cluster/main.tf create mode 100644 slurm-on-gke/modules/slurm-cluster/manifest-templates/00-configmap-slurm-config.yml create mode 100644 slurm-on-gke/modules/slurm-cluster/manifest-templates/00-configmap-slurmdb-config.yml create mode 100644 slurm-on-gke/modules/slurm-cluster/manifest-templates/00-secret-database-auth.yml create mode 100644 slurm-on-gke/modules/slurm-cluster/manifest-templates/00-secret-munge-key.yml create mode 100644 slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-slurm-shared-storage.yml create mode 100644 slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-var-lib-mysql.yml create mode 100644 slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-var-spool-slurmctld.yml create mode 100644 slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-login.yml create mode 100644 slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-mysql.yml create mode 100644 slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-slurmdbd.yml create mode 100644 slurm-on-gke/modules/slurm-cluster/manifest-templates/03-svc-statefulset-slurmctld.yml create mode 100644 slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-mysql.yml create mode 100644 slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-slurmctld-0.yml create mode 100644 slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-slurmdb.yml create mode 100644 slurm-on-gke/modules/slurm-cluster/outputs.tf create mode 100644 slurm-on-gke/modules/slurm-cluster/variables.tf create mode 100644 slurm-on-gke/modules/slurm-nodeset/main.tf create mode 100644 slurm-on-gke/modules/slurm-nodeset/manifest-templates/03-svc-statefulset-slurmd.yml create mode 100644 slurm-on-gke/modules/slurm-nodeset/manifest-templates/04-svc-slurmd.yml create mode 100644 slurm-on-gke/modules/slurm-nodeset/variables.tf create mode 100644 slurm-on-gke/providers.tf create mode 100644 slurm-on-gke/variables.tf diff --git a/slurm-on-gke/.gitignore b/slurm-on-gke/.gitignore new file mode 100644 index 000000000..cbfd8f8f8 --- /dev/null +++ b/slurm-on-gke/.gitignore @@ -0,0 +1,6 @@ + +.terraform* +terraform.tfstate +terraform.tfstate.backup +terraform.tfvars +*/.terraform \ No newline at end of file diff --git a/slurm-on-gke/README.md b/slurm-on-gke/README.md new file mode 100644 index 000000000..b127cd878 --- /dev/null +++ b/slurm-on-gke/README.md @@ -0,0 +1,448 @@ +# Slurm on GKE + +## Introduction + +This guide shows you how to deploy [Slurm](https://slurm.schedmd.com/documentation.html) on a Google Kubernetes Engine (GKE) cluster. + +Slurm (formerly known as Simple Linux Utility for Resource Management) is a powerful open-source workload manager that’s designed for Linux and Unix-like systems. It's used extensively in high performance computing (HPC) environments, including many of the world's supercomputers and large computer clusters. + +Slurm uses a centralized manager (`slurmctld`) to monitor resources and work, with an optional backup manager for fault tolerance. Each compute server (node) runs a `slurmd` daemon to execute work. An optional `slurmdbd` can record accounting information for multiple Slurm-managed clusters. More information about the Slurm architecture can be found on the [SchedMD documentation website](https://slurm.schedmd.com/overview.html#architecture). + +This guide is intended for platform administrators in an enterprise environment who are already managing Kubernetes or GKE clusters, and who need to set up Slurm clusters for AI/ML teams on Kubernetes. This guide is also for AI/ML startups that already use Kubernetes or GKE to run their workloads, such as inference workloads or web apps, and want to use their existing infrastructure to run training workloads with a Slurm interface. AI/ML teams that are already use Slurm can continue to use a familiar interface while onboarding on to Kubernetes or GKE. + +## Benefits + +Because Slurm is a workload orchestrator, it might seem counterintuitive to run it on Kubernetes–and this scenario is not a common practice. However, a team in an enterprise or an AI startup might run Slurm on Kubernetes for the following reasons: + +* To avoid the sparse allocation of scarce resources, such as GPUs, between different infrastructures–for instance, splitting GPUs between Kubernetes clusters and Slurm clusters running on VMs. +* To help teams that are familiar with Slurm, and less familiar with Kubernetes, learn how to use Kubernetes more quickly. These teams typically already use Kubernetes to run other types of workloads, such as inference workloads, web apps, or stateful apps. + +This solution lets a platform administrator team with previous Kubernetes experience, or an AI/ML team with little-to-no Kubernetes experience, set up the cluster with ready-to-use Terraform modules. + +## Document scope + +This solution, and the Slurm setup presented here, does not support all the Slurm features, and is not intended to be used outside the AI/ML domain. This guide is also not a recommendation document about how or when to use Slurm for HPC workloads. + +The following topics are also out of scope for this guide: + +* How to integrate the various combinations of the Google Cloud VM types. +* How to dynamically split Kubernetes nodes between two orchestrators. +* How to do any advanced configuration that’s different from the one that’s presented in the introduction. + +If you are searching for a guide or an automation tool that can help you set up Slurm on Google Cloud, with a focus on HPC workloads, we recommend that you use the [Google Cloud Cluster Toolkit](https://cloud.google.com/cluster-toolkit/docs/overview). + +## Solution architecture + +The implemented architecture is composed of multiple StatefulSets and Deployments in order to cover the different Slurm components. The following diagram shows a Slurm control plane and a data plane that are running in GKE. + +![Slurm on GKE - Architecture][image1] + +Although the function of each component won’t be described in detail, it’s important to highlight how each component is configured and how it can be customized. The previous diagram identifies two main components: the Slurm control plane and the data plane. + +* The Slurm control pane is hosted on `nodepool-nogpu-1`, and it contains at least three services: + * `The` `slurmctld-0` Pod runs `slurmctld`, which is piloted by the `slurmctld` StatefulSet. + * The `slurmdbd` Pod, which is controlled by a Deployment. + * The login Pod. +* The data plane consists of two different `slurmd-X` instances: + * The first instance, called `slurmd-00`, is hosted on the `G2-standard-4` node pool. + * The second instance, called `slurmd-01`, is hosted on the `N1-standard-8` node pool. + +### Control plane + +Before proceeding further, review the Terraform module configuration for this part of the Cluster. + +```HCL +module "slurm-cluster-001" { + source = "./modules/slurm-cluster/" + namespace = "slurm" + namespace_create = true + + cluster_config = { + name = "linux" + image = "IMAGE_URL_FROM_ARTIFACT_REGISTRY" + database = { + create = true + storage_size_gb = 1 + host = "mysql" + password = "SET_HERE_A_BASE64_ENCODED_PASSWORD" + } + storage = { + size_gb = 100 + type = "filestore" + mount_path = "/home" + } + munge = { + key = "PUT_HERE_YOUR_KEY" + } +} +``` + +By using `namespace` and `namespace_create`, you can specify where to run the control plane. Creating the namespace can be automatic if `namespace_create` is set to `true`. + +The `cluster_config` block, which the previous example code passes directly in the module for readability, specifies the following: + +* All the customizable options of the solution. +* The container image to use to run the cluster. +* The database. +* The storage. +* Munge parameters. + +The `database` block can auto-create a new MariaDB instance, which will be hosted on the cluster with an external volume. Alternatively, you can provide credentials to connect to a compatible external MySQL service, such as [CloudSQL](https://cloud.google.com/sql?hl=en). + +The `storage` block specifies where to mount the shared filesystem in the data plane pods (`slurmd`) that will execute the jobs. The configuration of all the Slurm components, except for the database, will auto-mount on the specified shared filesystem. + +The `munge` block specifies the Munge key that’s used for the secure communication between the cluster nodes. If you’re not familiar with it or with any previous Slurm setup, and you want to create a new one, the Munge key must be 32 kB. The Munge client has a [specific command](https://manpages.ubuntu.com/manpages/focal/man8/create-munge-key.8.html) to create a Munge key, and the SchedMD documentation also describes [another approach](https://slurm.schedmd.com/authentication.html#munge_setup) to creating a Munge key by using `dd`. + +For more information related to the image, see the Image section of this document. + +### Data plane and worker nodes + +The following streamlined configuration for a new set of worker nodes requires only a few parameters: + +```HCL +# example - 1 +module "slurm-workers-001" { + source = "./modules/slurm-nodeset" + name = "slurmd" + config = { + type = "g2-standard-4" + instances = 2 + namespace = module.slurm-cluster-001.namespace + image = var.config.image + } +} + +# example - 2 +module "slurm-workers-002" { + source = "./modules/slurm-nodeset" + name = "slurmd1" + config = { + type = "n1-standard-8" + instances = 1 + namespace = module.slurm-cluster-001.namespace + image = var.config.image + accelerator = { + type = "nvidia-tesla-t4" + count = 2 + } + } +} +``` + +The `config` block specifies nodeset names. To name nodesets, we recommend that you use names such as `slurmd`, `slurmd1`, or `slurmd2`. For more information about this topic, see the Limitations section of this document. + +In addition to nodeset names, you can use the `config` block to configure the following: + +* The type of the instance, which includes all the instance types that GKE supports. For example, N1, C2, A2, or A3. +* The number of instances to dedicate to Slurm. +* The namespace. +* The address of the Slurm image. + +The previous example creates two nodesets: the first nodeset is created on the `g2-standard-4` instances, and the second one is created on the `n1-standard-8 instance`s. Although the configurations for both instance types are similar, N1 instances require some additional parameters because they are machines that [are not bound to a specific GPU version](https://cloud.google.com/compute/docs/gpus). Because you can choose the GPU before the machine is created–for example, when you choose the `n1-standard` machine type–the Terraform module requires you to specify the number of GPUs and the related type. + +## Limitations + +Because Slurm is a workload orchestrator, it overlaps with some of the components, scope, and features of Kubernetes. For example, both Slurm and Kubernetes can scale up a new node to accommodate requested resources, or allocate resources within each single node. + +Although Kubernetes shouldn’t limit the number of StatefulSets that can be added to the Slurm cluster, this solution addresses DNS names and pointing by injecting the names of the StatefulSets directly into the manifests. This type of injection supports up to five static names, which include the three StatefulSets and two additional system domains. + +Because of this limitation, we recommend that you have no more than three nodesets per cluster and to always use nodeset names such as `slurmd`, `slurmd-1,` or `slurmd-2`. + +## Image + +The provided image configuration files, which are available in the `image` directory of the repository, contain all the needed Slurm binaries. It's a best practice to have one container image for each purpose, and Google recommends that you create different images for each purpose before you put this architecture in production. + +The provided Dockerfile and the `docker-entrypoint.sh` file are used to create the base Ubuntu-based image for the setup. Although it can be used, it's not maintained and should be considered only for use in experimentation and testing. + +Another best practice is to tag your images with the NVIDIA driver version that you will install. Doing so will help you manage different images because the tags match the NVIDIA driver that’s installed onto your GKE cluster. + +## Notes and requirements + +The current version of this guide does **not** address running the login pod with users different from root. +The following are required to be present, configured or available: + +* terraform v1.9.3 or newer +* docker 20.10.21 or newer +* kubectl +* A Google Cloud organization + +## Infrastructure and GKE + +This section describes the steps for creating and storing the container image. + +### Set up your environment + +In this tutorial, you use [Cloud Shell](https://cloud.google.com/shell) to manage resources that are hosted on Google Cloud. Cloud Shell is preinstalled with the software that you need for this tutorial, including [`kubectl`](https://kubernetes.io/docs/reference/kubectl/), the [Google Cloud CLI](https://cloud.google.com/sdk/gcloud), [Helm](https://helm.sh/), and [Terraform](https://cloud.google.com/docs/terraform). + +To set up your environment with Cloud Shell, follow these steps: + +1\. In the [Google Cloud console](http://console.cloud.google.com), launch a Cloud Shell session by clicking *Activate Cloud Shell*. This launches a session in the bottom pane of the Google Cloud console. + +2\. Set environment variables. + +```bash +export PROJECT_ID=YOUR_PROJECT_ID +export REGION=europe-west3 +``` + +Replace *YOUR_PROJECT_ID* with your Google Cloud [project ID](https://cloud.google.com/resource-manager/docs/creating-managing-projects#identifying_projects). + +3\. Set the default environment variables. + +```bash +gcloud config set project ${PROJECT_ID} +``` + +4\. Clone the code repository. + +```bash +git clone https://github.com/GoogleCloudPlatform/ai-on-gke +``` + +### Create your cluster infrastructure + +In this section, you run a Terraform script to create a private, highly available, regional GKE cluster. + +The Terraform module will also create a new project and anything that might be needed to set up the environment described in this guide. If you already have a GKE cluster where you can test the Slurm installation, you can skip this step. + +5\. Initialize Terraform. + +```bash +cd slurm-on-gke +cd infrastructure +terraform init +``` + +6\. Create the `terraform.tfvars` file with your own values. + +A `terraform.tfvars` file should be provided with all the values for the required variables. In the file, enter your own values as follows: + +```HCL +impersonate_service_account = "YOUR_SERVICE_ACCOUNT_ID" <-- if you are using one +region = "europe-west3" +project_id = "YOUR_PROJECT_ID" +billing_account_id = "YOUR_BILLING_ACCOUNT_ID" +folder_id = "folders/FOLDER_ID" +``` + +**Note:** Ensure your selected region or zone offers GPU availability. Consult the [Google Cloud documentation for a complete list](https://cloud.google.com/compute/docs/gpus/gpu-regions-zones). + +7\. After you fill out the file, use the following command to apply the Terraform configuration and create the infrastructure. + +```bash +terraform apply +``` + +When you are prompted, type `yes`. It might take several minutes for this command to complete and for the cluster to show a ready status. + +Terraform creates the following resources: + +* A Google Cloud project. +* A VPC network, a private subnet for the Kubernetes nodes, and a proxy-only subnet for the load balancers. +* A firewall rule to open the SSH protocol from the Identity-Aware Proxy (IAP) ranges. +* A router to access the internet through NAT. +* An Artifact Registry repository to host the Slurm image. +* A private GKE cluster in the `europe-west3` region. +* One node pool with autoscaling enabled (1-2 nodes per zone, 1 node per zone minimum) +* One node pool with enabled autoscaling and GPUs (1-2 nodes per zone, 1 node per zone minimum) +* Two ServiceAccount with logging, monitoring permissions, and Artifact Registry read permissions. +* Google Cloud Managed Service for Prometheus configuration for cluster monitoring. + +The output is similar to the following: + +```bash +... +Apply complete! Resources: 39 added, 0 changed, 0 destroyed. +``` + +An additional, commented-out Terraform configuration is already written over the `infrastructure/slurm.tf` file. The additional configuration is an example configuration for a `g2-standard-4 node` pool. + +### Create the image + +In this section, you build and store the container image that you will use to deploy Slurm over the newly created or provided GKE cluster. + +To build the container image, use the following commands: + +```bash +cd .. # (you were in the infrastructure directory) +cd image docker build -t europe-west3-docker.pkg.dev/$PROJECT_ID/slurm/slurmd:535 . +docker push europe-west3-docker.pkg.dev/$PROJECT_ID/slurm/slurmd:535 +``` + + +The output is similar to the following: + +```bash +... +The push refers to repository [europe-west3-docker.pkg.dev/YOUR_PROJECT_ID/slurm/slurmd] +df1644670bb2: Pushed +74f700e9690e: Pushed +676e6ba12678: Pushed +578df7510db0: Pushed +6551dcc8d929: Pushed +ddcaaa531045: Pushed +98c2ee5d21b6: Pushed +866b7df6f372: Pushed +139722e64731: Pushed +87c242b383a9: Pushed +1b9b7340fee7: Pushed +535: digest: sha256:ced97f7cb5d0eba7114a1909c2he2e2ke21a6db1b36669a41f34a3 +size: 2632 +``` + +Note the address of your container image because it will be requested in the following steps.t The address should be similar to the following: + +```bash +europe-west3-docker.pkg.dev/$PROJECT_ID/slurm/slurmd:535 +``` + +### Deploy Slurm + +In this section, you deploy the Slurm cluster over the newly created or provided GKE cluster. + +1\. Return to the repository root directory. + +```bash +cd .. # (we were in the image directory) +``` + +2\. Create the `terraform.tfvars` file with your own values. + +A `terraform.tfvars` file should be provided with all the values for the required variables. In the file, enter your own values as follows: + +```HCL +region = "europe-west3" +project_id = "YOUR_PROJECT_ID" +cluster_name = "cluster-1" + +config = { + name = "linux" + image = "europe-west3-docker.pkg.dev/YOUR_PROJECT_ID/slurm/slurmd:535" + database = { + create = true + storage_size_gb = 1 + host = "mysql" + password = "SET_HERE_A_BASE64_ENCODED_PASSWORD" + } + storage = { + size_gb = 100 + type = "filestore" + mount_path = "/home" + } + munge = { + key = "PUT_HERE_YOUR_KEY" + } +} + +impersonate_service_account = "YOUR_SERVICE_ACCOUNT_ID" <-- if you are using one + +``` + +3\. Gather the credentials for the GKE cluster. + +```bash +gcloud container clusters get-credentials cluster-1 --region europe-west3 +``` + +4\. Initialize the Terraform configuration and apply it. + +```bash +terraform init +terraform apply +``` + +When you are prompted, type `yes`. It might take several seconds for this command to complete and for the cluster to show a ready status. + +The output is similar to the following: + +```bash +... +Apply complete! Resources: 17 added, 0 changed, 0 destroyed. +``` + +5\. Check that the Slurm cluster is being deployed. + +```bash +kubectl get pods -n slurm -w +NAME READY STATUS RESTARTS AGE +login-96bffd678-nbqwp 0/1 Pending 0 32s +mysql-746bcd47c6-mxd4f 1/1 Running 0 2m28s +slurmctld-0 0/1 Pending 0 2m29s +slurmd1-0 0/1 Pending 0 2m27s +slurmdbd-7b67cf9b54-dj7p4 0/1 ContainerCreating 0 31s +``` + +After deployment is complete, the output is similar to the following: + +```bash +kubectl get pods -n slurm +NAME READY STATUS RESTARTS AGE +login-96bffd678-nbqwp 1/1 Running 0 4m12s +mysql-746bcd47c6-mxd4f 1/1 Running 0 6m8s +slurmctld-0 1/1 Running 0 4s +slurmd1-0 0/1 Running 0 19s +slurmdbd-7b67cf9b54-dj7p4 1/1 Running 0 4m11s +``` + +6\. Verify that the Slurm cluster is working properly by logging in to the login pod. + +```bash +kubectl -n slurm exec -it login-96bffd678-nbqwp -- bash +root@login:/opt# sinfo +PARTITION AVAIL TIMELIMIT NODES STATE NODELIST +all\* up infinite 79 idle~ slurmd-[0-39],slurmd1-[1-39] +all\* up infinite 1 idle slurmd1-0 +1gpunodes up infinite 40 idle slurmd-[0-39] +2gpunodes up infinite 39 idle~ slurmd1-[1-39] +2gpunodes up infinite 1 idle slurmd1-0 root@login:/opt +``` + +## Clean up + +To avoid incurring charges to your Google Cloud account for the resources that you used in this tutorial, either delete the project that contains the resources, or keep the project and delete the individual resources. + +### Delete the project + +The easiest way to avoid billing is to delete the project that you created for the tutorial. + +Caution: Deleting a project has the following effects: + +* Everything in the project is deleted. If you used an existing project for the tasks in this document, deleting the project also deletes any other work that you've done in the project. +* Custom project IDs are lost. When you created this project, you might have created a custom project ID that you want to use in the future. To preserve the URLs that use the project ID, such as an appspot.com URL, delete selected resources inside the project instead of deleting the whole project. + +If you plan to explore multiple architectures, tutorials, or quickstarts, reusing projects can help you avoid exceeding project quota limits. + +1. In the Google Cloud console, go to the **Manage resources** page. + [Go to Manage resources](https://console.cloud.google.com/iam-admin/projects). +2. In the project list, select the project that you want to delete, and then click **Delete**. +3. In the dialog, type the project ID, and then click **Shut down** to delete the project. + +### Delete the individual resources + +If you used an existing project and you don't want to delete it entirely, delete the +individual resources. + +1\. Run the terraform destroy command to delete all the Slurm resources that you created in the previous steps: + +```bash +cd slurm-on-gke terraform destroy +``` + +If you used an existing cluster, you can skip the following step. + +2\. Run the terraform destroy command on the infrastructure directory: + +```bash +cd infrastructure +terraform destroy +``` + +This step deletes all the resources that you created previously: the GKE cluster, the VPC network, the firewall rules, and the Google Cloud project. + +## License + +* The use of the assets contained in this repository is subject to compliance with [Google's AI Principles](https://ai.google/responsibility/principles/) +* See [LICENSE](https://github.com/GoogleCloudPlatform/ai-on-gke/blob/main/LICENSE) +* This is inspired by the [Stack HPC \- slurm k8s cluster project](https://github.com/stackhpc/slurm-k8s-cluster), copied or derived files from the original repository are under the original MIT license and/or [Giovanni Torres copyright](https://github.com/stackhpc/slurm-k8s-cluster/blob/main/LICENSE). + +[image1]: diff --git a/slurm-on-gke/image/Dockerfile b/slurm-on-gke/image/Dockerfile new file mode 100644 index 000000000..7f70fe7b5 --- /dev/null +++ b/slurm-on-gke/image/Dockerfile @@ -0,0 +1,121 @@ +# MIT License + +# Copyright (c) 2019 Giovanni Torres + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +FROM ubuntu:22.04 + +ARG SLURM_TAG=slurm-23.02 +ARG GOSU_VERSION=1.11 + + +RUN set -x \ + && apt-get update \ + && apt-get install -y \ + wget \ + gcc \ + git \ + make \ + munge \ + libmunge-dev \ + python3-dev \ + python3-pip \ + python3 \ + hwloc \ + libhwloc-dev \ + libpmix-dev \ + libhttp-parser-dev \ + libmysqlclient-dev \ + libjson-c-dev \ + psmisc \ + bzip2 \ + python3-http-parser \ + nvidia-utils-535 \ + nvidia-cuda-toolkit-gcc \ + nvidia-cuda-dev \ + libnvidia-compute-535 \ + mariadb-server \ + libdbus-1-dev \ + openmpi-common \ + openmpi-bin \ + vim + +RUN pip3 install Cython nose + +RUN set -ex \ + && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" \ + && wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64.asc" \ + && export GNUPGHOME="$(mktemp -d)" \ + && gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 \ + && gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu \ + && rm -rf "${GNUPGHOME}" /usr/local/bin/gosu.asc \ + && chmod +x /usr/local/bin/gosu \ + && gosu nobody true + +ENV SHELL=/bin/bash +RUN set -x \ + && git clone -b ${SLURM_TAG} --single-branch --depth=1 https://github.com/SchedMD/slurm.git \ + && cd slurm \ + && ./configure --enable-debug --prefix=/usr --sysconfdir=/etc/slurm \ + --with-mysql_config=/usr/bin --libdir=/usr/lib64 \ + && make install \ + && install -D -m644 contribs/slurm_completion_help/slurm_completion.sh /etc/profile.d/slurm_completion.sh \ + && cd .. \ + && rm -rf slurm + +RUN mkdir /var/spool/slurmd \ + /var/run/slurmd \ + /var/run/slurmdbd \ + /var/lib/slurmd \ + /var/log/slurm \ + /data \ + /etc/slurm \ + && touch /var/lib/slurmd/node_state \ + /var/lib/slurmd/front_end_state \ + /var/lib/slurmd/job_state \ + /var/lib/slurmd/resv_state \ + /var/lib/slurmd/trigger_state \ + /var/lib/slurmd/assoc_mgr_state \ + /var/lib/slurmd/assoc_usage \ + /var/lib/slurmd/qos_usage \ + /var/lib/slurmd/fed_mgr_state \ + && useradd -r --uid=990 slurm \ + && chown -R slurm:slurm /var/*/slurm* + +WORKDIR /opt +RUN export VERSION=1.18 OS=linux ARCH=amd64 \ + && wget https://dl.google.com/go/go$VERSION.$OS-$ARCH.tar.gz \ + && tar -xzvf go$VERSION.$OS-$ARCH.tar.gz \ + && export PATH=$PWD/go/bin:$PATH \ + && git clone https://github.com/vpenso/prometheus-slurm-exporter.git \ + && cd prometheus-slurm-exporter \ + && go build + +RUN mv /opt/prometheus-slurm-exporter/prometheus-slurm-exporter /usr/local/bin + +RUN mkdir -p /run/munge \ + && chown munge.munge /run/munge + +VOLUME /etc/slurm +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN chmod 755 /usr/local/bin/docker-entrypoint.sh +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] + +CMD ["slurmdbd"] \ No newline at end of file diff --git a/slurm-on-gke/image/docker-entrypoint.sh b/slurm-on-gke/image/docker-entrypoint.sh new file mode 100644 index 000000000..66ebac5d6 --- /dev/null +++ b/slurm-on-gke/image/docker-entrypoint.sh @@ -0,0 +1,146 @@ +# MIT License + +# Copyright (c) 2019 Giovanni Torres + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +#!/bin/bash +set -euo pipefail + +function start_munge(){ + + echo "---> Copying MUNGE key ..." + cp /tmp/munge.key /etc/munge/munge.key + chown munge:munge /etc/munge/munge.key + + echo "---> Starting the MUNGE Authentication service (munged) ..." + gosu munge /usr/sbin/munged "$@" +} + +if [ "$1" = "slurmdbd" ] +then + + start_munge + + echo "---> Starting the Slurm Database Daemon (slurmdbd) ..." + + cp /tmp/slurmdbd.conf /etc/slurm/slurmdbd.conf + echo "StoragePass=${StoragePass}" >> /etc/slurm/slurmdbd.conf + chown slurm:slurm /etc/slurm/slurmdbd.conf + chmod 600 /etc/slurm/slurmdbd.conf + { + . /etc/slurm/slurmdbd.conf + until echo "SELECT 1" | mysql -h $StorageHost -u$StorageUser -p$StoragePass 2>&1 > /dev/null + do + echo "-- Waiting for database to become active ..." + sleep 2 + done + } + echo "-- Database is now active ..." + + exec gosu slurm /usr/sbin/slurmdbd -D "${@:2}" + +elif [ "$1" = "slurmctld" ] +then + + start_munge + + echo "---> Waiting for slurmdbd to become active before starting slurmctld ..." + + until 2>/dev/null >/dev/tcp/slurmdbd/6819 + do + echo "-- slurmdbd is not available. Sleeping ..." + sleep 2 + done + echo "-- slurmdbd is now active ..." + + echo "---> Setting permissions for state directory ..." + chown slurm:slurm /var/spool/slurmctld + + echo "---> Starting the Slurm Controller Daemon (slurmctld) ..." + if /usr/sbin/slurmctld -V | grep -q '17.02' ; then + exec gosu slurm /usr/sbin/slurmctld -D "${@:2}" + else + exec gosu slurm /usr/sbin/slurmctld -i -D "${@:2}" + fi + +elif [ "$1" = "slurmd" ] +then + echo "---> Set shell resource limits ..." + #ulimit -l unlimited + #ulimit -s unlimited + #ulimit -n 131072 + #ulimit -a + + start_munge + + cgroup_dir=`find /sys/fs/cgroup/kubepods.slice -type d -name "kubepods-pod*"` + mkdir -p "$cgroup_dir/system.slice/slurmstepd.scope/system.slice/slurmstepd.scope" + mkdir -p "/var/spool/slurmd" + + echo "---> Starting the Slurm Node Daemon (slurmd) ..." + echo "${@:1}" + exec slurmd -D -s -vvv --conf-server="slurmctld-0:6820-6830" -Z -N $POD_NAME + +elif [ "$1" = "login" ] +then + + start_munge + while true; do sleep 30; done; + +elif [ "$1" = "check-queue-hook" ] +then + start_munge + + scontrol update NodeName=all State=DRAIN Reason="Preventing new jobs running before upgrade" + + RUNNING_JOBS=$(squeue --states=RUNNING,COMPLETING,CONFIGURING,RESIZING,SIGNALING,STAGE_OUT,STOPPED,SUSPENDED --noheader --array | wc --lines) + + if [[ $RUNNING_JOBS -eq 0 ]] + then + exit 0 + else + exit 1 + fi + +elif [ "$1" = "undrain-nodes-hook" ] +then + start_munge + scontrol update NodeName=all State=UNDRAIN + exit 0 + +elif [ "$1" = "generate-keys-hook" ] +then + mkdir -p ./temphostkeys/etc/ssh + ssh-keygen -A -f ./temphostkeys + kubectl create secret generic host-keys-secret \ + --dry-run=client \ + --from-file=./temphostkeys/etc/ssh \ + -o yaml | \ + kubectl apply -f - + + exit 0 + +elif [ "$1" = "debug" ] +then + start_munge --foreground + +else + exec "$@" +fi \ No newline at end of file diff --git a/slurm-on-gke/infrastructure/main.tf b/slurm-on-gke/infrastructure/main.tf new file mode 100644 index 000000000..0ff4d291b --- /dev/null +++ b/slurm-on-gke/infrastructure/main.tf @@ -0,0 +1,196 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "project" { + source = "github.com/terraform-google-modules/cloud-foundation-fabric//modules/project?ref=v33.0.0" + billing_account = var.billing_account_id + name = var.project_id + parent = var.folder_id + services = [ + "compute.googleapis.com", + "stackdriver.googleapis.com", + "container.googleapis.com", + "file.googleapis.com", + "servicenetworking.googleapis.com" + ] +} + +module "vpc" { + source = "github.com/terraform-google-modules/cloud-foundation-fabric//modules/net-vpc?ref=v33.0.0" + project_id = module.project.project_id + name = "default" + subnets = [ + { + ip_cidr_range = "10.0.0.0/24" + name = "subnet-1" + region = var.region + secondary_ip_ranges = { + pods = "172.16.0.0/20" + services = "192.168.0.0/24" + } + }, + { + ip_cidr_range = "10.10.0.0/24" + name = "subnet-lb-1" + region = var.region + } + ] + subnets_proxy_only = [ + { + ip_cidr_range = "10.0.1.0/24" + name = "regional-proxy" + region = var.region + active = true + } + ] +} + + + +module "firewall" { + source = "github.com/terraform-google-modules/cloud-foundation-fabric//modules/net-vpc-firewall?ref=v33.0.0" + project_id = module.project.project_id + network = module.vpc.network.name + default_rules_config = { + admin_ranges = ["10.0.0.0/8"] + } + ingress_rules = { + # implicit allow action + allow-ingress-ssh = { + description = "Allow SSH from IAP" + source_ranges = ["35.235.240.0/20"] + rules = [{ protocol = "tcp", ports = [22] }] + } + } +} + +module "nat" { + source = "github.com/terraform-google-modules/cloud-foundation-fabric//modules/net-cloudnat?ref=v33.0.0" + project_id = module.project.project_id + region = var.region + name = "default" + router_network = module.vpc.network.self_link +} + +module "docker_artifact_registry" { + source = "github.com/terraform-google-modules/cloud-foundation-fabric//modules/artifact-registry?ref=v33.0.0" + project_id = module.project.project_id + location = var.region + name = "slurm" + format = { docker = { standard = {} } } +} + +module "cluster_nodepool_sa" { + source = "github.com/terraform-google-modules/cloud-foundation-fabric//modules/iam-service-account?ref=v33.0.0" + project_id = module.project.project_id + name = "cluster-nodepool-sa" + iam_project_roles = { + "${module.project.project_id}" = [ + "roles/monitoring.metricWriter", + "roles/logging.logWriter", + "roles/artifactregistry.reader", + ] + } +} + + +module "cluster-1" { + source = "github.com/terraform-google-modules/cloud-foundation-fabric//modules/gke-cluster-standard?ref=v33.0.0" + project_id = module.project.project_id + name = "cluster-1" + location = var.region + release_channel = "RAPID" + deletion_protection = false + vpc_config = { + network = module.vpc.self_link + subnetwork = module.vpc.subnets["${var.region}/subnet-1"].self_link + secondary_range_names = { + pods = "pods" + services = "services" + } + master_ipv4_cidr_block = "172.19.27.0/28" + } + enable_addons = { + gce_persistent_disk_csi_driver = true + http_load_balancing = true + horizontal_pod_autoscaling = true + gcp_filestore_csi_driver = true + gcs_fuse_csi_driver = true + } + private_cluster_config = { + enable_private_endpoint = false + master_global_access = true + } + enable_features = { + dataplane_v2 = true + workload_identity = true + image_streaming = true + intranode_visibility = true + + dns = { + provider = "CLOUD_DNS" + scope = "CLUSTER_SCOPE" + } + } + + backup_configs = { + enable_backup_agent = false + } + monitoring_config = { + enable_api_server_metrics = true + enable_controller_manager_metrics = true + enable_scheduler_metrics = true + } + logging_config = { + enable_workloads_logs = true + } +} + +module "cluster-1-nodepool-1" { + source = "github.com/terraform-google-modules/cloud-foundation-fabric//modules/gke-nodepool?ref=v33.0.0" + project_id = module.project.project_id + cluster_name = module.cluster-1.name + location = var.region + name = "nodepool-1" + service_account = { + create = false + email = module.cluster_nodepool_sa.email + oauth_scopes = [ + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + "https://www.googleapis.com/auth/monitoring.write", + "https://www.googleapis.com/auth/cloud-platform" + ] + } + node_config = { + machine_type = "n1-standard-8" + disk_size_gb = 100 + disk_type = "pd-ssd" + ephemeral_ssd_count = 1 + gvnic = true + } + nodepool_config = { + autoscaling = { + max_node_count = 10 + min_node_count = 2 + } + management = { + auto_repair = true + auto_upgrade = true + } + } +} + diff --git a/slurm-on-gke/infrastructure/providers.tf b/slurm-on-gke/infrastructure/providers.tf new file mode 100644 index 000000000..73b326e04 --- /dev/null +++ b/slurm-on-gke/infrastructure/providers.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +provider "google" { + impersonate_service_account = var.impersonate_service_account +} + +provider "google-beta" { + impersonate_service_account = var.impersonate_service_account +} diff --git a/slurm-on-gke/infrastructure/slurm.tf b/slurm-on-gke/infrastructure/slurm.tf new file mode 100644 index 000000000..0dfcbe9d1 --- /dev/null +++ b/slurm-on-gke/infrastructure/slurm.tf @@ -0,0 +1,111 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +module "slurm_nodepool_sa" { + source = "github.com/terraform-google-modules/cloud-foundation-fabric//modules/iam-service-account?ref=v33.0.0" + project_id = module.project.project_id + name = "slurm-nodepool-sa" + # non-authoritative roles granted *to* the service accounts on other resources + iam_project_roles = { + "${module.project.project_id}" = [ + "roles/monitoring.metricWriter", + "roles/logging.logWriter", + "roles/artifactregistry.reader", + ] + } +} + +module "cluster-1-nodepool-2" { + source = "github.com/terraform-google-modules/cloud-foundation-fabric//modules/gke-nodepool?ref=v33.0.0" + project_id = module.project.project_id + cluster_name = module.cluster-1.name + location = var.region + name = "slurm-001" + service_account = { + create = false + email = module.slurm_nodepool_sa.email + oauth_scopes = [ + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + "https://www.googleapis.com/auth/monitoring.write", + "https://www.googleapis.com/auth/cloud-platform" + ] + } + node_locations = ["${var.region}-b"] + node_config = { + machine_type = "n1-standard-8" + disk_size_gb = 100 + disk_type = "pd-ssd" + gvnic = true + spot = false + guest_accelerator = { + type = "nvidia-tesla-t4" + count = 2 + gpu_driver = { + version = "DEFAULT" + } + } + } + + nodepool_config = { + autoscaling = { + max_node_count = 10 + min_node_count = 1 + } + management = { + auto_repair = true + auto_upgrade = true + } + } +} + +module "cluster-1-nodepool-3" { + source = "github.com/terraform-google-modules/cloud-foundation-fabric//modules/gke-nodepool?ref=v33.0.0" + project_id = module.project.project_id + cluster_name = module.cluster-1.name + location = var.region + name = "slurm-002" + service_account = { + create = false + email = module.slurm_nodepool_sa.email + oauth_scopes = [ + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + "https://www.googleapis.com/auth/monitoring.write", + "https://www.googleapis.com/auth/cloud-platform" + ] + } + node_locations = ["${var.region}-b"] + node_config = { + machine_type = "g2-standard-4" + disk_size_gb = 100 + disk_type = "pd-ssd" + gvnic = true + spot = false + } + + nodepool_config = { + autoscaling = { + max_node_count = 10 + min_node_count = 2 + } + management = { + auto_repair = true + auto_upgrade = true + } + } +} diff --git a/slurm-on-gke/infrastructure/variables.tf b/slurm-on-gke/infrastructure/variables.tf new file mode 100644 index 000000000..b95d2cf08 --- /dev/null +++ b/slurm-on-gke/infrastructure/variables.tf @@ -0,0 +1,46 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "billing_account_id" { + description = "Google Cloud Billing Account ID" + type = string + nullable = false +} + +variable "folder_id" { + description = "Google Cloud Folder ID" + type = string + nullable = false +} + +variable "impersonate_service_account" { + description = "Service account to be used while using Google Cloud APIs" + type = string + nullable = true + default = null +} + +variable "region" { + description = "Google Cloud Region where the GKE cluster is located" + type = string + nullable = false +} + +variable "project_id" { + description = "Google Cloud Project ID" + type = string + nullable = false +} diff --git a/slurm-on-gke/main.tf b/slurm-on-gke/main.tf new file mode 100644 index 000000000..3f60ec179 --- /dev/null +++ b/slurm-on-gke/main.tf @@ -0,0 +1,49 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "slurm-cluster-001" { + source = "./modules/slurm-cluster/" + namespace = "slurm" + namespace_create = true + + cluster_config = var.config +} + +module "slurm-workers-001" { + source = "./modules/slurm-nodeset" + name = "slurmd1" + config = { + type = "n1-standard-8" + instances = 1 + namespace = module.slurm-cluster-001.namespace + image = var.config.image + accelerator = { + type = "nvidia-tesla-t4" + count = 2 + } + } +} + +module "slurm-workers-002" { + source = "./modules/slurm-nodeset" + name = "slurmd" + config = { + type = "g2-standard-4" + instances = 2 + namespace = module.slurm-cluster-001.namespace + image = var.config.image + } +} diff --git a/slurm-on-gke/modules/slurm-cluster/main.tf b/slurm-on-gke/modules/slurm-cluster/main.tf new file mode 100644 index 000000000..3093b030b --- /dev/null +++ b/slurm-on-gke/modules/slurm-cluster/main.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +locals { + wl_templates = [ + for f in fileset(local.wl_templates_path, "[0-9]*yml") : + "${local.wl_templates_path}/${f}" + ] + wl_templates_path = ( + var.templates_path == null + ? "${path.module}/manifest-templates" + : pathexpand(var.templates_path) + ) +} + +resource "kubernetes_namespace" "default" { + count = var.namespace_create ? 1 : 0 + metadata { + name = var.namespace + } +} + +resource "kubernetes_manifest" "default" { + for_each = toset(local.wl_templates) + manifest = yamldecode(templatefile(each.value, { + namespace = var.namespace + cluster_config = var.cluster_config + })) + + timeouts { + create = "30m" + } + field_manager { + force_conflicts = true + } +} diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-configmap-slurm-config.yml b/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-configmap-slurm-config.yml new file mode 100644 index 000000000..e7228ff71 --- /dev/null +++ b/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-configmap-slurm-config.yml @@ -0,0 +1,122 @@ +# +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v1 +kind: ConfigMap +metadata: + name: slurm-conf-configmap + namespace: ${namespace} +data: + slurm.conf: | + # slurm.conf + # + # See the slurm.conf man page for more information. + # + ClusterName=linux + SlurmctldHost=slurmctld-0 + # + SlurmUser=slurm + SlurmctldPort=6820-6830 + SlurmdPort=6818 + AuthType=auth/munge + StateSaveLocation=/var/spool/slurmctld + SlurmdSpoolDir=/var/spool/slurmd + SwitchType=switch/none + MpiDefault=pmix + SlurmctldPidFile=/var/run/slurmd/slurmctld.pid + SlurmdPidFile=/var/run/slurmd/slurmd.pid + ProctrackType=proctrack/linuxproc + ReturnToService=2 + # + # TIMERS + SlurmctldTimeout=300 + SlurmdTimeout=30 + InactiveLimit=0 + MinJobAge=300 + KillWait=30 + Waittime=0 + # + # SCHEDULING + SchedulerType=sched/backfill + SelectType=select/cons_tres + SelectTypeParameters=CR_CPU_Memory + # + # LOGGING + SlurmctldDebug=3 + SlurmctldLogFile=/var/log/slurm/slurmctld.log + SlurmdDebug=3 + SlurmdLogFile=/var/log/slurm/slurmd.log + JobCompType=jobcomp/filetxt + JobCompLoc=/var/log/slurm/jobcomp.log + # + # ACCOUNTING + JobAcctGatherType=jobacct_gather/linux + JobAcctGatherFrequency=30 + # + AccountingStorageType=accounting_storage/slurmdbd + AccountingStorageHost=slurmdbd + AccountingStoragePort=6819 + # + SlurmctldParameters=cloud_reg_addrs + + # CLOUD CONFIGURATIONS + MaxNodeCount=64000 + include cloud.conf + cloud.conf: | + PrivateData=cloud + SlurmctldParameters=enable_configless + ## GRES + GresTypes=gpu + AccountingStorageTRES=gres/gpu + DebugFlags=Gres + TreeWidth=128 + + # NODES + NodeName=DEFAULT State=UNKNOWN RealMemory=15000 CPUs=4 CoresPerSocket=2 ThreadsPerCore=2 Gres=gpu:1 + NodeName=slurmd-[0-39] State=CLOUD Gres=gpu:1 + NodeSet=slurmdnodeset Nodes=slurmd-[0-39] + + NodeName=DEFAULT State=UNKNOWN RealMemory=30000 CPUs=8 CoresPerSocket=2 ThreadsPerCore=2 Gres=gpu:2 + NodeName=slurmd1-[0-39] State=CLOUD Gres=gpu:2 + NodeSet=slurmd1nodeset Nodes=slurmd1-[0-39] + + # PARTITIONS + PartitionName=all Default=yes Nodes=ALL MaxTime=INFINITE State=UP + + PropagateResourceLimitsExcept=MEMLOCK + + PartitionName=1gpunodes Nodes=slurmdnodeset State=UP DefMemPerCPU=7007 SuspendTime=300 Oversubscribe=Exclusive PowerDownOnIdle=YES ResumeTimeout=300 SuspendTimeout=120 + PartitionName=2gpunodes Nodes=slurmd1nodeset State=UP DefMemPerCPU=7007 SuspendTime=300 Oversubscribe=Exclusive PowerDownOnIdle=YES ResumeTimeout=300 SuspendTimeout=120 + + cloud_gres.conf: | + NodeName=slurmd-[0-39] Name=gpu File=/dev/nvidia0 + NodeName=slurmd1-[0-39] Name=gpu File=/dev/nvidia[0-1] + gres.conf: | + NodeName=slurmd-[0-39] Name=gpu File=/dev/nvidia0 + NodeName=slurmd1-[0-39] Name=gpu File=/dev/nvidia[0-1] + cgroup.conf: | + ### + # + # Slurm cgroup support configuration file + # + # See man slurm.conf and man cgroup.conf for further + # information on cgroup configuration parameters + #-- + ConstrainCores=yes + ConstrainDevices=yes + ConstrainRAMSpace=yes + ConstrainSwapSpace=yes + IgnoreSystemd=yes \ No newline at end of file diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-configmap-slurmdb-config.yml b/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-configmap-slurmdb-config.yml new file mode 100644 index 000000000..e82c28285 --- /dev/null +++ b/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-configmap-slurmdb-config.yml @@ -0,0 +1,50 @@ +# MIT License + +# Copyright (c) 2019 Giovanni Torres + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: slurmdbd-conf-configmap + namespace: ${namespace} +data: + slurmdbd.conf: | + # + # Example slurmdbd.conf file. + # + # See the slurmdbd.conf man page for more information. + # + # Authentication info + AuthType=auth/munge + # + # slurmDBD info + DbdAddr=slurmdbd + DbdHost=slurmdbd + SlurmUser=slurm + DebugLevel=4 + LogFile=/var/log/slurm/slurmdbd.log + PidFile=/var/run/slurmdbd/slurmdbd.pid + # + # Database info + StorageType=accounting_storage/mysql + StorageHost=${cluster_config.database.host} + StorageUser=${cluster_config.database.user} + diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-secret-database-auth.yml b/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-secret-database-auth.yml new file mode 100644 index 000000000..3033d3912 --- /dev/null +++ b/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-secret-database-auth.yml @@ -0,0 +1,29 @@ +# MIT License + +# Copyright (c) 2019 Giovanni Torres + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +apiVersion: v1 +kind: Secret +metadata: + name: database-auth-secret + namespace: ${namespace} +data: + password: ${cluster_config.database.password} \ No newline at end of file diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-secret-munge-key.yml b/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-secret-munge-key.yml new file mode 100644 index 000000000..5d20f540e --- /dev/null +++ b/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-secret-munge-key.yml @@ -0,0 +1,29 @@ +# MIT License + +# Copyright (c) 2019 Giovanni Torres + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +apiVersion: v1 +kind: Secret +metadata: + name: munge-key-secret + namespace: ${namespace} +data: + munge.key: ${base64encode(cluster_config.munge.key)} \ No newline at end of file diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-slurm-shared-storage.yml b/slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-slurm-shared-storage.yml new file mode 100644 index 000000000..992418286 --- /dev/null +++ b/slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-slurm-shared-storage.yml @@ -0,0 +1,34 @@ +# MIT License + +# Copyright (c) 2019 Giovanni Torres + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: slurm-shared-storage + namespace: ${namespace} +spec: + storageClassName: standard-rwx + accessModes: + - ReadWriteMany + resources: + requests: + storage: ${cluster_config.storage.size_gb}Gi \ No newline at end of file diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-var-lib-mysql.yml b/slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-var-lib-mysql.yml new file mode 100644 index 000000000..f6a86768b --- /dev/null +++ b/slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-var-lib-mysql.yml @@ -0,0 +1,36 @@ +# MIT License + +# Copyright (c) 2019 Giovanni Torres + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + app.kubernetes.io/name: slurm + app.kubernetes.io/component: mysql + name: var-lib-mysql + namespace: ${namespace} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: ${cluster_config.database.storage_size_gb}Gi \ No newline at end of file diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-var-spool-slurmctld.yml b/slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-var-spool-slurmctld.yml new file mode 100644 index 000000000..2e6eef12e --- /dev/null +++ b/slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-var-spool-slurmctld.yml @@ -0,0 +1,36 @@ +# MIT License + +# Copyright (c) 2019 Giovanni Torres + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + app.kubernetes.io/name: slurm + app.kubernetes.io/component: slurmctld + name: var-spool-slurmctld + namespace: ${namespace} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Mi \ No newline at end of file diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-login.yml b/slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-login.yml new file mode 100644 index 000000000..a1a6e75cc --- /dev/null +++ b/slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-login.yml @@ -0,0 +1,86 @@ +# MIT License + +# Copyright (c) 2019 Giovanni Torres + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: slurm + app.kubernetes.io/component: login + name: login + namespace: ${namespace} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: slurm + app.kubernetes.io/component: login + strategy: + type: Recreate + template: + metadata: + labels: + app.kubernetes.io/name: slurm + app.kubernetes.io/component: login + spec: + containers: + - args: + - login + image: ${cluster_config.image} + name: login + ports: + - containerPort: 22 + volumeMounts: + - mountPath: ${cluster_config.storage.mount_path} + name: slurm-jobdir + - mountPath: /etc/slurm/ + name: slurm-config-volume + - mountPath: /tmp/munge.key + name: munge-key-secret + subPath: munge.key + hostname: login + dnsPolicy: "None" + dnsConfig: + nameservers: + - "169.254.169.254" + searches: + - slurmd.${namespace}.svc.cluster.local + - slurmd1.${namespace}.svc.cluster.local + - slurmd2.${namespace}.svc.cluster.local + - svc.cluster.local + - cluster.local + - ${namespace}.svc.cluster.local + options: + - name: ndots + value: "5" + restartPolicy: Always + volumes: + - name: slurm-jobdir + persistentVolumeClaim: + claimName: slurm-shared-storage + - name: slurm-config-volume + configMap: + name: slurm-conf-configmap + - name: munge-key-secret + secret: + secretName: munge-key-secret + defaultMode: 0400 \ No newline at end of file diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-mysql.yml b/slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-mysql.yml new file mode 100644 index 000000000..c111f906b --- /dev/null +++ b/slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-mysql.yml @@ -0,0 +1,70 @@ +# MIT License + +# Copyright (c) 2019 Giovanni Torres + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: slurm + app.kubernetes.io/component: mysql + name: mysql + namespace: ${namespace} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: slurm + app.kubernetes.io/component: mysql + strategy: + type: Recreate + template: + metadata: + labels: + app.kubernetes.io/name: slurm + app.kubernetes.io/component: mysql + spec: + containers: + - env: + - name: MYSQL_DATABASE + value: slurm_acct_db + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + name: database-auth-secret + key: password + - name: MYSQL_RANDOM_ROOT_PASSWORD + value: "yes" + - name: MYSQL_USER + value: "slurm" + image: mariadb:10.10 + name: mysql + ports: + - containerPort: 3306 + volumeMounts: + - mountPath: /var/lib/mysql + name: var-lib-mysql + hostname: mysql + restartPolicy: Always + volumes: + - name: var-lib-mysql + persistentVolumeClaim: + claimName: var-lib-mysql \ No newline at end of file diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-slurmdbd.yml b/slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-slurmdbd.yml new file mode 100644 index 000000000..4a23f7cdf --- /dev/null +++ b/slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-slurmdbd.yml @@ -0,0 +1,95 @@ +# MIT License + +# Copyright (c) 2019 Giovanni Torres + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: slurm + app.kubernetes.io/component: slurmdbd + name: slurmdbd + namespace: ${namespace} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: slurm + app.kubernetes.io/component: slurmdbd + strategy: + type: Recreate + template: + metadata: + labels: + app.kubernetes.io/name: slurm + app.kubernetes.io/component: slurmdbd + spec: + containers: + - args: + - slurmdbd + - -vvv + image: ${cluster_config.image} + name: slurmdbd + ports: + - containerPort: 6819 + volumeMounts: + - mountPath: /etc/slurm/slurm.conf + name: slurm-config-volume + subPath: slurm.conf + - mountPath: /tmp/munge.key + name: munge-key-secret + subPath: munge.key + - mountPath: /tmp/slurmdbd.conf + name: dbd-config-volume + subPath: slurmdbd.conf + env: + - name: StoragePass + valueFrom: + secretKeyRef: + name: database-auth-secret + key: password + dnsPolicy: "None" + dnsConfig: + nameservers: + - "169.254.169.254" + searches: + - slurmd.${namespace}.svc.cluster.local + - slurmd1.${namespace}.svc.cluster.local + - slurmd2.${namespace}.svc.cluster.local + - svc.cluster.local + - cluster.local + - ${namespace}.svc.cluster.local + options: + - name: ndots + value: "5" + hostname: slurmdbd + restartPolicy: Always + volumes: + - name: dbd-config-volume + configMap: + name: slurmdbd-conf-configmap + - name: slurm-config-volume + configMap: + name: slurm-conf-configmap + - name: munge-key-secret + secret: + secretName: munge-key-secret + defaultMode: 0400 \ No newline at end of file diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/03-svc-statefulset-slurmctld.yml b/slurm-on-gke/modules/slurm-cluster/manifest-templates/03-svc-statefulset-slurmctld.yml new file mode 100644 index 000000000..18d69a31f --- /dev/null +++ b/slurm-on-gke/modules/slurm-cluster/manifest-templates/03-svc-statefulset-slurmctld.yml @@ -0,0 +1,103 @@ +# MIT License + +# Copyright (c) 2019 Giovanni Torres + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app.kubernetes.io/name: slurm + app.kubernetes.io/component: slurmctld + name: slurmctld + namespace: ${namespace} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: slurm + app.kubernetes.io/component: slurmctld + template: + metadata: + labels: + app.kubernetes.io/name: slurm + app.kubernetes.io/component: slurmctld + spec: + containers: + - args: + - slurmctld + - -vvv + - -i + - -D + image: ${cluster_config.image} + name: slurmctld + ports: + - containerPort: 6817 + - containerPort: 6820 + - containerPort: 6821 + - containerPort: 6822 + - containerPort: 6823 + - containerPort: 6824 + - containerPort: 6825 + - containerPort: 6826 + - containerPort: 6827 + - containerPort: 6828 + - containerPort: 6829 + - containerPort: 6830 + resources: {} + volumeMounts: + - mountPath: ${cluster_config.storage.mount_path} + name: slurm-jobdir + - mountPath: /etc/slurm/ + name: slurm-config-volume + - mountPath: /tmp/munge.key + name: munge-key-secret + subPath: munge.key + - mountPath: /var/spool/slurmctld + name: slurmctld-state + dnsPolicy: "None" + dnsConfig: + nameservers: + - "169.254.169.254" + searches: + - slurmd.${namespace}.svc.cluster.local + - slurmd1.${namespace}.svc.cluster.local + - slurmd2.${namespace}.svc.cluster.local + - svc.cluster.local + - cluster.local + - ${namespace}.svc.cluster.local + options: + - name: ndots + value: "5" + restartPolicy: Always + volumes: + - name: slurm-jobdir + persistentVolumeClaim: + claimName: slurm-shared-storage + - name: slurmctld-state + persistentVolumeClaim: + claimName: var-spool-slurmctld + - name: slurm-config-volume + configMap: + name: slurm-conf-configmap + - name: munge-key-secret + secret: + secretName: munge-key-secret + defaultMode: 0400 \ No newline at end of file diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-mysql.yml b/slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-mysql.yml new file mode 100644 index 000000000..af26731ff --- /dev/null +++ b/slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-mysql.yml @@ -0,0 +1,38 @@ +# MIT License + +# Copyright (c) 2019 Giovanni Torres + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: slurm + app.kubernetes.io/component: mysql + name: mysql + namespace: ${namespace} +spec: + ports: + - name: mysql + port: 3306 + targetPort: 3306 + selector: + app.kubernetes.io/name: slurm + app.kubernetes.io/component: mysql \ No newline at end of file diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-slurmctld-0.yml b/slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-slurmctld-0.yml new file mode 100644 index 000000000..c9de9bd15 --- /dev/null +++ b/slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-slurmctld-0.yml @@ -0,0 +1,71 @@ +# MIT License + +# Copyright (c) 2019 Giovanni Torres + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: slurm + app.kubernetes.io/component: slurmctld + name: slurmctld-0 + namespace: ${namespace} +spec: + ports: + - name: slurmctld + port: 6817 + targetPort: 6817 + - name: slurmctld-20 + port: 6820 + targetPort: 6820 + - name: slurmctld-21 + port: 6821 + targetPort: 6821 + - name: slurmctld-22 + port: 6822 + targetPort: 6822 + - name: slurmctld-23 + port: 6823 + targetPort: 6823 + - name: slurmctld-24 + port: 6824 + targetPort: 6824 + - name: slurmctld-25 + port: 6825 + targetPort: 6825 + - name: slurmctld-26 + port: 6826 + targetPort: 6826 + - name: slurmctld-27 + port: 6827 + targetPort: 6827 + - name: slurmctld-28 + port: 6828 + targetPort: 6828 + - name: slurmctld-29 + port: 6829 + targetPort: 6829 + - name: slurmctld-30 + port: 6830 + targetPort: 6830 + selector: + app.kubernetes.io/name: slurm + app.kubernetes.io/component: slurmctld \ No newline at end of file diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-slurmdb.yml b/slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-slurmdb.yml new file mode 100644 index 000000000..a305a5595 --- /dev/null +++ b/slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-slurmdb.yml @@ -0,0 +1,38 @@ +# MIT License + +# Copyright (c) 2019 Giovanni Torres + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: slurm + app.kubernetes.io/component: slurmdbd + name: slurmdbd + namespace: ${namespace} +spec: + ports: + - name: slurmdbd + port: 6819 + targetPort: 6819 + selector: + app.kubernetes.io/name: slurm + app.kubernetes.io/component: slurmdbd \ No newline at end of file diff --git a/slurm-on-gke/modules/slurm-cluster/outputs.tf b/slurm-on-gke/modules/slurm-cluster/outputs.tf new file mode 100644 index 000000000..92785d1de --- /dev/null +++ b/slurm-on-gke/modules/slurm-cluster/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "namespace" { + description = "Slurm cluster namespace" + value = var.namespace +} + +output "image" { + description = "Slurm cluster image" + value = var.cluster_config.image +} diff --git a/slurm-on-gke/modules/slurm-cluster/variables.tf b/slurm-on-gke/modules/slurm-cluster/variables.tf new file mode 100644 index 000000000..41ab625a3 --- /dev/null +++ b/slurm-on-gke/modules/slurm-cluster/variables.tf @@ -0,0 +1,59 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "namespace" { + description = "Namespace used for Slurm cluster resources." + type = string + nullable = false + default = "default" +} + +variable "namespace_create" { + description = "Create namespace use for Slurm cluster resources." + type = bool + nullable = false + default = false +} + +variable "cluster_config" { + description = "Configure Slurm cluster statefulset parameters." + type = object({ + name = optional(string, "linux") + image = optional(string, "") + database = object({ + create = optional(bool, true) + host = optional(string, "mysql") + user = optional(string, "slurm") + password = optional(string, "") + storage_size_gb = optional(number, 1) + }) + munge = object({ + key = optional(string, "") + }) + storage = optional(object({ + mount_path = optional(string, "/home") + type = optional(string, "filestore") + size_gb = optional(number, 100) + })) + }) + nullable = false +} + +variable "templates_path" { + description = "Path where manifest templates will be read from. Set to null to use the default manifests" + type = string + default = null +} diff --git a/slurm-on-gke/modules/slurm-nodeset/main.tf b/slurm-on-gke/modules/slurm-nodeset/main.tf new file mode 100644 index 000000000..c49e910f8 --- /dev/null +++ b/slurm-on-gke/modules/slurm-nodeset/main.tf @@ -0,0 +1,58 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +locals { + wl_templates = [ + for f in fileset(local.wl_templates_path, "[0-9]*yml") : + "${local.wl_templates_path}/${f}" + ] + wl_templates_path = ( + var.templates_path == null + ? "${path.module}/manifest-templates" + : pathexpand(var.templates_path) + ) + + config = ( + strcontains(var.config.type, "n1-standard") + ? { + gpu_instances = var.config.accelerator.count + gpu = var.config.accelerator.type + } + : { + gpu_instances = var.accelerator_types[var.config.type].count + gpu = var.accelerator_types[var.config.type].type + } + ) +} + +resource "kubernetes_manifest" "default" { + for_each = toset(local.wl_templates) + manifest = yamldecode(templatefile(each.value, { + name = var.name + namespace = var.config.namespace + image = var.config.image + instances = var.config.instances + config = local.config + })) + + timeouts { + create = "30m" + } + field_manager { + force_conflicts = true + } +} diff --git a/slurm-on-gke/modules/slurm-nodeset/manifest-templates/03-svc-statefulset-slurmd.yml b/slurm-on-gke/modules/slurm-nodeset/manifest-templates/03-svc-statefulset-slurmd.yml new file mode 100644 index 000000000..a4975ccdf --- /dev/null +++ b/slurm-on-gke/modules/slurm-nodeset/manifest-templates/03-svc-statefulset-slurmd.yml @@ -0,0 +1,124 @@ +# MIT License + +# Copyright (c) 2019 Giovanni Torres + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app.kubernetes.io/name: slurm + app.kubernetes.io/component: slurmd + name: ${name} + namespace: ${namespace} +spec: + replicas: ${instances} + selector: + matchLabels: + app.kubernetes.io/name: slurm + app.kubernetes.io/component: slurmd + serviceName: ${name} + template: + metadata: + labels: + app.kubernetes.io/name: slurm + app.kubernetes.io/component: slurmd + spec: + nodeSelector: + cloud.google.com/gke-accelerator: ${config.gpu} + tolerations: + - key: "nvidia.com/gpu" + operator: "Exists" + effect: "NoSchedule" + containers: + - args: + - slurmd + - -D + - -s + - -vvv + - --conf-server="slurmctld-0:6820-6830" + - -Z + - -N + - "$(POD_NAME).${name}" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + image: ${image} + imagePullPolicy: Always + name: slurmd + readinessProbe: + tcpSocket: + port: 6818 + failureThreshold: 3 + initialDelaySeconds: 1 + periodSeconds: 20 + successThreshold: 1 + timeoutSeconds: 1 + ports: + - containerPort: 6818 + hostPort: 6818 + resources: + limits: + nvidia.com/gpu: ${config.gpu_instances} + cpu: 3 + memory: "10Gi" + requests: + nvidia.com/gpu: ${config.gpu_instances} + cpu: 3 + memory: "10Gi" + volumeMounts: + - mountPath: /run/dbus/system_bus_socket + name: system-bus-socket + - mountPath: /tmp/munge.key + name: munge-key-secret + subPath: munge.key + - mountPath: /home + name: slurm-jobdir + securityContext: + privileged: true + hostNetwork: true + dnsPolicy: "None" + dnsConfig: + nameservers: + - "169.254.169.254" + searches: + - slurmd.${namespace}.svc.cluster.local + - slurmd1.${namespace}.svc.cluster.local + - slurmd2.${namespace}.svc.cluster.local + - svc.cluster.local + - cluster.local + - ${namespace}.svc.cluster.local + options: + - name: ndots + value: "5" + restartPolicy: Always + volumes: + - name: system-bus-socket + hostPath: + path: /run/dbus/system_bus_socket + - name: slurm-jobdir + persistentVolumeClaim: + claimName: slurm-shared-storage + - name: munge-key-secret + secret: + secretName: munge-key-secret + defaultMode: 0400 \ No newline at end of file diff --git a/slurm-on-gke/modules/slurm-nodeset/manifest-templates/04-svc-slurmd.yml b/slurm-on-gke/modules/slurm-nodeset/manifest-templates/04-svc-slurmd.yml new file mode 100644 index 000000000..00e93bcd4 --- /dev/null +++ b/slurm-on-gke/modules/slurm-nodeset/manifest-templates/04-svc-slurmd.yml @@ -0,0 +1,39 @@ +# MIT License + +# Copyright (c) 2019 Giovanni Torres + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: slurm + app.kubernetes.io/component: slurmd + name: ${name} + namespace: ${namespace} +spec: + ports: + - name: slurmd + port: 6818 + targetPort: 6818 + selector: + app.kubernetes.io/name: slurm + app.kubernetes.io/component: slurmd + clusterIP: None \ No newline at end of file diff --git a/slurm-on-gke/modules/slurm-nodeset/variables.tf b/slurm-on-gke/modules/slurm-nodeset/variables.tf new file mode 100644 index 000000000..d7ddc1a24 --- /dev/null +++ b/slurm-on-gke/modules/slurm-nodeset/variables.tf @@ -0,0 +1,91 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "name" { + description = "Name used for Slurm worker nodes configuration" + type = string + nullable = false +} + +variable "templates_path" { + description = "Path where manifest templates will be read from. Set to null to use the default manifests." + type = string + default = null +} + +variable "config" { + type = object({ + type = string # Machine Type + instances = number # Number of Slurm instances + namespace = string # Cluster namespace + image = string #"Container image used for Slurm cluster pods" + accelerator = optional(object({ + type = string # "Needed when the configured instances is an N1-standard-* since GPUs are not known in advanced." + count = number + })) + storage = optional(object({ + mount_path = optional(string, "/home") + type = optional(string, "filestore") + })) + }) + default = null +} + +variable "accelerator_types" { + type = map(object({ + type = optional(string) + count = optional(number) + cpu = optional(number) + memory = optional(number) + })) + + default = { + "g2-standard-4" = { + type = "nvidia-l4" + count = 1 + } + "g2-standard-8" = { + type = "nvidia-l4" + count = 1 + } + "g2-standard-12" = { + type = "nvidia-l4" + count = 1 + } + "g2-standard-16" = { + type = "nvidia-l4" + count = 1 + } + "g2-standard-24" = { + type = "nvidia-l4" + count = 2 + } + "g2-standard-32" = { + type = "nvidia-l4" + count = 1 + } + "g2-standard-48" = { + type = "nvidia-l4" + count = 4 + } + "g2-standard-96" = { + type = "nvidia-l4" + count = 8 + } + } + + +} diff --git a/slurm-on-gke/providers.tf b/slurm-on-gke/providers.tf new file mode 100644 index 000000000..5cc9c6b03 --- /dev/null +++ b/slurm-on-gke/providers.tf @@ -0,0 +1,44 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +provider "google" { + impersonate_service_account = var.impersonate_service_account +} + +provider "google-beta" { + impersonate_service_account = var.impersonate_service_account +} + +data "google_client_config" "identity" { + count = var.credentials_config.fleet_host != null ? 1 : 0 +} + +provider "kubernetes" { + config_path = ( + var.credentials_config.kubeconfig == null + ? null + : pathexpand(var.credentials_config.kubeconfig.path) + ) + config_context = try( + var.credentials_config.kubeconfig.context, null + ) + host = ( + var.credentials_config.fleet_host == null + ? null + : var.credentials_config.fleet_host + ) + token = try(data.google_client_config.identity.0.access_token, null) +} diff --git a/slurm-on-gke/variables.tf b/slurm-on-gke/variables.tf new file mode 100644 index 000000000..4e92ba97b --- /dev/null +++ b/slurm-on-gke/variables.tf @@ -0,0 +1,87 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +variable "cluster_name" { + description = "Cluster Name to use" + type = string + nullable = false +} + +variable "project_id" { + description = "Google Cloud Project ID" + type = string + nullable = false +} + +variable "region" { + description = "Google Cloud Region where the GKE cluster is located" + type = string + nullable = false +} + +variable "impersonate_service_account" { + description = "Service account to be used while using Google Cloud APIs" + type = string + nullable = true +} + +variable "config" { + description = "Configure Slurm cluster statefulset parameters." + type = object({ + name = optional(string, "linux") + image = optional(string, "") + database = object({ + create = optional(bool, true) + host = optional(string, "mysql") + user = optional(string, "slurm") + password = optional(string, "") + storage_size_gb = optional(number, 1) + }) + munge = object({ + key = optional(string, "") + }) + storage = object({ + type = optional(string, "filestore") + size_gb = optional(number, 100) + }) + }) + nullable = false +} + +variable "credentials_config" { + description = "Configure how Terraform authenticates to the cluster." + type = object({ + fleet_host = optional(string) + kubeconfig = optional(object({ + context = optional(string) + path = optional(string, "~/.kube/config") + })) + }) + nullable = false + validation { + condition = ( + (var.credentials_config.fleet_host != null) != + (var.credentials_config.kubeconfig != null) + ) + error_message = "Exactly one of fleet host or kubeconfig must be set." + } + default = { + kubeconfig = { + path = "~/.kube/config" + } + } +} From e9044b247a8b63621d8af337e5b31a8eea3430f5 Mon Sep 17 00:00:00 2001 From: Daniel Marzini <44803752+danielmarzini@users.noreply.github.com> Date: Wed, 30 Oct 2024 08:07:25 +0100 Subject: [PATCH 02/13] Update README.md --- slurm-on-gke/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slurm-on-gke/README.md b/slurm-on-gke/README.md index b127cd878..c797e4592 100644 --- a/slurm-on-gke/README.md +++ b/slurm-on-gke/README.md @@ -443,6 +443,6 @@ This step deletes all the resources that you created previously: the GKE cluster * The use of the assets contained in this repository is subject to compliance with [Google's AI Principles](https://ai.google/responsibility/principles/) * See [LICENSE](https://github.com/GoogleCloudPlatform/ai-on-gke/blob/main/LICENSE) -* This is inspired by the [Stack HPC \- slurm k8s cluster project](https://github.com/stackhpc/slurm-k8s-cluster), copied or derived files from the original repository are under the original MIT license and/or [Giovanni Torres copyright](https://github.com/stackhpc/slurm-k8s-cluster/blob/main/LICENSE). +* This project is adapted from the [Stack HPC \- slurm k8s cluster project](https://github.com/stackhpc/slurm-k8s-cluster) by Giovanni Torres. Copied or derived files from the original repository, as specified within their headers, are licensed under the original MIT license and copyrighted by [Giovanni Torres](https://github.com/stackhpc/slurm-k8s-cluster/blob/main/LICENSE). [image1]: From b71baa6d7c147f6611ace228a0e7854a0ef385c6 Mon Sep 17 00:00:00 2001 From: Daniel Marzini <44803752+danielmarzini@users.noreply.github.com> Date: Wed, 30 Oct 2024 22:14:52 +0100 Subject: [PATCH 03/13] remove empty line --- slurm-on-gke/.gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/slurm-on-gke/.gitignore b/slurm-on-gke/.gitignore index cbfd8f8f8..4b4e41cc8 100644 --- a/slurm-on-gke/.gitignore +++ b/slurm-on-gke/.gitignore @@ -1,6 +1,5 @@ - .terraform* terraform.tfstate terraform.tfstate.backup terraform.tfvars -*/.terraform \ No newline at end of file +*/.terraform From 8ccce2521d13c947004b0c9910cc5cdc15c0bc5e Mon Sep 17 00:00:00 2001 From: Daniel Marzini <44803752+danielmarzini@users.noreply.github.com> Date: Wed, 30 Oct 2024 22:15:41 +0100 Subject: [PATCH 04/13] add new lines at the end of each file --- .../manifest-templates/00-configmap-slurm-config.yml | 3 ++- .../manifest-templates/00-configmap-slurmdb-config.yml | 1 - .../manifest-templates/00-secret-database-auth.yml | 2 +- .../slurm-cluster/manifest-templates/00-secret-munge-key.yml | 2 +- .../manifest-templates/01-pvc-slurm-shared-storage.yml | 2 +- .../slurm-cluster/manifest-templates/01-pvc-var-lib-mysql.yml | 2 +- .../manifest-templates/01-pvc-var-spool-slurmctld.yml | 2 +- .../manifest-templates/02-svc-deployment-login.yml | 2 +- .../manifest-templates/02-svc-deployment-mysql.yml | 2 +- .../manifest-templates/02-svc-deployment-slurmdbd.yml | 2 +- .../manifest-templates/03-svc-statefulset-slurmctld.yml | 2 +- .../modules/slurm-cluster/manifest-templates/04-svc-mysql.yml | 2 +- .../slurm-cluster/manifest-templates/04-svc-slurmctld-0.yml | 2 +- .../slurm-cluster/manifest-templates/04-svc-slurmdb.yml | 2 +- slurm-on-gke/modules/slurm-nodeset/main.tf | 1 + .../manifest-templates/03-svc-statefulset-slurmd.yml | 2 +- .../modules/slurm-nodeset/manifest-templates/04-svc-slurmd.yml | 2 +- slurm-on-gke/modules/slurm-nodeset/variables.tf | 3 +-- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-configmap-slurm-config.yml b/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-configmap-slurm-config.yml index e7228ff71..c27cde51c 100644 --- a/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-configmap-slurm-config.yml +++ b/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-configmap-slurm-config.yml @@ -119,4 +119,5 @@ data: ConstrainDevices=yes ConstrainRAMSpace=yes ConstrainSwapSpace=yes - IgnoreSystemd=yes \ No newline at end of file + IgnoreSystemd=yes + \ No newline at end of file diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-configmap-slurmdb-config.yml b/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-configmap-slurmdb-config.yml index e82c28285..9460060e7 100644 --- a/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-configmap-slurmdb-config.yml +++ b/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-configmap-slurmdb-config.yml @@ -47,4 +47,3 @@ data: StorageType=accounting_storage/mysql StorageHost=${cluster_config.database.host} StorageUser=${cluster_config.database.user} - diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-secret-database-auth.yml b/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-secret-database-auth.yml index 3033d3912..2379b803e 100644 --- a/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-secret-database-auth.yml +++ b/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-secret-database-auth.yml @@ -26,4 +26,4 @@ metadata: name: database-auth-secret namespace: ${namespace} data: - password: ${cluster_config.database.password} \ No newline at end of file + password: ${cluster_config.database.password} diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-secret-munge-key.yml b/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-secret-munge-key.yml index 5d20f540e..dbea49a61 100644 --- a/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-secret-munge-key.yml +++ b/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-secret-munge-key.yml @@ -26,4 +26,4 @@ metadata: name: munge-key-secret namespace: ${namespace} data: - munge.key: ${base64encode(cluster_config.munge.key)} \ No newline at end of file + munge.key: ${base64encode(cluster_config.munge.key)} diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-slurm-shared-storage.yml b/slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-slurm-shared-storage.yml index 992418286..e77ed2bf1 100644 --- a/slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-slurm-shared-storage.yml +++ b/slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-slurm-shared-storage.yml @@ -31,4 +31,4 @@ spec: - ReadWriteMany resources: requests: - storage: ${cluster_config.storage.size_gb}Gi \ No newline at end of file + storage: ${cluster_config.storage.size_gb}Gi diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-var-lib-mysql.yml b/slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-var-lib-mysql.yml index f6a86768b..432a6cc71 100644 --- a/slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-var-lib-mysql.yml +++ b/slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-var-lib-mysql.yml @@ -33,4 +33,4 @@ spec: - ReadWriteOnce resources: requests: - storage: ${cluster_config.database.storage_size_gb}Gi \ No newline at end of file + storage: ${cluster_config.database.storage_size_gb}Gi diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-var-spool-slurmctld.yml b/slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-var-spool-slurmctld.yml index 2e6eef12e..71d099fbb 100644 --- a/slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-var-spool-slurmctld.yml +++ b/slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-var-spool-slurmctld.yml @@ -33,4 +33,4 @@ spec: - ReadWriteOnce resources: requests: - storage: 100Mi \ No newline at end of file + storage: 100Mi diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-login.yml b/slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-login.yml index a1a6e75cc..f16d5b8dd 100644 --- a/slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-login.yml +++ b/slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-login.yml @@ -83,4 +83,4 @@ spec: - name: munge-key-secret secret: secretName: munge-key-secret - defaultMode: 0400 \ No newline at end of file + defaultMode: 0400 diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-mysql.yml b/slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-mysql.yml index c111f906b..aec6db619 100644 --- a/slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-mysql.yml +++ b/slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-mysql.yml @@ -67,4 +67,4 @@ spec: volumes: - name: var-lib-mysql persistentVolumeClaim: - claimName: var-lib-mysql \ No newline at end of file + claimName: var-lib-mysql diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-slurmdbd.yml b/slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-slurmdbd.yml index 4a23f7cdf..8c470c468 100644 --- a/slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-slurmdbd.yml +++ b/slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-slurmdbd.yml @@ -92,4 +92,4 @@ spec: - name: munge-key-secret secret: secretName: munge-key-secret - defaultMode: 0400 \ No newline at end of file + defaultMode: 0400 diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/03-svc-statefulset-slurmctld.yml b/slurm-on-gke/modules/slurm-cluster/manifest-templates/03-svc-statefulset-slurmctld.yml index 18d69a31f..d3ca95397 100644 --- a/slurm-on-gke/modules/slurm-cluster/manifest-templates/03-svc-statefulset-slurmctld.yml +++ b/slurm-on-gke/modules/slurm-cluster/manifest-templates/03-svc-statefulset-slurmctld.yml @@ -100,4 +100,4 @@ spec: - name: munge-key-secret secret: secretName: munge-key-secret - defaultMode: 0400 \ No newline at end of file + defaultMode: 0400 diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-mysql.yml b/slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-mysql.yml index af26731ff..8b1816cbf 100644 --- a/slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-mysql.yml +++ b/slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-mysql.yml @@ -35,4 +35,4 @@ spec: targetPort: 3306 selector: app.kubernetes.io/name: slurm - app.kubernetes.io/component: mysql \ No newline at end of file + app.kubernetes.io/component: mysql diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-slurmctld-0.yml b/slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-slurmctld-0.yml index c9de9bd15..25222b256 100644 --- a/slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-slurmctld-0.yml +++ b/slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-slurmctld-0.yml @@ -68,4 +68,4 @@ spec: targetPort: 6830 selector: app.kubernetes.io/name: slurm - app.kubernetes.io/component: slurmctld \ No newline at end of file + app.kubernetes.io/component: slurmctld diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-slurmdb.yml b/slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-slurmdb.yml index a305a5595..0ebb34cef 100644 --- a/slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-slurmdb.yml +++ b/slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-slurmdb.yml @@ -35,4 +35,4 @@ spec: targetPort: 6819 selector: app.kubernetes.io/name: slurm - app.kubernetes.io/component: slurmdbd \ No newline at end of file + app.kubernetes.io/component: slurmdbd diff --git a/slurm-on-gke/modules/slurm-nodeset/main.tf b/slurm-on-gke/modules/slurm-nodeset/main.tf index c49e910f8..6235dacea 100644 --- a/slurm-on-gke/modules/slurm-nodeset/main.tf +++ b/slurm-on-gke/modules/slurm-nodeset/main.tf @@ -56,3 +56,4 @@ resource "kubernetes_manifest" "default" { force_conflicts = true } } + diff --git a/slurm-on-gke/modules/slurm-nodeset/manifest-templates/03-svc-statefulset-slurmd.yml b/slurm-on-gke/modules/slurm-nodeset/manifest-templates/03-svc-statefulset-slurmd.yml index a4975ccdf..09c8fb83f 100644 --- a/slurm-on-gke/modules/slurm-nodeset/manifest-templates/03-svc-statefulset-slurmd.yml +++ b/slurm-on-gke/modules/slurm-nodeset/manifest-templates/03-svc-statefulset-slurmd.yml @@ -121,4 +121,4 @@ spec: - name: munge-key-secret secret: secretName: munge-key-secret - defaultMode: 0400 \ No newline at end of file + defaultMode: 0400 diff --git a/slurm-on-gke/modules/slurm-nodeset/manifest-templates/04-svc-slurmd.yml b/slurm-on-gke/modules/slurm-nodeset/manifest-templates/04-svc-slurmd.yml index 00e93bcd4..5abd1bdec 100644 --- a/slurm-on-gke/modules/slurm-nodeset/manifest-templates/04-svc-slurmd.yml +++ b/slurm-on-gke/modules/slurm-nodeset/manifest-templates/04-svc-slurmd.yml @@ -36,4 +36,4 @@ spec: selector: app.kubernetes.io/name: slurm app.kubernetes.io/component: slurmd - clusterIP: None \ No newline at end of file + clusterIP: None diff --git a/slurm-on-gke/modules/slurm-nodeset/variables.tf b/slurm-on-gke/modules/slurm-nodeset/variables.tf index d7ddc1a24..bf51e7a9e 100644 --- a/slurm-on-gke/modules/slurm-nodeset/variables.tf +++ b/slurm-on-gke/modules/slurm-nodeset/variables.tf @@ -86,6 +86,5 @@ variable "accelerator_types" { count = 8 } } - - } + From ffdc0ba7a7c87565d75dcd497acebd65b2c5fffe Mon Sep 17 00:00:00 2001 From: Daniel Marzini <44803752+danielmarzini@users.noreply.github.com> Date: Wed, 30 Oct 2024 22:21:50 +0100 Subject: [PATCH 05/13] remove embedded image --- slurm-on-gke/README.md | 4 +--- slurm-on-gke/solution-architecture.png | Bin 0 -> 40937 bytes 2 files changed, 1 insertion(+), 3 deletions(-) create mode 100644 slurm-on-gke/solution-architecture.png diff --git a/slurm-on-gke/README.md b/slurm-on-gke/README.md index c797e4592..648fe02fd 100644 --- a/slurm-on-gke/README.md +++ b/slurm-on-gke/README.md @@ -35,7 +35,7 @@ If you are searching for a guide or an automation tool that can help you set up The implemented architecture is composed of multiple StatefulSets and Deployments in order to cover the different Slurm components. The following diagram shows a Slurm control plane and a data plane that are running in GKE. -![Slurm on GKE - Architecture][image1] +![Slurm on GKE - Architecture](solution-architecture.png) Although the function of each component won’t be described in detail, it’s important to highlight how each component is configured and how it can be customized. The previous diagram identifies two main components: the Slurm control plane and the data plane. @@ -444,5 +444,3 @@ This step deletes all the resources that you created previously: the GKE cluster * The use of the assets contained in this repository is subject to compliance with [Google's AI Principles](https://ai.google/responsibility/principles/) * See [LICENSE](https://github.com/GoogleCloudPlatform/ai-on-gke/blob/main/LICENSE) * This project is adapted from the [Stack HPC \- slurm k8s cluster project](https://github.com/stackhpc/slurm-k8s-cluster) by Giovanni Torres. Copied or derived files from the original repository, as specified within their headers, are licensed under the original MIT license and copyrighted by [Giovanni Torres](https://github.com/stackhpc/slurm-k8s-cluster/blob/main/LICENSE). - -[image1]: diff --git a/slurm-on-gke/solution-architecture.png b/slurm-on-gke/solution-architecture.png new file mode 100644 index 0000000000000000000000000000000000000000..54c806901e9eb6f1b61e5e710f14ff76b5f57741 GIT binary patch literal 40937 zcmeFZby!tf)ILf{iliVQAt|kNhoE#wN^Fqs?k*LOu1$wZO4p`KBsX1~29-^B=jOLO z=jic#_s{#>d;hqf&jT#iT63;B#~gFaF~-bO(?oZ!jj zvJcTjNBbj3Cn&GNm6e9GnU9aMm5Tt@v6$E57CpWQa@zf2pK>pQYUL~HAe$p1;S^&< za(NySK-5}hujI^?l#p0}YfL2MP%9)f;0hV|AqRedQHg(tgaP~|1b(D*P<}l{1?8ar zx<;V^o*_x9OUcOrf7MN#&CKjwEFD~zGqSsZs-~jJwZBAO6zg}^*5v6^lq)H>@;A}?2$MJ;Y z39T454GoQmv#Gg|n)Hi*iUa=>rL}Z*brj;{^ziWD@ZjcfaJJxlDkvz(`Gkv;i;Eq2 zgWbi;-qpyH-QI=n?@E5vBW>nl;%w#UYUN;0^Rr$fV+S`^QCixcANu|8?{=DbTK)Sa zdzXK@1$2<}=M&DS98WlZ*9{aE`FU4J)ymV%R!7>(4iFFU4KZFG9+AJ^|Gzx>_ltj& zeD!Zh0lueymi*(%|17ELV&*L6UWJkh2)J z2})HdT3V*h);-S_1kN&s{d|1d ze7@fSJ&}{eqzObq`S%wULGN;$ejhP18omV5?_cfM_^t=f{830viZ-QdSIYTj=F(9AfaYk`*}4~VJW~iU5A{AEqh(=~9%WFh9mbHt;x<{X z75w$agot#N+3MMZM;ShYPV5H+5o3T9>b}aB9v%J01 z?MXDutOWab$Sp==4LXI9S9Ja2*ow@*$4C6Wtoul9=YV4UYpI&!<|K}{{@#Pkd?*~q znW`pm$b%WDFftj?W47PwP{bCyonv#ed*uE-QiB6y=22i^GMX!G2JY%v+MC}>xJ(vg ziUATbF$(wX6C2>pW70;$ueV0N@iVAhP_&bM_FLWd!z8Lc6-4^6-{zblpvyo$2Xw*qSQk z8c8v|H^}xw%#(kJFYEVc{+x?@Sh-#!C$OdJT!-6&Qt5Fs(o^de?w3Lb}cQ3L&UakK`So&74SEC5i=Ui{EnbpwmB+AkzZR%TsN)Pmd zR49DZ9<;*fa=lC%Dtl%R+BqEk)RU_hlLdKRX8abL%jo->`~0l1Llh(^11uRcoWxr2 z#(aoP<{ouiTzM;Fd7ID}y3a1$&9Ka{yv*;jB9YcCGOZ7U=7{NvI%D*S@0?-68Q;Q z4+Wj2Ys@}Zl3A>x={MeFHO=)4@3oXiIE@vl7ub@1h%GwH9N}^O;(Kmu<+=Iic@*d zhU;nCdlF`2>db)m_gSqAyqB6hvO!5q9=if?JRF>a_4Nl$N}DUU&O!u!W}%rWDKHXYCy`W7VH? zmF#ibnW^R<67u_af`f}2SGzS8zPgn=X|9HAw=lM>z%*U2kNuS0#K4>@(cLb>+f&ZvTF^kMRq@_@ED-p8q&CBiK1u@hgAO-5Xh*WV{>(j7T=csi$KGyi0! zRTOeZ$wKs@eZIHD!e+DpyY0|Z5!ye@j>Ix{u9ZSE7noCfNLl&dGg&o{#jXf!VolI_ zC3bN&;_aSYz?oU?7Ob4fqt1raP#Gbg<%hU)ahx#ir0PAg=$F5hh0s@wGxyb0Q}gt! zPkrzArkTgKex2>@)BJS9i$gd(k2Je3NHd(U3+~2^+}#T^NIqM<9YK%66Q6z=O)dV2 z+_X#O*5_m0`gChHqo?$LM*};ltl6I$%st^l5r)R`g|+DSiAr z<*i?!DXgtN>Jwz^@N;|bVTaofW-neD&KB?R8}o)!ar-MU&5o1>?ryxX`qpnhMJ?_> zZ!c`!NV@3l!+PQS#WOq%(_?9}w&Y~|z_=@dRON|)%T6x}iUIM1GoM7Z1wwWc>b~z? zYL3>wTnHM+J7YCIM!qlX7~Y3ro}~sVKnu9hy7B5=-(TqI)(~gHRrOYsGXx&}7&!CZ zc=PbvH-?i6w1g=Yy-`AIT!nokt^Y{B9au)so&^w36n*y*nN>nhanf&afyaO9iI&R} zX)xPf=O=mygU7h#4*X|UkRb<`(Kjvz82WG%yil02WZWCsyJgY&h=#09xHFCWue^ee z*(64iB8#(mZM2GBw&-$2zy6OmB)f-VKslbYcu*w%r>B9|iX^GMY-nN~_S-oFbjIMP zKTj)G@TcD&)J{sXj(u&Ti2K_+1Vq&yfr17y__{XnXRGngQ3edwKfye^A%az|OV%@f z&TZR*4JxNEs1oTcqAx71n88apro=^v$|YDZ_FwLXrw)zEYK9uUo+NWxOY8;nWBzxB zOi72}1Lqi%Exb;5O4)@Qm~Ewo_4DTk{GolzH+f2~6=G7MJm1 zM*phnx7welwU2T$nci-6R_#p?=+yfZ`PO`AWzH^0oHuAp)i-QzO&rjffW24r!j~li zM9pK=dMNke_9#43@2QS8Ng++C0za6s%UVIUTTxpqR`j=_rM@p<>b=1>+w{VJ;jo|? zvM%NI*?@FI^+K7NoxST>y9*$etZF6aARjJRz$?F53e)CT6QkQ9P>=B`&jN zXL^;GmVbt!-4IPznkRlg{oT33C^Hj^HhcT_TWzJ3!@Krb*E#y? zh4IIq7r}w7@W_YZ|BSakttzw1o92K{Zs+}pM`S{K?BdM4T|dJJ%*L|CC(i|sua3<6 z@QLX?lB~cVe+Xj!Qn`KYpr8|~_`a?De(f#)Qy7b?Jz6{ipo45>6Ej|8Y0U26FEYgt zxy-wLcYc?lL#9+akRJ4b=XsgH*9N(73Rj5l2PI+w-5g-=lG*UEi_#O+*)4XANbJ*O z>$-5H`K7GO!zd@%nqbCNR;L&221m>{5Oq*EB@dF=)!WC~K`v`u^&?Ly$s9|8b?8aXo^?h_Z z9B35BUczTRkL-Di;IjFmG^W(weMfM_^1IxdaXAcZ(+)YLi!B8D{gJ1r&FHM}1`Sj+ zEFb;Pyj@;Hx%1_)w=BWurkRA8I_T@quGYz&urqz4Rmz4B6lHOeAAl3d`?NW}bank)G&d-R-%Dd0vbpncQX|*uO&%bpN&_u9B|Di{63j`7a6K%Vq;J zW8E#(5}4_w zS+qGhFL1h*qxKRTTrV(n+Po9C=c*0+wH`AtD3HduTpxU5GM)LZNk~LVr>W%kJ=;(v z*Q?a9}cFC&~axiUr>c_>?)Q;*LsnVq6ep--)g{-dp&c1xyThD<56s;d~j-$AEV zVoFe?_9?i!@1$@pI+g2rBJ($0kKp~S-ILF9?;_`q96SCsfoK)Lh-8E2t<=r? z4ae8yqZ&0$XFh6{jny@-X<02Gv;_S&BO}RAv#=BqZg$XzBrJB+mdt~CJS3FW$NkH7 zimGb!=y-8VDl<1c`y&#Ts}9vWM>#p=d3K@TvF}3^+b+1vFYc1FlM4tGN&XNTgNNVa zbG$VFu5vgI8_!VU%UM5N2~jJ5spQnJ-|D;}0RODnmUg@i>qC1)X8LWL;qcpfxbyD3 zpf;6mcPr;{V#!QZ7%*Hp zF4tr3D)S5Pi;E^Q=MFTVd{4D$G($61a8;`t`9$oe(3lx#mCJQKuRz~ zSET}OY;01sia}iF=NN}wOiR!ceqQHE_qkQdvGaH6YNIv1mhCkzbM`YPx^cS!xuqE~ z;c>&_2xarz)^>D|#jtc=amW&+V^TfDNA@88IlRCjA9|dPsOgX=N+{>B*&2^ec}%N7Z_k!vmWA(PoO&h3&|D!(NOPqH){_oySvZtPhN!{N zyl4A)><6UDZWm2A7S|q~8Bc3T*nJ@9_OHik=1K)O(dEB&sK$-`zMb-hNoeEURMAB# zY^J5wO;J5sf5uLt2)fsL8t%*E9)G-kRWV`rN?p589fSexXRyibIC0x?LHst1_?Y^C z2^Uhj+xooB#y^Vv=DQbEv98rFpLOohT;5(jSjlu~;4R88fFZ@Y%F zQs^w)y$WS~C}el$TqIg=zmqmVbo?5=LAv>dbpqHbf~Na^#8sUv?KodgW6jalF1m6~ zP}?z#)tGmoTX`SVLMT3gza@CI)^v>fW$wjSusK;++urVH@{NrDsM#{S-R10;A9v1` z)D92(=&bZ(mTT+G%efoPYo^L#bivl+nV-IWBZ@V2iYTNp91I%g&CgTP>!^WSUx*j) zG@9f?mm;$fRZu}^b8fhvL-N8cGnK)_>dENzuu|rh+=aYZF2b#L!9(uQ`4#wyl{>UH zc{|`H3Kz7+qQz>lRa9!IJoeh7b9N97bPw})5G8>GI13jWUF^D56~!wvE??0ft$iGt z;jjz2YNln1&Q5gXHkThCva3|jk;#rAz={x6)ccTe@Jgq;%iU>w*9oZeh5Eq7gPCfZ zY|zwGbymf#$OPAUjqb=wlP)TXk>{;ErB+aqQXb?nmW#`3=0nF}pPdIx7PrO>?*$_G z1SX!UUY$|ZY_;BATU;0&mhoVb{ATfq6J-0I6xbTqkU4ahGdnV zQmZkrkR7tC+TwKsyg@swYJsYjSFs6w(aQlxTVj1XdHK`sHrRP}XxG_Q&#rr*%YX<( zN=R<3hfL{Uk!iq5_{%v6lvw_&^St-411B^tfm2QX(3s1o}}ikNQ3Ee7?WMH4{qlg;zDcY!ZKBL^&=a>j?*nHf`*xVo;i@7;>| z3iC4B<&VZVB4({~*ZC8457T?K@N_Rl9v&-bEP8~Hob1rNpk4c_zwHkSiZFC*hr3>|=$SD& z7T1qBnS%RoVOMiyVD5YbI~{s7c+-t#PUkLf%GeL8U8iy%t<#4~%4sY&p~sM(wZ7yz z#dl7RhUlrhB_`AE!%?{pdareEdwKJ(KA`(@2c*+A9IZwsP^NZ6VP!%bZUpy{==CQ~3&6L$3>qq+m^O z9PW#54epM2`OltX%l63ZRYHfv>uRjc8Cz4#^y+JDa-y%Cy9GDUs6Gw8p-$-bvuR@E zd(x2-&Fd1cEa8;Drb7`qcG1*MRf5b|%iZs2h znT@g2-EXGOF832BA~C^3gSQlP2_ya`O&U3uJ?y5cH$!U7Dqr&`)V-pVeVx8f_bIe0 zY+0*(Vh0HwWh)PTb8T=r6rpx8INcF_-CE8f?K+>BAm(+2GuJqA_SQgcNU^AON%8z; zy=4$u4D7%}(51^iG{wdDT86SSX~DPW)is!BL8T{iBXOHtLU2imc3 zl+qCVYRuaTCv$3stP?!lS|ane*A}#b<6FIfH{Ai& zZoBetERobZ*sllC{Mh?6T&sWid<1}RelM(Qw6XiPcbWUbP5!<2*A)jrPTm$Fp;j)W zy(M#2cJ3ZK=fz^#gc#n%Z>JZ4a(+got)7}FC5p2Qqc=WPC?(z+7{5Gd38Ww>_DIWq zm5A*ha|`HenQ~yx*5B=9 zir?P)UmdC!LN@1Y77~wsu}m05O}dgq7uVsu|7x9K?p4q~VOTQuLqVV9j3_~-7WeE% zS2`Pf*^d&I`83RFOd%S~S(5gj(jAREPe5JO|nEmHw7wvMeC1?;>#V=GGY?YAnjRnYk51x zacW!v=vMlD{9h>7#*JP>)rHYS*c?|(E3WIG#E4A4F=WP5PZ0ej8%Ld>FAdzXXX#RA zUzT1bFL1K+A7VXauNBm8$F^Ne*KxLJdA(bRW_!Qr!LK6A{3uGrXwXf{r_MZM{N7Xq zt`~y}p~06`?YmMR)OgAruGOxph|HvZ1r}q^B$y^fYRqmgY6KSU4@-`?J*Rd5txkWb zjv5CLojMCg|BpcjFwXxk-^IiaI$9fsHo9$@0YHqH_NM@T2Z-5GL$Fu9L?|9Ll;SB=OMxVsb!YlN7Mp!9;7pdh{X*@PG0Pv8xKAWEC%qq(R zV=N%)`vRBZtKg!S^GYv4YTN4%`t>ww1@21BsyTrRxg8x5x)MUi+qnR}@hYliW{ z+8jmNb?vuruNdQpQ1ljtMuXLMs$d+_{aili~ zXVpschQ}Vl`@^ri?oo;vpRrOBJmR&Pij9rMPaj6yT$yb_c77sx%=(3~pgJ3v{Y5==G+Hpd^L9-3%*JGp!@GI*4UN;zDyiWC6 zz4bz#3)9A-EYp=fY9&<*RWcEiay>-5Z=z&C1Y}M{lJ`PpO3#-p}DfwuJymadtrLuOHv;p9ZBkQ6cFXHw>=4V2;A>ZHc!lI8*&;mtDN!|Zi zZqdso5~M{`JL)Vb7xJ~PC$h?I)a_GGD+3Xu|E?_<$es76)!O8B@Sd1WP?U$eWC73) z^bpRf51*1a2sv8>i+COOZVS4YeeKnTR~m@scHW%LG}oNSFBp0vbQ(^;>7a`yPD?)k zLo!F1*&aI?&Du@B_7=3S--Bs>*#9?ixba*b_8 z*!TOPYeYTJI>P|wTARtlWJ51aK8MBc#*6h%36{l?Su!CJ08%NRU2qv5O`ssJ15V8` zo8C`mNVkgHT&jz?OL-WK>(5h>@x&>9OJpb*Qo#;_unomP>>HR$mqIA1V%M@M9|MVR zUgT-DnL3BI1$e6D@JyhBUR%w8wd?h{)5gq|liq3@BFhYw!!DCt%rz|X;Zru{{Y`(W z7IeIMkMhBS12Hd!K-CwiOHg_@`{Ar6P%xWyR5~K|P z#}p}+u&yl%LnaJ7$|~d>cfScFMbtEC(sR8n9u?er`^Hx!JG&k}#e9bVK%Cc!dNy|? zaQpD-!bPLo9$ch0X14Zl5N7CBI*wFmlRR{K0WQ@uyS5-wS|F7>0$5Y5lZ04!cy-80 z=*7CLo%uLW^4su_#RW#=IID4RgXI&@1?TGaDxd@e z-%elNjt0v1_-#Dkz#VYknKi515TIw;a6fx$s8?rSo$hKoe(>YgO?~32(#w}Ghm07P z2%Vx``t#NV+%{_#IB99;#Wuwcpj9{DQ}Wd8%|@L}bd++c{d)3#q*kTdd&~;0Wnq)B zts^F)Q=PR|hLYwuvP3D&P#@Y2j&n76=&{`=us?g*go<%*8h(A}a8Bl?$8V-~j7`98 z(yj7@#i&ttmw<9ONmXcTHGhtJXSNF>D|FSpx42P)_eLhTOrrk%1oAHzMb-jk7dhT% zvRI?cG=wWPRUS6ckq5svid5yX{5j_V!dHP&NC zY)?yH?{2K7+q(vkk3i8t$bMc$4}YzFv8`yXU+!j}OZM=(ZA?kAq%B(vH3RF+`b)26 zgpVEmR88Lrm;V>qMX8XH}=IFgi>&(BI`+_zy1JF~M|HH~i8bE$U=D742TUY_nQSeU2l zC#dpm7*i5iM76 z4>;~9@r&U)`twv1LmEHjnfAqZz0Ku5XfZQCX4NVw9Q;v@MP!%5yPZe+LvgLFnwW%H z?Nw{WbyDV+R^7?ZT$h6|+mXb?Dq?osuR{+D+3jb)9Cs#C@7#?D$)=ze;$y8H|Iy;p z=XVCFW3;Ybd|(wZ3joMrUzKw3$Z1BSNS#Obv;eF?M%9YfzRV_tJm#?47Gn- z0e~EIr9ur>AAPCDVA!A`8Wepn_VfjV{8vg3xIba06DONXN+x{RujQ)BJyY0&Lzjof zDIT4ouGZGB;9bD&Lfy`G;G++ZPMaDt{N||>)JS&F$;iC=(VcdCyp465t~NZmGBLm2 z2sa<=&)!#0FUTlM6H560I*BwSs>d7KcGpJCiib6l8E|Qd8cN=ISDI9Io`&IB64rOteu|KB#fJL#Znh2 zCoL~8qqRxsGOdgRvQ*}}%N;0epG)YdoPCjKh@U|HWl@0esF=7Apm$`VBL+6cPn!nB zxR>tdm-=4b7zqhtGMk~)0BB9rr+?kau}TSp@nG&7Sdsf`tAxo2r{sV0Q-zdWv{GylxVhooiJeAMBOsLZ1{ShMq`49pi+>(C%D(aso>Th2n_Z@(8SU--(_+>8s zf&*j2fV;F$O8&$&kk4QZWL+Xiv% zkSe;@cZ9?yBtL7FvrO7(G+D1=ACYL9gUEAx|GLT&}jeP6C$nBKEyRQ4r0=^WR z8E*C{uYdngU_u*B$YYo7QtFumfUbB_(%=W8T*~Mztv88`F(BE~S;jKqeu;U{l(8Y$HmWWw$xGEjxpEYguPc0bTWiH^GU z*c$e@kXx-J z>^+<{S9zz=S2hN3J9Ljg#g5TYGN@v&Gv?2h%2(iobclV6Mj;HWB97UnydWX3q3!N1Iwe- zXDh`Ht+i{n&WEtELL*cl0zYMdr;7dX5Ck#@6J?s*Ly&Wonr-hv@3T!HMAk@thr0<~ z9n@PnOAU|*VxRAOLr0z{L5IR0IxZ#*{gdvqYR$kF+4+|qC$Va}PM#)uu}5b=kxS#Z zVKE!rYmcTB-a(wz*s%7*P}h}j2-Vv39g_vf{@5x@YPmX<&zf%-w6SU0o@vr1H=L7o zhW9g%=ihR8Dd9VAmGUbDMI@B2@Ab}m;S7YXMt+nm@p&H}9v34<#(8zRlgy&Y(q~t< zNK2-eEXckWBYwkH3@b~beT!2tmoC1mb&$N*(3GiBuBFKn+j6iMg|OcSJ%4hM$x_YWbkSLaT3Y z<5YYhOel=Axm3N{>wDxOovmG>Y5=iaV9cc2wA8yE&Wx~wUv{H`y^dCs0KZ}e2I{ma zd%70~6VUuY5X1wRX?YiC&mi$UE>YIvOla4O~Vu-{=zgww3FmE~!5Aslk;E`$Zg-JQJR)*{P$Ri4g z+P0?Bly!N?-n0=a8*U@SZpmIeY%{bE5UG6RMMa_HOch}H{ab{pY-h0NqF4Y@GeAzx z^%aN9^x27(7n;|$6+3{YO(SEPh z?{Jpl6TL^7KA+{K)22p23gjxv$40`^G7J~&hH*nvD%a!kT>8zN(58#p7~1uX$1?{^ z->U^Y=NK7tyhO9D5&_c1cAE$Ezp+}$Gy>=Q;_#9!d zB>x+bx%qMC6Tk`~ zZ6aRIP>&^*N@OV?B1K=m0~ePB3ouW8e(Uy9fW%ap9@9_arg$Pe0tLcaWra2ITDs@a z>GsW=5?;~!$1RtM4SYYuE~OXAgmnSJ868x+gl9oGyP`C`(Qk9OO;$H*U&M1i){E13 zp&4eL1Y{giC45_3%0jZpUWc&n&=!>!R8$O{UT$4wqk}G~n!YwG3(W_VZ@Cq8nAdm> z`k#;Xs5NT5A!Z@wPg&~kYfpc5MK%Ik>Jnx{6(Lxb2Cw|E)>>()tP) z^1v>r3H~*lXslL&$%lWDOeHB%-jXy8`Q}pgbbAK$qXL+qv_GlS;;UCP{?j{?H!qoW zYr^c@#+EDvUoOW0T6ClyOK5ejV>jGy$9`R<-oJW@OyviT7Exbv5kZ zEM{WFXFEr4L=B@K3a)666K>~Lne|UEA^Vj{&WViE=9k*ces?O~Se-AkqYy@HG2lfB zI`@Bzi2*gb)gix+M%Dx<#=FFJl(4|9ROFlOt59@P(E;3Q*wRCQmB{;I(ff*WbnDX| zoE$U2ezEi9osMoUgF^BwlHb=6(w!CRldZ|>g~B@98SCJqE{KR|lSlP)EbRW*2r72@ zidPH@iLpJ@TUJd)o9FG5HdFYn43E3|Yw&?Nk7G6zBoT-uK>d}1r@nhJZmLqeE(K3? zcF-d4;z_wjMs4~B#+ED5)6zO6)*43pD7p(yO?G~xJ7-ICi{-&(S*szR-0Sp! z9Ous1v1~!;Hl<%_1S8XHr*4mpOuw^SkA2={XO*F|S5_X>!yce4f3P;m@x#xW*t$%D5{mL zk`Z>3S;>0QQ)3H%x~I;osG!2!aPV&RBgkmU`%Lr*k=f}^p+W1X@-sTw@h;>ia4oTy zI-=TZPPQrj-n8cz%b3NGel_2I!A#{f7 zkZ5!+o02<;DVWh>EZ6gxO+Q)M-`LF}> zkbioldpp+b4=~l&KYQIf5*n?1H4A6W#>)3k=Ub>9ti`8DZpbpQ?`ZyN2iW=&%C)>7 zQ9SLJJ9YIJ93aTR5dP<&FRtA7YI*~;wfrK@x!2bHISbu@Z)n@pmGnNWex7-J0YcFE zZuTi?_daZQ`4UkQ6bbRSnzgWRE^P7l3x#YpwP|k+v5Nq<7O9*g?lMIn}0qd`|H{ zw+76$$TpE|P#%~orKjOJJ|D!$bsV4EXFbB;(Ws#q?=gSf+(y6SUqH?0CK4X)HHyc; zzc4MLe85}Z40$Ov=Y94wuQCATF5j!gbM2|88x9=r>5~) zv2ovCL*0dams?NE)U9kZp>w*^;kd_ARwvez`g4T3BRLn9Sqo-0R;dqpSJ|&Rwss*N zsT@&BM6;Q0a>V;}z?7gAc-=kG{KPcHj?&T>MCYV?H-lwsh4fPR$>Ai-PQoEgI#Y?+ z-*&Pp5BJbb*V|5T0*mloDowu}!~5h4b_b!N$i;{2^Ba;)MS96Yhu!HW+vm?SD`c7y zy(Uk8khxvCpjd()M0v7eu%E@Gq|`zcxo`kpp)sEsRrIRY5wb5nb) zYrZEwmc?@|XY+QbtbTj_N!$fq4x2_yrq)J`s)F{79$RSC08ynHROVJiONi3Sfp5C| zQE162Ew}3QH!Lj`Un4Gj!=!rigM_Ql-=ZinRM_#h2(5c82IsIJ+|sa88|wSReojfr zAr8PKOWP?7I~)^}zK0t4j8JZy%n`@NG;6WOSm!Joo~=iQhA1P^?mP+!BJ_xkEMJ9#CBc!PPmf$ zr#1`=#jSFPZ=U%W*lFj>y6bb-;Vb8Ivzvtx*!HAvUt_EC$w}J$M74!Vky>e$tLJ*q z3Kh9UfS=z~5l9<|@E=oXva3I2V351TD|c^8h$zpU8urv*%HZL{0$J@V-H3$LIij1e&wJ06&t5Ln1l;=T zTo0=WR~I92N81gT@UBEp#Ovrp>rygcvYqd6MK9~7U25-UWKiIWJObG4#>UB!2Ia); zAcHnEVkG2SqLd?#??i^{)E&U~h<u*%QjUKis~rm6jiUHGqcS*nb_S6BP|8G{G3n{u&QS*&62H7L1K%aKn2gHE?y* zOVGK^7V4cKw*(y*vU+m=ws8|?zwWv}dq*o_D67cOMe?9dFnOmf)_A+&bN-8%TPsK~e}YIH4!-Oy|MTy%P`uLQEC zu!2=?yVdfILtbxyzAs2b#7-a>(N%e}#)PJi@I6s$+Xi^a4gfFt{EdXQALf2|zbYo0 z+RMPDD2=c5tiaZ}9@si!JQS|snyAsIJw%sp`aZDk&VzCI4r}{Aw1vOZzBD<2-k@9_ zstNucYap|?4*}S)b>&EF|8QIWN_qwbk^oy9)}adUKN6A>?Q;N$WU&<`gZkghD6lI< z7f2Z9aB?R7W*+=3;{edlysoev{r~(HrX`>dfvwFN_x`M)nH|V_{BL9bSF-;@4P2Zt ztbUyZ@Kay@SAYM1JK0d5<#-~ht8QMaN=&?!|4l4Ga8x)Ad+EKcdh16we-Nd1ex;dr zytcjF^Hd4O%C>>(zgxJA>**NpBj=~-alCj0a zK$*?3BQ)^NqJ&xfhvX0b0wl+RBvu;OoOgW-*?I|+?pZ6`7iDm~KE9*V!2SY4CBWID z_eOp&EHX<-v+9UpU7n8&gC@4VeE~)p`}td(GY{9RdsNWG^Jv>&)bHf>v`)R1-kF54 z#%-lCH#VAB{h({X;1$Hyvs};CyJ}@&4sPpo>+-vkrU7VWG^&;7MXcK$=KmO9N7As_ znpb?(ORSmcOS`RUW@soc+UuUR2O8zk{<$;MO&4wx>u0 z=0w2$gE7%8O9JaU-uHOkKi72{iIFawUjy*>TC$%5EI1fWegDN#Xx6{;{@$ea_x?B4 zz#M2uz3G2_3gAc%^4}x+SA`;v@wYpg`^bY_nJTta3l&qd4c)2k#-a9b=Qmxd8~xoE zHAdAbswvB%@I=a?mwkkf!TcdVkEsATU?z>dwG5FXF~IgE7-M6gKR){lD&f=7bo%aAS_GQTB_Fxn+tTtPSH@!i}kgN11TS;>!%j+5Zu<=$< zSrn`ByW>6@uxV^t-~kH`pDCEh;cPoy>v}9xfRgPh`i(3igZ9H}6n6?0%0aki^v_{i z#z0x)_}cpxXN`ePpqe_7&+)y$ep%$$>7?w2n~nJ>M`)IO!H_{L`_(X%|7f^xk&bkw z>vUsJh{)AzbMkrXhWzW!p(o6m`^_suB#GuH(GB?Y7)m(W;n%c+CJy-C$Ma9uLg-n9 zdaqyJCB$38Ke80<{cA64`x)A^VB@B`gENMma-|RRWTp#c513P+F4;z0sbQb=k`8Yrp|90k0)n`L8npf>4?sF@j9OSVl7$u%CWA%jg8 zm=!jQi^^x&t))(t@d_KS*ihTPqyxZ0!#mhv5?@f_EVxqm``*;$t?KSPDf-b>v*h78 z+e}?xe$a>W)2OuLq47*KzD7XT8@frvRri$%LL$Aw$Xt>{+3fbqi;3B!TE$FITdLlw zi}#k1#l75hct~4n7@gSX@@ZxF-J`{zD37U$a;s*decYUTk)M8pyXr5wdGc7Wvj zyPy7=&=Sv3-1!#5yQ^gmgJ^9|0Rsu<4lY3ohT)d3y#wyGkbmK)$)2eZz1Q9sG~ zZ$d+vB3>V@vJ>9F&%huLj%714>JtKIjst{b)}0bTqF;@{cSMOGxuxrT(ahH5u@|mh zr)S?PcX@h|4IC;mS!P(__(4Q;Uobo}vSygueBiN^fu{kG*d<@9UlG%(vv1j6s8g|? zD&!D!SykNq);D|G(81Z?ctWu zlf*KRkb<;yy<)mpC&b!{p@w0tEBPICrrIKJi!KvhnI$?fP*1k<^L&Pl@d999Ujb*^ zrS2uJ=L_K6m9+07*9Q`{zEH_;Z+IBrirB@;8w-C+@M z{`$lUI47Y@pL%;g@F$G;S+mp_X7&Vm>O7_=RqNx)PmHU{zOy^(aIR*Mp<;H|4+>_lE7kWN0C+s~oAZK2dIb>n@o$}>W~Uhq z&O;?O)63+{`t=p8Ham><08>`46-f2=U*b~GH=D{eW!o@SI|7s>oqESOpn5f)!hzUF zpRUX)_Sp~?;Tu{FFXfY2%(mi~)ft;&z|Wz{{0>xosdNR`K+eQn=V-LXq(@n&#(@F+mN;(I0sL-&EWQutc2EnT^JzauD|Jy}FPR!$hUjHmQvuPNskmZq%r>QlzT zXj0}h-T8qAc#z#kWr+QqXq{VR{p1Y&-6$&-<3)vN8;1*qq>0+-ReKtr-O`5huY${* zdD^9KUb%H=W@{Eezb|LsxS;hG0^Gzfab@Mz&&XwZVAJLJ>a6>F%Wh7^9!o4$PO#D_ zS<3$gR9bB~XLVnY2Hk$hIDvK)8lvx?aM#w6oi$?%6wutv;0U1;fpBCVSBQk=ad!X_=X*&c9@N;)tUpG% z;>7I9llR`muF#PZdzxApmUY5KLGEA%KFGc%zO%#;^_Am>qbM}6!vhq1gK&ytK@2V9 z@~y>B^&r;kh&}lpcc0{=EkI6zJ+=1sZLB{l7LDKa#qP-#fH|qP&iD0Q_Jq037@Wsq zd@;JekAB*_<6d3Az1c03G0eYM6rQLH$wefIe*TjdU#8_UtHuV zgVL$~H!%ModZffi)eyXB=J-vumbt#FKY8k2CEz%Jr z@FK8C08x*!?yURwks0U=0|vzyrn&2NHz*%5QPHf@$%Rrz2azy;ud*h>vcq05-ro&N>U7LJY3H6E(FNzHii7Y%&RevoiUoP}um#LY0~`{enE} z5ZM%TCIeXu2^AEduc{J%1KTiG+&Nmc=3Y1um$6W3`0UP;n7ZUV|3h921V7vKKk6&! z*>0^if^}doO@)bWi9zD28(# zD`Z^Dy3;GDN z@zT++>R?K3j=g+)w}b-?1WZ>PCIeMFz@r6vP(Hqfoiih7-18Ee>{F(@AqOuz=wmNuZrt6ISSc|nK9~&!z<nnp$si%1Ve*kXa{di=aL<8t zw7~EJM7~C=#TMroa`5( z(Rh1>S7j4p)#PU%_Y%3^OeH_u8f&aXjMYqh4$y2 z+axT#u^iA(FJ*%P4ohh{$*NL_FvFU_j<>1z@qV@Xw>4yda9+ov8bCKT4}m(63)=+P zmJV;vOc%595EPXS|Cc<<)SF-73O4VIr}w|z>Ft^`5>vdgs2CViT;RMLok1wR|K2qJSt9#e zh?6AKHB7`42@Cin#QWVVyeJH?u(FmbHBBZN#Yhm|=WcHEcBgs;jGkh2mS&pN0ECDR?{xeF+fJQ6le(L$sxRd`5)rBjYJeh+n$w+* z!_PpdN@K_y6avId`YN=@v6x$9>CrN5I&_T6o3aOk-Ik@l6lePJP6Y(AmiybuFJq4( z30jlS_xT69#%ew9CVf=@m9l7#2BujtT^r~kc<9GY7bvdgxqIx+$0YU>&-$~qq z@Dn#ias*_dHF*(0z}c)<2&wnB7#D0|&r5XHzLtO58^s}VG5AQ`@dssI!YeHzV%?IOfEe{kfmNT9FmWU`*;B1^<)Roe@J{zVbOTJ09y*9>w3&n$-$tmL5(D~Mptm3kEH^&)i{JZ2 z|HQ$^K;h9?9w)qe`&Qvewvu7qNZxSH;+uzZV}&{;uhDUFcVN34O%dWo5Sgai>gWGZ z3%-8Z6~)eWOrP|#6>|em>G)-t*Lmyh+VzRY7knu#KR`8*&VU+WnrclumH}OGns38} z?gs(-TH}NE3i~!NWF4XSvG&Zlv3?S23%&+UI4;L07oNh>XPB>x11f#0>o(r+C;?GF zk3Ao$iyho^A6fY|Vm%#vts}%k8US+UVNVovvQK$6-`=3Ox%>#=UfokY*kF;AlqBo# z@#nxBXdH+wxnU*^P-OUR#+iEU8)O+&(G%8{tHaez71m%lNxwMI7*vahnaphUPyP`z zx&F!LO3F=C_eLhr?$hOKaTR7;7q%{9kH)m%`Pn}&Mw?co;@QF zrkDLkxK~dOP!FWvH5gwJB4A}yO8Oy8hzDVlP>i)VqEUGF52hvS7dN;VV%W_Why_$V zTs9ae`0b)LAgT9DnuK&y$OrmkmI%sEJ9@+yLu0BHWp_G2^O+nyTxW}kC6G@of!DJf zKL}A>crg$+Ut+Z>pTue2;RcHm-NwhDjhP2})wYof9bt-zV@1f5hV`_fmmcbNbLs8; zcM_V6!$pfzah>cSTUYA~*`EKxVBp>a`ibBNi3m|H+v5$8H^8Mt)`4SKqZyR%PPzu- zk;Q~Rcnd2Ih6+GbZcq!;-TlW3Do4;$+R?1G(VXFvKFee7A~@_N?C zlN!Bc3t%=lL@eqCM$_#$9=DX9coFD~j*SkBqxNBO0ux+LGXK=>ab(g(=rq?@QkD5YbNum@6aKQx)xo*!XUg)&^_#U3IAExHwQgQ0BW%mNW&%WTCyyKMTLDtQteKb!DKxWPD};Pn#Sh) zG)VDAIPaSgxW7!?tShSOq@Q2lTIH{{Ce@OZ>8!%SwL8AtLgoq!)lPxD-uNy< zno92H4&b}=djY9cnhcJ4J^eUr(_R}ttFmcN_#p4)5siCRH^*4mwjZAbG zcEgsw`oBZ|An)_=?g8lR-uVVnNf_NnJy;a$dA_ja+0WzRU)I-jE?XEj*)WnhGt}CH zvSZsY5kp1qPz+)gGmnH`+HH$EoHz30SWo=$AQyk}rr~@}a8ggoeny!`lyOQO=(}4i z$Bpc{Qw)_k-w!nJv$4mb7Kn6K0vG_M{QZ0gQ>tWY1oKB_Xv=7U)?|~Ohlv{Fiy>UW z|D^@0r>$l%lqYfPgr+LVEuoY53Hsiq5k($2Dpu|OMQ`$yy9LW2dM!*GCa0T#UMgSi zDV@eB(YC3MPfhZkoPz0AU@L-?G)kqg9~w5qB|m#wf7j~VJ=sU=yhf9IavwEm?=~tD zBc!yJ&_|o)7twc1v-~juBLz?LrVp&jwth`cub+33JWTQJlMpoDr2t(&bj^12-;WUVLDryV(AxdfRn1unvT=^utyyV^Z#fvD zAQpR9ImONh`rra7VY3rGlgILE3t89Ia@XFp#n7$uIyrtSjCHD# zrahR>JvSU?qa{L>lT|K`NN!lWdj_RaWyz8pdNGVWAU_Fe+b?xXSJssYMy0l?0?OQ=z_Y*e*j4Eao5N&|T=d>QsasA1r=^ z8kU+g0G!YVmmN}BB8c(nhj8!%puZsMOV5i44kJUv&W{Dh?ZFgEBc)TNT^5^NO?w>lYBd%v zd~pHg&ZzE|mgDy&iW^lWaP*9bC&VhEHXxHzA5V1+?`%h))=Jc;*J7VlHRUAR3JsfV zaI-l?LJo&iC`!Dyo6t)l6j2DVpD5@2=0M`Bxw*MmHU~9GfVQWB{zH1wPeL8^S**hR zLTEA;k&c~}3oZb}+nb3B(}!_jyK2%DeHzGpx;S1R2 zVl(r0MvkcSqU|eGRx3wkiu)!UNZhy2_5Y`Y+~P5%V#!?%Mb5A z?gzDU9W>rD``t#x8!&08 z;jl6wi5*QvY~Rs$O_8GNJTpS!TFb;5;goIueJFc`CB^i=Y?+T`eqgkm{R@>SALJc;*c03 zX&~|ZZvYWUN{e9T1*)I_`SMr1PcGQ@?J57gvCr5~>aPF4-)_q7CpMlHjBB!4ml4>- zk7}H@dU~RHAYBnb{3MY}*;>^~-3h~C@g>O<1gu`WzoYZlS7!&Xp48pw4a>Os{c_$K*-@Q7)$K(alp|M3m;2Jrj6LphcIbtR;;9~RTKa_A@ zKR539BB}6b^-lj7M*?%qz_)5+1>Vv4x}PuU=@8%w^wXI7G#F%f<1@u=)^;PIZZlCx zGWv9En=rF@Y5`dnDx9D4{eDkw(;|Y}ot8i|z!?m0(IU3htkuYV-SlP6*C81&_*z1M@1Hk(5l1Tm zcWG#)y!z(1t6f~Zn3xvfWDrS2^tT(omiNu$)>tO{ zf5r;~oWr5>lp6Q{%w@a2*4=+W7PjK>i>A^ zztigfmg&!Q3p-(CCJVdZWpvEV4}-)!zh06|hVNOuS>HA*)hH+X-2$31;YgRj=)_=m z%{0p*4Zi&N>^!_iUFp@pUu&8n@f~q->7vzd`LdYPdw-5|+41~g7&x-mjB(xQzA`BECk`Qlq!7|rE9!w2qJRTz1W?06= z33#3+y;fuecc~vPuX5FT#n6K0&uNG$RvgUgP>fc~+|H8LW9++R(GVG6CJt9wtD4u{ zjVd8FAPI|@V2BUNP&m>at|vV}s8 zV3}{o#k*G_YGtI9S114}x}AXM@cd$F_hTYss;)-wy2u`#nGY#!}2w+iqa z!>2>PxkUTGwkh z2wdoiCI8-){=t+5wpHc?3c3G&`Ui0Qy{}6Q0$W(n(zj>-&Pw1niKoDBE6G**hrn^^ zrcZeRNRnRbc>C`)jFu=0fR|5|kpBS&IJcDnc=^hX^7qvEd3h{)0AAiE7Wwau$r8Wk zn0RsVRcu_OO4w-Gp8T<=8R8A-g~+e}goVR8;ZniB0sH(yJPX@q@kK6HsMVdEY(5iB z;fD+wv9k^Cz+-ItZr=8jzIjKWz5#N4#)(o1*37u8^UjGm^p^{JU~wlD*nqOlSv%jS z2*(;Li=T^5*B-m(Fyt$q1l$Ut^I%sVA`!J!R+eVvw3oH?7OD(7He#>LjkGN$g{X#= zm6=3^BTlY6@(KOYw`gbo;;tIAZ)%<^dGzC@xBz31(QIrjxySK2$XeB-Dq$?B8=UiU zBEgh9lzGuaGfV#v&Ci@Y5RBrbGiZO@`5|40c68UKQM|a$P8`mlKNqdt#P^J%s-0z~ zXMi>O&WyD)-%k_~Bu&dxUlx^I6rFg7UX3#4Yf{FukJ_$c43ZnCN!f2lUy_iuL{syl zzK@lG%SY<$lI+A7<>f~3Y$M2&&RQD@<+WC`&;b*;^}E272@TMkwr$j z{OP&Rm8nh4>ZvNqTJQrkSGOEBVOE(&Lq3*o`MTH#d%58*By-@WHza9Gn&TsDzf|2~&){oZ$7};_)_@7r3 z^o|G+W@8Z>em;n=WK}mh3orJ`P~-cNJ(RC06tKX6cQaWeok^w{8T1dPn4f{?%j8U! z5o%C)sLODx=7blHWrt^^L!fTb&QIZoSp&bvqb8MXe3OM5@#GN-6Gt!4>Y~*A^GlD7 z-`Gcg`vi!zG14QL=>RnbB3n$5W}D#7*9_EL`k(n3PCIZ?NOvX}oXK))BIwRdiDn$r zguqOx+Vq>>f6rG5N4yGd7W%hkaz$UZhK4n@x%U6=5x~H@eotaZ6NeS_ua@}*NWIPi zeEPrnE>TM5J-nMl_5TKdGBM3S{kpKSUH()kU%(mFPV!{~F_Vpsn2-UOjZ zlz%jf7Bicu0RaW?xJ7+pDj-y3n8rIL9@EG$GFps2h1r zmQp2Yh>@X$2jCu|gZ1fpLXYq7OvKcd*<)8ezdKn2>Ma9{TA*Q|C`9N%DH4?gENh)G zXS3fZzF8eJFE7u-Y}Oiktdrr1z&=E010tZNqhszq^^$2g8<+zp=LM-g$F+C*meA?| z{pT@9!)_2zK}$yyisu%T)JXxyKaH06H_f7FGX$mHl09}@PR|?$lw+(V zFgU5zMin{&&&ksnW?~I z+BkPITAUsa-u;?;Q$_8z&5#*!@l5qpc07jkF|!6!$qVV3k@^RZS0^_hZpAy(2jbs- z2)q{QE3IHsZqv@G@^R)%ZBOf5ET^ZI8C)B8Z$*YEL#4wWH*o;N=~@3OCB5l5Z-F3C1b z44ny({qpNvS6{6gfrTd7YiojQm+*;=;7MyYsJCNmlHWY!_bZ2zA57Y-L>%VCLn|$Z z87l3;Hk|LuRdIpfGn(j8?4OC*z+g=LW>{5Z6fx=!?*Us`rYy5n<#hF}b?G7BP=L5X zn};goiF0B*8T317poU?4VC^O}{|sUaP>_(1b!!~ak!WFXpb|-Xc`JoyG}G|`JtL#Y z2dHHZXh6pz<#W@yVXo);T8@^Hw{q3H??w!&@#$-x`%jB>`Sy;}z5+`8WVyM|i}H^} zH@`$Wz-CIE`-LXu`!I=FJws#IbXSHH>vB){rjrZ{(P5&WOPY>Ul0{sHp4mDr~lj!EFN5_HZoq&X~`;bfr&3dhxbM9m(qYpb;u*97IC&7#pA=XT?1>(IRs^I>Q6 zcUbItRr)D@& z@dT9GO$+z6kUB3Pza=_;$8y|n8|!xv3sBUA2(QF;RpTZWMFQqhM8e?*)aUp5`T;R0 zA#u0>ZY)ZaV)Ms>N`&Qd@wCTDvI@5V_$$0GsmlAtj}I&tnW#a0EI)}(-0=YGZYlw) zO+wdth{jeSm!{KRNDRw{*Rryv*Iv<;DvJv3r>a;)OhSGv92={FU_giX}B7DOJxWUN}+hI?-b8*^YCvG-S#11~8Gl~+P6u-P7_ByeAJ$OQEsM@M zIjuMlD{sH?xuTuD>xCz#t-SJHA&f9X*&_l7saQG)JDKqlP#K==bxco@&j{&54aE*UtW-GxRUT6;_*Px&T^_uM&5+UT*5)9(H;leJl6a4SBN`{H?obR06^?LvpQYEa~$RdrtJ3Gha z$C0fu7EwvE+v0jesE#A+8e7DX<&%P&b$aOHe zO_s<6qt0z`ET7`b=@&{)Cujyck-{Ve1t1W^$4S{`nLi>iXtG_*~#u zrcy#6P9u`U)Uk##c>^Z_&bu;JYgWR|WuEQ^09YC42Z!-88=sZSYcxkf5h&NlxW&w^ z;sCqc=$%?bq@s{wC7BB#s^T}#ydqdXeFL|`NcJI!UF%syWx9T_$qfVFxxjaJ-!Ca! zTDWwnlbwj#-Tq;gS7r)Y*vgzgJ>|7Fur{`SR2=;~X`J>0`;MXZwygx##bWpbyNb}u zu=Jr>^gwoU+yaT3)n#A+`~`x!>$2G0@)*Ek%_5kc%OFq$n@GR)4ocv4&dZUXq+36n zn?UCL)7$$>z_-$JF>ya83Kx(wB&4ukzX(!+I2yV7eqYQ$F_xWe0$1ww$X(|Wc8bLF zZ=LrVZs)M418DN*oFM*hlp{gwfrX6NoA%s|Xna@-oTu0-Cs^8hW}PxA^&q7Yh`vut zey4_hK48F8-#}ie_uY9wm$z2)jqY(vJKgxGsLZy2fGM(;!&h2X`DMzKz?>CY(d6Ob zsa#0~Yb<_gz25My3(#)V5EjyLbo|^{yCCUrcI^Ul`cYztoZ6o?QMkygJued#$3vs_ z0CAXM+zuLGnesNaw|Znf#>%D#(Fa5lG|ID;lRdq$*G=ukPERT<75JOhR3FV9HperA77gbCwS>KY^96`tA31mL`un%!t36PBj2YuY1qtv~L%l{Y+n zFp~oi%MfcEVj{D9i~x4Is1x$%m1UW{Q{UA-e`jXjV;MY+9`Uu<9Uf2JYjN&h`4Oq& zQnho*{R3R#F_`mls?UDyPW@5!7O@j%-DfS;X$NoB~gjp7tn@jPu$-x%H7FUlLbOfpS3WQ+aE3Mj6W~@)z;?UrFEH6^Dkpi^4XCm21;_^gH=lnX_i^A{4E+=pYE@v7;%Nx%_gfp;00#<@ zTV zv#Pe<`~3~S_qXE!Eoug(q?LxwWTzo{D>!=mY?aGt}^5?f`z?(+y`*#uMPu`8{qtRM!Zq$?Z3qB{x zpcStm@-p-pp}5c_i)Q{0o1Lb2u#bFJBLn5C;*0nONaGn1;N)JW#D69}hw)>GXtP#T z_ZfCD=v$yZE`EQYyY>c;W30xgV=ci>O9Z-Qifl(NX$RWP@K>RfKS}$Xu z@?C-Fy1EG$NT!c72q0gfh!F)Pjq45WhmpnhP3MyE_>zL+hAHxLJ0tLUE7wGwSul^- z41<9`oHG5#Q0GL0doUpS79RfD{s7`g&%j{S(~-JT1}k^nn0AZz=3o0JCeR()QN2@a zv;7?6A1TI z$Y*<-1OiQ>I$W~yyK=raw>p3e=ow5geKGzDSJAGXnwpe^(wzj@bgmkw-EvK%ZRSJh>1@+%?U74abxTRmPvhh46bs zakSWvr}EA4mJtCTB3)rwv4ZlU$zZd0coQ<;LR-EBTZNqL2`m>p%VqxvwUIidg z-GrTSZ@b2Jr^b-TDJ$jB+7*`pZ|?=>8X2I2lNi!1MGq&#g965mqi+)r6fx ziP8!y*idbs{X$1KO?OB;SJn z2Tn-nrI={X*Q@UP%5sYn7uK}XZonRa`g)zUH<3?0Sr|f+hrHyv`V~A1SYw zuTfNH^ri;j$H1*UBRQ6F;g{3zzFIST++e!9GY_wKkX;SJoKK*$KR`aR0v1yE1y2-a ziURTqyf1$|As`p-Ay2oHQ}VKZTW~D~h@3SyK2q8)IW*Fq_?blL*mQsoCKfaOmtk>0X5pykvKCaWwDP zhw@+@CI4IGz-}Bqs3P-0q74U7mQ;KI_tx9#rTCPbW5#%YQZrda%n?q4Mk@Fw_7F*c zb*fqfrdrRZ(7PqHBsFZcE{Lfd^Y(0R_mmT(LjBsZorh@#@m(qbRb1n;+^6j~BzsCK zYzFm#{1G%Nuc_dyv_r7|lowQ3KB1A(O0ndIGx^gfPnwG*1UE-xcQ+q|2|L%A_s1_Z zz{iE!tEZjH)g#{$P{g@wUr8dlV*AEK3kT@|%Q|i;yuZQ$>f8r7HR(XazK(BSJ&CG1 z=?!|bpVO4n?XzN0as&iBNju|nYyDzpnUTCOlN+Rp7wLevjeQHM-`Pd%;r;nq1xr0! z!izs5kq?<}(gOK=jE^$!abzNzflQV$0}hy@p0~k!xGeRK-MC2G{V&uVl+!Aa461`7 zt?+1+d;$<5%yz#!9q(%`BaXIj+Ocz8ck!zoGlh9pQ-ZV?ucVv=6&v(YEOdB`G}OXg zFoNLb zn(&6}2l)?ujdhqt)ij*ctfcktJ92NfCY!CetxshE`yp0JOP>9V@J9I+RRw*Zp3#=* z`w!7(j6}{(KlDeZI^QfgoFd6PV`ZjjQtUvgln=7KUA*+sCi-gQQ4GKcUuS*(qNAHk zOK2x#IGMr_BH$E7985Jop4}U+YdP}>AUwsL051nQm17MgGy8HP`7O)k=`PE8&3c&o zDLQ*=P;uj|H?_E>29=w}S!n@1V13TgaqQ0kw__U!1n-yhzQ2Q^g*RThy>CblVlqlX zEN*j1z3@*Fw)9d#$|a@%!ku9*Vv-f|T3%l3B@aZ}TK_+3PrbDdv#3(S53*y>XWM9- z>HO#&ME2Qkxl~|~g3A8%Z6SxI@_u3U{op?=4oW53gOVhFROi|S2)>7fe7$3=z?n*W z)j8$OzWkFXICl5Kyi|-gillxLFVdJWqm?(Hgm52{-D^IwwO2c2LIG&}qk<&-uaxt9 z-{j$*r_DH)g`5H86mP8^aDS&sfz&n7-DL%kjMA$Rd4j*{Lzfxz0BPS$7RoOD|7c>T zKL7Z$uml9scUCWG{qaKVj2IA&B$g-`B~7*rlSw?$EM=-shf zDFS8jU*gYjfa5~qZu__mCqScFyu&(7=hHpUnkPU!Bl}VEA`|zEAOIjCQGiFZ zQTdSXKRhO~3m#G~7tWtj{v{9Tf=ATIpY`}Z|1pXuV3Zka^$VWkzc#EL=C>y5yjJpq2cpL$k79V!_~#ioZ|2V z=)I#Z&AI=~eVGV}4~zzD)e2|gQA_NxHuEIfV|I}q1QAwR5q2U*!nH!3T4{t~f3I}S zXPERcF1>mW8_yznE&ZZWbT~CYF%>WUM-B|~p79~l_lE*I;^*EuV$-y;{C!)#zIs?K z2`**{SgZY6LvbI5YJ}9DgXZp#7Dc9<>)$;}bQ%3Hgx&GPOo3#b68T@Do&a@P)&er~ zxRgS2#mpJ=T;o)MtW*oKBnNu>C+aETt5Ej17btT?J4e>=i6V`11pfxpYa_$|a44}9 z)MztB4Rl~I-8!+yjWeG>@tU~)4@p=K_l4_VR21>8kJ)R5r0Y1;aw>PT!w&wyTdZia!x8JhFOS*p69Gq3IqTu4?x>jJHD*@< z#g>z9;$?;XwZy%luxv7GuTPyL9OjVmUh}wn|5=HN9Q-A|d(u}>tPTgQqW=p`{j z3;`wdsbSu9xLSE@*5DFll~S+8Iu!1HOAMakB?nNBNz{ibUwE$c^~eAAE5D~Wh46%; z{KbR)_W|>mNy9gBIyYNJcr82Y&I0SFIVx@L6RbUS^=3ZUv%aQ?7o)(>C6cbWzCWUY z=)0D+ZJq)pTe667M|>>sDk0?BX@BhX%C0aeyuRst@D1M(E=Fe(SVa6LPP9bp-TF$9 zO<`@gTh%Jo?)SH=9!Y-+x`G;hX6NRXquH#LzyWOVcUJCS7Nm!U`7HfR?RqPXO65l> zcXVgwRcny@QQYoE(Nzi50oCW@ZYN)!9Wm;IN@~0Jr=e{9YPY&yX?c9IUaqHqHIJi^ zW}d~HJZK*5ys(jWgXwMZOOx_wGdn?*KfW1spL4f^*TCmx`dtkId4*5*Peee80nJfS zYJ;Qoq!B5_@TEm-q)lOhDo68=oZfk|j4T9L!A#vL>?0q_XmHo5@z^Nu*}D1iplEf% ziF0F&#llREv(~Xk@2PXgHk{>D&pKOUv?(vK75t+34yj);<9Pu(Lt$->@a1UD{D_T*jM zy*C2KDU&b!&!Nv&)rJqIe45gOT{=o@`{m*c%iNc&B`gqS!b8nima5H39Oo&Cpzo)u zbX11ZP3=_Tz%x8RILWC8um+*EIEKm8nb-QwI`0ax3z% zR~NiI5$Wqjt2bAhH1{p;iik*0i_onDc{+Pox!d&!J&zHo2p6iA$b%HmWnEFBiJpaH zIpH;2geQ+tcT96Yw-}*Pjok^Y@gRPr3ry{ELxr9HhU3fzv9i-rRc3)lJhN_1r$)lO zrJo8);BAplfdE7P`HIJ?V}eG;%PQu#v#t5Id0W|XY=*8oh^!lWvbC4&ao-$lHJ<%k z4X^Zh6Y1@oyd2bxET<~yLrVJTkOUnIQskFlkPR%pXlF7$C6Xw1)Vw$$(PJd&>>SxE z(o4xYJe=uO_wLHFbM_Nm!%){~jqp!0a0!Oa5SfNmQ@Q^?); z5)aqwDs^aWR)gVt2b6ZNM{-|^dT$?8>()DGbwO9J_tb$>B15k&wa5j9)>KwOeB@MwpC192+Kh->cVnKp_b!#fqVL z-cDpTdL*7hdyBk0sm!TBo-OL7-5Vr_h}#i0ALoZ*K7%v8tkj;ICAUKFj0w2=Zkx-ow7{+_EJ>+*SU; zFrTDk+8r!41+%U`Mm#h5{`Z3i;1LaaXU3iU#d6oUcx(AzA)FxA*!uR_;8@z5ql{1U z?tV`Xii==LS(SQay;?DLq5`L_jCfYfFIDRxhmC29^5huZ0M|y_Dx$wQCw8aEc)V;2 z!Xf5b>~04^E~C)0Y^aEB>}XfLsp`1LFZJKBTX|)F>p53QrbtQ0svE$w6053LqP-`k z*Sl&n#7JyX;i#9+zp;7F;-Zrl;D3QDiTp6pgW{gz;bF~piOM{&kP0D;yunantvbfJ zGU4_Fm#p|qE7=qTt7G=Nd>;dz^DZy$8C2QS)lQDN?WW^M`R~`6{lKfDuY`kb!eGaH zLldq;S0i5?1vhGq14{Pb#(Xr}`XMZ<6Lyzx!`Zvqri}fSLxjki``5AF5D7Ls&ObIR z<9;s+`?+TTEZes|Xz|`j8`95prnwRGQd}}(HG<=KeoygkTMt!Qhn2WJ@>!oleixo;Nd3m&ep{ zAo8{I8$2r4^DVORpO**ijpjGYosEe-1TzErNW zQH^cHa55%GLv-je-XcD77vS4XF<06?Bv@y!ZIhaBw9ZdI=~yu|=xJ+=ybPvPf}6ZaL|uBE_;0;`JvRj3(pNIkIEtC^y;>d*JVcT-8Tt# z>>T@dmWB7#vpb)P>zHyY{fRe-WHG~3-SrF-lS{f2lF4K%v-1a8pBBn%1?f0$`YAe* z)qN)%o9Q|1l7wifaPCZ^og+2)>D~GZuANZ#C&bpHde5JX_T1X0L7{GQtJm71@H z%Z){=Sb@?+70{I%^`Qfqe9n8TJzQm}hyO-I9C)XdmLKd6w;pQp9k6@P9_lsdYp%X+ z7Q+O55yoTW5K=qmIO$zn3Xg18%&J~KT@t;9y({K9o)m7{D{^Go%BgoY=py zC0Yd+nT|aXY?6ByPeNpbjE1wUPww}cZRC2uRy|$~B_`B6mY!hJ-Tw`afJqE7+k@#8j*OsIrX)x$vCgR#%`BUO?O9?3qlB<9C)b6_xKrod`NWUpuq zCTYtTN2RW)8NQw7)iIufCGL`CHWXomLwnprgPEA1rEO?f=Jx9D++BX_kcsEE*$)V< zGeavUJK4EZ|LnZ?HH{b6i;lffMhOrf$dSVl&9pO;pf1}i_DRtV#|KYM{XWqf=(#Dx z#*N}sCz22lYV;wPL_OC1_P=|h4t>n2wVtocXcjm($~H@%Fo@F$5`N;Hln)!>!49TCgSlj&-$?(yP-OR#LBO9`pyEZ zYQqWyE??(z3Z#?w^R*z)qdq(}!1B5PkVwS2?u)lL5Rmce90QlN9Z1ODH{O4Arcg#( zT7i@5Q9MKKM5ErZd-}+?!q=}$5PPe;N%kadZ~9&%9EYaK651aHc@LOMYT-}s)5Izd zgt$6adUSXc!Ae!E-RuU;ce?*gmepvcMbz@C^hR0bn`NtvFzYE>jEU>8F$*|Z*R%yn zjAKaw{-Nt7(7J2tZ5)y7mG2THUzSp@y?UHqVWTM~f}D;rjrd}e`l%{@*@6@(AF?NL z9WI?|sj?mxHUP2vbK`MU%Y;%o=@J8@yF|bJ;U7ODoPY+^l|ufZTDe7)`( zd!2i89~Ox31erd*zpjOBAy+kGfz5FKnR=H&(=5N=rvsNGmPfO9^E_nF%jZvyR-(;S zv{CaSG#3BKe_#cv)7r>uVe=TY6|-t51?#(PyjprqrISRNbSdHZCw0YS==X)*fFDWf zn%SY$uTe^cyaWa)Hsn`tpWs(RL7CnWH}U7y{CQR=r} z*k7k~8d|g{Jvi2>e5hBTe5>k(UbsmADWDMc|BQ)4WU<4}ONOh0EssT-9M+rT2rqgB zIOaQSRE>SeJyRR*A#gu)qVfUvdlN5*xq#NA!29%q8o*)9#$Gr0YHDzE!V)-|HL8^8 b5cto3GI^21#8V2&3_#!syn=C^Q$iB}<6?kM literal 0 HcmV?d00001 From 5940e2fd298e9c684946edf5f8a29a680df70989 Mon Sep 17 00:00:00 2001 From: Daniel Marzini <44803752+danielmarzini@users.noreply.github.com> Date: Mon, 4 Nov 2024 11:07:16 +0100 Subject: [PATCH 06/13] modules moved to the shared modules directory --- {slurm-on-gke/modules => modules}/slurm-cluster/main.tf | 0 .../manifest-templates/00-configmap-slurm-config.yml | 0 .../manifest-templates/00-configmap-slurmdb-config.yml | 0 .../slurm-cluster/manifest-templates/00-secret-database-auth.yml | 0 .../slurm-cluster/manifest-templates/00-secret-munge-key.yml | 0 .../manifest-templates/01-pvc-slurm-shared-storage.yml | 0 .../slurm-cluster/manifest-templates/01-pvc-var-lib-mysql.yml | 0 .../manifest-templates/01-pvc-var-spool-slurmctld.yml | 0 .../slurm-cluster/manifest-templates/02-svc-deployment-login.yml | 0 .../slurm-cluster/manifest-templates/02-svc-deployment-mysql.yml | 0 .../manifest-templates/02-svc-deployment-slurmdbd.yml | 0 .../manifest-templates/03-svc-statefulset-slurmctld.yml | 0 .../slurm-cluster/manifest-templates/04-svc-mysql.yml | 0 .../slurm-cluster/manifest-templates/04-svc-slurmctld-0.yml | 0 .../slurm-cluster/manifest-templates/04-svc-slurmdb.yml | 0 {slurm-on-gke/modules => modules}/slurm-cluster/outputs.tf | 0 {slurm-on-gke/modules => modules}/slurm-cluster/variables.tf | 0 {slurm-on-gke/modules => modules}/slurm-nodeset/main.tf | 0 .../manifest-templates/03-svc-statefulset-slurmd.yml | 0 .../slurm-nodeset/manifest-templates/04-svc-slurmd.yml | 0 {slurm-on-gke/modules => modules}/slurm-nodeset/variables.tf | 0 21 files changed, 0 insertions(+), 0 deletions(-) rename {slurm-on-gke/modules => modules}/slurm-cluster/main.tf (100%) rename {slurm-on-gke/modules => modules}/slurm-cluster/manifest-templates/00-configmap-slurm-config.yml (100%) rename {slurm-on-gke/modules => modules}/slurm-cluster/manifest-templates/00-configmap-slurmdb-config.yml (100%) rename {slurm-on-gke/modules => modules}/slurm-cluster/manifest-templates/00-secret-database-auth.yml (100%) rename {slurm-on-gke/modules => modules}/slurm-cluster/manifest-templates/00-secret-munge-key.yml (100%) rename {slurm-on-gke/modules => modules}/slurm-cluster/manifest-templates/01-pvc-slurm-shared-storage.yml (100%) rename {slurm-on-gke/modules => modules}/slurm-cluster/manifest-templates/01-pvc-var-lib-mysql.yml (100%) rename {slurm-on-gke/modules => modules}/slurm-cluster/manifest-templates/01-pvc-var-spool-slurmctld.yml (100%) rename {slurm-on-gke/modules => modules}/slurm-cluster/manifest-templates/02-svc-deployment-login.yml (100%) rename {slurm-on-gke/modules => modules}/slurm-cluster/manifest-templates/02-svc-deployment-mysql.yml (100%) rename {slurm-on-gke/modules => modules}/slurm-cluster/manifest-templates/02-svc-deployment-slurmdbd.yml (100%) rename {slurm-on-gke/modules => modules}/slurm-cluster/manifest-templates/03-svc-statefulset-slurmctld.yml (100%) rename {slurm-on-gke/modules => modules}/slurm-cluster/manifest-templates/04-svc-mysql.yml (100%) rename {slurm-on-gke/modules => modules}/slurm-cluster/manifest-templates/04-svc-slurmctld-0.yml (100%) rename {slurm-on-gke/modules => modules}/slurm-cluster/manifest-templates/04-svc-slurmdb.yml (100%) rename {slurm-on-gke/modules => modules}/slurm-cluster/outputs.tf (100%) rename {slurm-on-gke/modules => modules}/slurm-cluster/variables.tf (100%) rename {slurm-on-gke/modules => modules}/slurm-nodeset/main.tf (100%) rename {slurm-on-gke/modules => modules}/slurm-nodeset/manifest-templates/03-svc-statefulset-slurmd.yml (100%) rename {slurm-on-gke/modules => modules}/slurm-nodeset/manifest-templates/04-svc-slurmd.yml (100%) rename {slurm-on-gke/modules => modules}/slurm-nodeset/variables.tf (100%) diff --git a/slurm-on-gke/modules/slurm-cluster/main.tf b/modules/slurm-cluster/main.tf similarity index 100% rename from slurm-on-gke/modules/slurm-cluster/main.tf rename to modules/slurm-cluster/main.tf diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-configmap-slurm-config.yml b/modules/slurm-cluster/manifest-templates/00-configmap-slurm-config.yml similarity index 100% rename from slurm-on-gke/modules/slurm-cluster/manifest-templates/00-configmap-slurm-config.yml rename to modules/slurm-cluster/manifest-templates/00-configmap-slurm-config.yml diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-configmap-slurmdb-config.yml b/modules/slurm-cluster/manifest-templates/00-configmap-slurmdb-config.yml similarity index 100% rename from slurm-on-gke/modules/slurm-cluster/manifest-templates/00-configmap-slurmdb-config.yml rename to modules/slurm-cluster/manifest-templates/00-configmap-slurmdb-config.yml diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-secret-database-auth.yml b/modules/slurm-cluster/manifest-templates/00-secret-database-auth.yml similarity index 100% rename from slurm-on-gke/modules/slurm-cluster/manifest-templates/00-secret-database-auth.yml rename to modules/slurm-cluster/manifest-templates/00-secret-database-auth.yml diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/00-secret-munge-key.yml b/modules/slurm-cluster/manifest-templates/00-secret-munge-key.yml similarity index 100% rename from slurm-on-gke/modules/slurm-cluster/manifest-templates/00-secret-munge-key.yml rename to modules/slurm-cluster/manifest-templates/00-secret-munge-key.yml diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-slurm-shared-storage.yml b/modules/slurm-cluster/manifest-templates/01-pvc-slurm-shared-storage.yml similarity index 100% rename from slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-slurm-shared-storage.yml rename to modules/slurm-cluster/manifest-templates/01-pvc-slurm-shared-storage.yml diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-var-lib-mysql.yml b/modules/slurm-cluster/manifest-templates/01-pvc-var-lib-mysql.yml similarity index 100% rename from slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-var-lib-mysql.yml rename to modules/slurm-cluster/manifest-templates/01-pvc-var-lib-mysql.yml diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-var-spool-slurmctld.yml b/modules/slurm-cluster/manifest-templates/01-pvc-var-spool-slurmctld.yml similarity index 100% rename from slurm-on-gke/modules/slurm-cluster/manifest-templates/01-pvc-var-spool-slurmctld.yml rename to modules/slurm-cluster/manifest-templates/01-pvc-var-spool-slurmctld.yml diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-login.yml b/modules/slurm-cluster/manifest-templates/02-svc-deployment-login.yml similarity index 100% rename from slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-login.yml rename to modules/slurm-cluster/manifest-templates/02-svc-deployment-login.yml diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-mysql.yml b/modules/slurm-cluster/manifest-templates/02-svc-deployment-mysql.yml similarity index 100% rename from slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-mysql.yml rename to modules/slurm-cluster/manifest-templates/02-svc-deployment-mysql.yml diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-slurmdbd.yml b/modules/slurm-cluster/manifest-templates/02-svc-deployment-slurmdbd.yml similarity index 100% rename from slurm-on-gke/modules/slurm-cluster/manifest-templates/02-svc-deployment-slurmdbd.yml rename to modules/slurm-cluster/manifest-templates/02-svc-deployment-slurmdbd.yml diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/03-svc-statefulset-slurmctld.yml b/modules/slurm-cluster/manifest-templates/03-svc-statefulset-slurmctld.yml similarity index 100% rename from slurm-on-gke/modules/slurm-cluster/manifest-templates/03-svc-statefulset-slurmctld.yml rename to modules/slurm-cluster/manifest-templates/03-svc-statefulset-slurmctld.yml diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-mysql.yml b/modules/slurm-cluster/manifest-templates/04-svc-mysql.yml similarity index 100% rename from slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-mysql.yml rename to modules/slurm-cluster/manifest-templates/04-svc-mysql.yml diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-slurmctld-0.yml b/modules/slurm-cluster/manifest-templates/04-svc-slurmctld-0.yml similarity index 100% rename from slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-slurmctld-0.yml rename to modules/slurm-cluster/manifest-templates/04-svc-slurmctld-0.yml diff --git a/slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-slurmdb.yml b/modules/slurm-cluster/manifest-templates/04-svc-slurmdb.yml similarity index 100% rename from slurm-on-gke/modules/slurm-cluster/manifest-templates/04-svc-slurmdb.yml rename to modules/slurm-cluster/manifest-templates/04-svc-slurmdb.yml diff --git a/slurm-on-gke/modules/slurm-cluster/outputs.tf b/modules/slurm-cluster/outputs.tf similarity index 100% rename from slurm-on-gke/modules/slurm-cluster/outputs.tf rename to modules/slurm-cluster/outputs.tf diff --git a/slurm-on-gke/modules/slurm-cluster/variables.tf b/modules/slurm-cluster/variables.tf similarity index 100% rename from slurm-on-gke/modules/slurm-cluster/variables.tf rename to modules/slurm-cluster/variables.tf diff --git a/slurm-on-gke/modules/slurm-nodeset/main.tf b/modules/slurm-nodeset/main.tf similarity index 100% rename from slurm-on-gke/modules/slurm-nodeset/main.tf rename to modules/slurm-nodeset/main.tf diff --git a/slurm-on-gke/modules/slurm-nodeset/manifest-templates/03-svc-statefulset-slurmd.yml b/modules/slurm-nodeset/manifest-templates/03-svc-statefulset-slurmd.yml similarity index 100% rename from slurm-on-gke/modules/slurm-nodeset/manifest-templates/03-svc-statefulset-slurmd.yml rename to modules/slurm-nodeset/manifest-templates/03-svc-statefulset-slurmd.yml diff --git a/slurm-on-gke/modules/slurm-nodeset/manifest-templates/04-svc-slurmd.yml b/modules/slurm-nodeset/manifest-templates/04-svc-slurmd.yml similarity index 100% rename from slurm-on-gke/modules/slurm-nodeset/manifest-templates/04-svc-slurmd.yml rename to modules/slurm-nodeset/manifest-templates/04-svc-slurmd.yml diff --git a/slurm-on-gke/modules/slurm-nodeset/variables.tf b/modules/slurm-nodeset/variables.tf similarity index 100% rename from slurm-on-gke/modules/slurm-nodeset/variables.tf rename to modules/slurm-nodeset/variables.tf From b1e2626816ef1c41332ba4715999a43ea9393312 Mon Sep 17 00:00:00 2001 From: Daniel Marzini <44803752+danielmarzini@users.noreply.github.com> Date: Mon, 4 Nov 2024 11:08:03 +0100 Subject: [PATCH 07/13] module references updated --- slurm-on-gke/main.tf | 26 +++++++++++++------------- slurm-on-gke/variables.tf | 1 + 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/slurm-on-gke/main.tf b/slurm-on-gke/main.tf index 3f60ec179..6aa01b7a9 100644 --- a/slurm-on-gke/main.tf +++ b/slurm-on-gke/main.tf @@ -15,7 +15,7 @@ */ module "slurm-cluster-001" { - source = "./modules/slurm-cluster/" + source = "../modules/slurm-cluster/" namespace = "slurm" namespace_create = true @@ -23,27 +23,27 @@ module "slurm-cluster-001" { } module "slurm-workers-001" { - source = "./modules/slurm-nodeset" - name = "slurmd1" + source = "../modules/slurm-nodeset" + name = "slurmd" config = { - type = "n1-standard-8" - instances = 1 + type = "g2-standard-4" + instances = 2 namespace = module.slurm-cluster-001.namespace image = var.config.image - accelerator = { - type = "nvidia-tesla-t4" - count = 2 - } } } module "slurm-workers-002" { - source = "./modules/slurm-nodeset" - name = "slurmd" + source = "../modules/slurm-nodeset" + name = "slurmd1" config = { - type = "g2-standard-4" - instances = 2 + type = "n1-standard-8" + instances = 1 namespace = module.slurm-cluster-001.namespace image = var.config.image + accelerator = { + type = "nvidia-tesla-t4" + count = 2 + } } } diff --git a/slurm-on-gke/variables.tf b/slurm-on-gke/variables.tf index 4e92ba97b..51f78fde9 100644 --- a/slurm-on-gke/variables.tf +++ b/slurm-on-gke/variables.tf @@ -37,6 +37,7 @@ variable "impersonate_service_account" { description = "Service account to be used while using Google Cloud APIs" type = string nullable = true + default = null } variable "config" { From 4bb954068a8a2288bb219134482e6a4be3b241ae Mon Sep 17 00:00:00 2001 From: Daniel Marzini <44803752+danielmarzini@users.noreply.github.com> Date: Mon, 4 Nov 2024 11:08:37 +0100 Subject: [PATCH 08/13] Update README.md --- slurm-on-gke/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/slurm-on-gke/README.md b/slurm-on-gke/README.md index 648fe02fd..42f937397 100644 --- a/slurm-on-gke/README.md +++ b/slurm-on-gke/README.md @@ -53,7 +53,7 @@ Before proceeding further, review the Terraform module configuration for this pa ```HCL module "slurm-cluster-001" { - source = "./modules/slurm-cluster/" + source = "../modules/slurm-cluster/" namespace = "slurm" namespace_create = true @@ -102,7 +102,7 @@ The following streamlined configuration for a new set of worker nodes requires o ```HCL # example - 1 module "slurm-workers-001" { - source = "./modules/slurm-nodeset" + source = "../modules/slurm-nodeset" name = "slurmd" config = { type = "g2-standard-4" @@ -114,7 +114,7 @@ module "slurm-workers-001" { # example - 2 module "slurm-workers-002" { - source = "./modules/slurm-nodeset" + source = "../modules/slurm-nodeset" name = "slurmd1" config = { type = "n1-standard-8" From d05cdd80de567882407d77e2b370e2b8e3c97f44 Mon Sep 17 00:00:00 2001 From: Daniel Marzini <44803752+danielmarzini@users.noreply.github.com> Date: Mon, 4 Nov 2024 14:54:11 +0100 Subject: [PATCH 09/13] Update README.md --- slurm-on-gke/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/slurm-on-gke/README.md b/slurm-on-gke/README.md index 42f937397..1a5a3a504 100644 --- a/slurm-on-gke/README.md +++ b/slurm-on-gke/README.md @@ -269,7 +269,6 @@ cd image docker build -t europe-west3-docker.pkg.dev/$PROJECT_ID/slurm/slurmd:53 docker push europe-west3-docker.pkg.dev/$PROJECT_ID/slurm/slurmd:535 ``` - The output is similar to the following: ```bash From 76703f0e63b15cc741c3048dddb439d6d08a6e46 Mon Sep 17 00:00:00 2001 From: Daniel Marzini <44803752+danielmarzini@users.noreply.github.com> Date: Fri, 8 Nov 2024 10:28:22 +0100 Subject: [PATCH 10/13] Update README.md --- slurm-on-gke/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/slurm-on-gke/README.md b/slurm-on-gke/README.md index 1a5a3a504..efb1d6941 100644 --- a/slurm-on-gke/README.md +++ b/slurm-on-gke/README.md @@ -438,6 +438,10 @@ terraform destroy This step deletes all the resources that you created previously: the GKE cluster, the VPC network, the firewall rules, and the Google Cloud project. +## Disclaimer + +This project welcomes contributions but operates with a best-effort maintenance approach. Please review our [CONTRIBUTING](https://github.com/GoogleCloudPlatform/ai-on-gke/blob/main/contributing.md) for guidelines on how to contribute effectively and understand the review process. + ## License * The use of the assets contained in this repository is subject to compliance with [Google's AI Principles](https://ai.google/responsibility/principles/) From 8748ab7c1476b19d2e1396f7abd58ec775abc20c Mon Sep 17 00:00:00 2001 From: Daniel Marzini <44803752+danielmarzini@users.noreply.github.com> Date: Wed, 13 Nov 2024 14:40:12 +0100 Subject: [PATCH 11/13] Revert "Update README.md" This reverts commit 76703f0e63b15cc741c3048dddb439d6d08a6e46. --- slurm-on-gke/README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/slurm-on-gke/README.md b/slurm-on-gke/README.md index efb1d6941..1a5a3a504 100644 --- a/slurm-on-gke/README.md +++ b/slurm-on-gke/README.md @@ -438,10 +438,6 @@ terraform destroy This step deletes all the resources that you created previously: the GKE cluster, the VPC network, the firewall rules, and the Google Cloud project. -## Disclaimer - -This project welcomes contributions but operates with a best-effort maintenance approach. Please review our [CONTRIBUTING](https://github.com/GoogleCloudPlatform/ai-on-gke/blob/main/contributing.md) for guidelines on how to contribute effectively and understand the review process. - ## License * The use of the assets contained in this repository is subject to compliance with [Google's AI Principles](https://ai.google/responsibility/principles/) From ca8f266aef53a6f84ae049ce1eda82e81623a550 Mon Sep 17 00:00:00 2001 From: Daniel Marzini <44803752+danielmarzini@users.noreply.github.com> Date: Fri, 15 Nov 2024 15:45:26 +0100 Subject: [PATCH 12/13] pinned version --- slurm-on-gke/providers.tf | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/slurm-on-gke/providers.tf b/slurm-on-gke/providers.tf index 5cc9c6b03..4f250bafe 100644 --- a/slurm-on-gke/providers.tf +++ b/slurm-on-gke/providers.tf @@ -14,6 +14,24 @@ * limitations under the License. */ +terraform { + required_version = ">= 1.9.3" + required_providers { + google = { + source = "google" + version = "~> 6.9.0" + } + google-beta = { + source = "google-beta" + version = "~> 6.9.0" + } + kubernetes = { + source = "kubernetes" + version = "~> 2.33.0" + } + } +} + provider "google" { impersonate_service_account = var.impersonate_service_account } From da30a6b33263053cbcc0a8a0230b93163b570c43 Mon Sep 17 00:00:00 2001 From: Daniel Marzini <44803752+danielmarzini@users.noreply.github.com> Date: Fri, 15 Nov 2024 15:45:44 +0100 Subject: [PATCH 13/13] Update providers.tf --- slurm-on-gke/providers.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slurm-on-gke/providers.tf b/slurm-on-gke/providers.tf index 4f250bafe..1a2f18ccf 100644 --- a/slurm-on-gke/providers.tf +++ b/slurm-on-gke/providers.tf @@ -15,7 +15,7 @@ */ terraform { - required_version = ">= 1.9.3" + required_version = "~> 1.9.3" required_providers { google = { source = "google"