Merge pull request #196 from unity-sds/195-upgrade-airflow-to-version…

…-2.10.0

195 upgrade airflow to version 2.10.0

CHANGELOG.md

@@ -5,6 +5,38 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+# [Unity Release 24.3] - 2024-09-22
+
+## Tags
+
+- SPS Version 2.2.0
+- OGC API Version 1.1.0
+- OGC Python Client Version 1.1.0
+
+## Repositories
+
+- unity-sps: <https://github.com/unity-sds/unity-sps/releases/tag/2.2.0>
+- unity-sps-ogc-processes-api: <https://github.com/unity-sds/unity-sps-ogc-processes-api/releases/tag/1.1.0>
+- unity-sps-ogc-processes-api-client-python: <https://github.com/unity-sds/unity-sps-ogc-processes-api-client-python/releases/tag/1.1.0>
+
+## Epics
+
+- EPIC: `Security`
+  - [[Bug]: Upgrade EKS 1.27 AMIs](https://github.com/unity-sds/unity-sps/issues/159)
+- EPIC: `Scaling`
+  - [[New Feature]: Increase ephemeral disk space for Airflow workers](https://github.com/unity-sds/unity-sps/issues/152)
+  - [[New Feature]: Enable users to select the EC2 type to execute a workload](https://github.com/unity-sds/unity-sps/issues/153)
+  - [[New Feature]: Set the DAG run status to "failed" if the main worker task failed](https://github.com/unity-sds/unity-sps/issues/189)
+- EPIC: `Airflow/WPS-T Integration`
+  - [[New Feature]: Create test to deploy, execute and undeploy the CWL DAG](https://github.com/unity-sds/unity-sps/issues/131)
+- EPIC: `Production Venue Deployments`
+  - [[New Feature]: Airflow HTTPD Proxy development and configuration](https://github.com/unity-sds/unity-sps/issues/125)
+  - [[New Feature]: Expose SPS health check endpoints](https://github.com/unity-sds/unity-sps/issues/127)
+- EPIC: `SPS Infrastructure`
+  - [[New Feature]: Update documentation for SPS deployment](https://github.com/unity-sds/unity-sps/issues/116)
+  - [[New Feature]: Review the SPS GitBook documentation](https://github.com/unity-sds/unity-sps/issues/118)
+  - [[New Feature]: Store SPS Terraform state on S3](https://github.com/unity-sds/unity-sps/issues/132)
+  - [[New Feature]: Parametrize the SPS Integration Tests](https://github.com/unity-sds/unity-sps/issues/155)
 
 # [Unity Release 24.2] - 2024-07-01
 

airflow/dags/cwl_dag.py

@@ -26,9 +26,7 @@
 POD_NAMESPACE = "sps"
 POD_LABEL = "cwl_task"
 # SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.1.0"
-SPS_DOCKER_CWL_IMAGE = (
-    "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:186-ecr-cwl-dag"  # TODO Update with next release
-)
+SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0-beta-3"
 
 NODE_POOL_DEFAULT = "airflow-kubernetes-pod-operator"
 NODE_POOL_HIGH_WORKLOAD = "airflow-kubernetes-pod-operator-high-workload"
@@ -80,6 +78,7 @@
 dag = DAG(
     dag_id="cwl_dag",
     description="CWL DAG",
+    dag_display_name="CWL DAG",
     tags=["CWL"],
     is_paused_upon_creation=False,
     catchup=False,
@@ -100,7 +99,7 @@
         "request_memory": Param(
             "4Gi",
             type="string",
-            enum=["8Gi", "16Gi", "32Gi", "64Gi", "128Gi", "256Gi"],
+            enum=["4Gi", "8Gi", "16Gi", "32Gi", "64Gi", "128Gi", "256Gi"],
             title="Docker container memory",
         ),
         "request_cpu": Param(
@@ -112,7 +111,7 @@
         "request_storage": Param(
             "10Gi",
             type="string",
-            enum=["10Gi", "50Gi", "100Gi", "200Gi", "300Gi"],
+            enum=["10Gi", "50Gi", "100Gi", "150Gi", "200Gi", "250Gi"],
             title="Docker container storage",
         ),
         "use_ecr": Param(False, type="boolean", title="Log into AWS Elastic Container Registry (ECR)"),

airflow/dags/docker_cwl_pod.yaml

@@ -21,7 +21,7 @@ spec:
 
   containers:
   - name: cwl-docker
-    image: ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.1.0
+    image: ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0-beta-1
     imagePullPolicy: Always
     command: ["/usr/share/cwl/docker_cwl_entrypoint.sh"]
     securityContext:

airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py

@@ -23,7 +23,7 @@
 # The Kubernetes namespace within which the Pod is run (it must already exist)
 POD_NAMESPACE = "sps"
 POD_LABEL = "sbg_task"
-SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.1.0"
+SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0-beta-1"
 
 # The path of the working directory where the CWL workflow is executed
 # (aka the starting directory for cwl-runner).

airflow/dags/sbg_preprocess_cwl_dag.py

@@ -17,7 +17,7 @@
 # The Kubernetes namespace within which the Pod is run (it must already exist)
 POD_NAMESPACE = "sps"
 POD_LABEL = "sbg_preprocess_task"
-SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.1.0"
+SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.2.0-beta-1"
 
 # The path of the working directory where the CWL workflow is executed
 # (aka the starting directory for cwl-runner).

airflow/docker/custom_airflow/Dockerfile

@@ -1,4 +1,4 @@
-FROM apache/airflow:2.9.1-python3.11
+FROM apache/airflow:2.10.0-python3.11
 
 RUN pip install cwltool==3.1.20240112164112
 RUN pip install boto3==1.34.89

airflow/docker/cwl/docker_cwl_entrypoint.sh

@@ -72,7 +72,7 @@ fi
 # List contents when done
 pwd
 ls -lR
-cwl-runner --tmp-outdir-prefix "$PWD"/ --no-read-only "$cwl_workflow" "$job_args"
+cwl-runner --debug --tmp-outdir-prefix "$PWD"/ --no-read-only "$cwl_workflow" "$job_args"
 ls -lR
 
 # Optionally, save the requested output file to a location

terraform-unity/README.md

@@ -187,12 +187,12 @@ terraform apply -no-color 2>&1 | tee apply_output.txt
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_airflow_docker_images"></a> [airflow\_docker\_images](#input\_airflow\_docker\_images) | Docker images for the associated Airflow services. | <pre>object({<br>    airflow = object({<br>      name = string<br>      tag  = string<br>    })<br>  })</pre> | <pre>{<br>  "airflow": {<br>    "name": "ghcr.io/unity-sds/unity-sps/sps-airflow",<br>    "tag": "2.1.2"<br>  }<br>}</pre> | no |
+| <a name="input_airflow_docker_images"></a> [airflow\_docker\_images](#input\_airflow\_docker\_images) | Docker images for the associated Airflow services. | <pre>object({<br>    airflow = object({<br>      name = string<br>      tag  = string<br>    })<br>  })</pre> | <pre>{<br>  "airflow": {<br>    "name": "ghcr.io/unity-sds/unity-sps/sps-airflow",<br>    "tag": "2.2.0-beta-1"<br>  }<br>}</pre> | no |
 | <a name="input_airflow_webserver_password"></a> [airflow\_webserver\_password](#input\_airflow\_webserver\_password) | The password for the Airflow webserver and UI. | `string` | n/a | yes |
 | <a name="input_airflow_webserver_username"></a> [airflow\_webserver\_username](#input\_airflow\_webserver\_username) | The username for the Airflow webserver and UI. | `string` | `"admin"` | no |
 | <a name="input_dag_catalog_repo"></a> [dag\_catalog\_repo](#input\_dag\_catalog\_repo) | Git repository that stores the catalog of Airflow DAGs. | <pre>object({<br>    url                 = string<br>    ref                 = string<br>    dags_directory_path = string<br>  })</pre> | <pre>{<br>  "dags_directory_path": "airflow/dags",<br>  "ref": "develop",<br>  "url": "https://github.com/unity-sds/unity-sps.git"<br>}</pre> | no |
 | <a name="input_helm_charts"></a> [helm\_charts](#input\_helm\_charts) | Helm charts for the associated services. | <pre>map(object({<br>    repository = string<br>    chart      = string<br>    version    = string<br>  }))</pre> | <pre>{<br>  "airflow": {<br>    "chart": "airflow",<br>    "repository": "https://airflow.apache.org",<br>    "version": "1.13.1"<br>  },<br>  "keda": {<br>    "chart": "keda",<br>    "repository": "https://kedacore.github.io/charts",<br>    "version": "v2.14.2"<br>  }<br>}</pre> | no |
-| <a name="input_karpenter_node_classes"></a> [karpenter\_node\_classes](#input\_karpenter\_node\_classes) | Configuration for karpenter\_node\_classes | <pre>map(object({<br>    volume_size = string<br>  }))</pre> | <pre>{<br>  "airflow-kubernetes-pod-operator-high-workload": {<br>    "volume_size": "200Gi"<br>  },<br>  "default": {<br>    "volume_size": "30Gi"<br>  }<br>}</pre> | no |
+| <a name="input_karpenter_node_classes"></a> [karpenter\_node\_classes](#input\_karpenter\_node\_classes) | Configuration for karpenter\_node\_classes | <pre>map(object({<br>    volume_size = string<br>  }))</pre> | <pre>{<br>  "airflow-kubernetes-pod-operator-high-workload": {<br>    "volume_size": "300Gi"<br>  },<br>  "default": {<br>    "volume_size": "30Gi"<br>  }<br>}</pre> | no |
 | <a name="input_karpenter_node_pools"></a> [karpenter\_node\_pools](#input\_karpenter\_node\_pools) | Configuration for Karpenter node pools | <pre>map(object({<br>    requirements : list(object({<br>      key : string<br>      operator : string<br>      values : list(string)<br>    }))<br>    nodeClassRef : string<br>    limits : object({<br>      cpu : string<br>      memory : string<br>    })<br>    disruption : object({<br>      consolidationPolicy : string<br>      consolidateAfter : string<br>    })<br>  }))</pre> | <pre>{<br>  "airflow-celery-workers": {<br>    "disruption": {<br>      "consolidateAfter": "1m",<br>      "consolidationPolicy": "WhenEmpty"<br>    },<br>    "limits": {<br>      "cpu": "80",<br>      "memory": "320Gi"<br>    },<br>    "nodeClassRef": "default",<br>    "requirements": [<br>      {<br>        "key": "karpenter.k8s.aws/instance-family",<br>        "operator": "In",<br>        "values": [<br>          "m7i",<br>          "m6i",<br>          "m5",<br>          "t3",<br>          "c7i",<br>          "c6i",<br>          "c5",<br>          "r7i",<br>          "r6i",<br>          "r5"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Gt",<br>        "values": [<br>          "1"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Lt",<br>        "values": [<br>          "9"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Gt",<br>        "values": [<br>          "8191"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Lt",<br>        "values": [<br>          "32769"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-hypervisor",<br>        "operator": "In",<br>        "values": [<br>          "nitro"<br>        ]<br>      }<br>    ]<br>  },<br>  "airflow-core-components": {<br>    "disruption": {<br>      "consolidateAfter": "1m",<br>      "consolidationPolicy": "WhenEmpty"<br>    },<br>    "limits": {<br>      "cpu": "40",<br>      "memory": "160Gi"<br>    },<br>    "nodeClassRef": "default",<br>    "requirements": [<br>      {<br>        "key": "karpenter.k8s.aws/instance-family",<br>        "operator": "In",<br>        "values": [<br>          "m7i",<br>          "m6i",<br>          "m5",<br>          "t3",<br>          "c7i",<br>          "c6i",<br>          "c5",<br>          "r7i",<br>          "r6i",<br>          "r5"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Gt",<br>        "values": [<br>          "1"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Lt",<br>        "values": [<br>          "17"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Gt",<br>        "values": [<br>          "8191"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Lt",<br>        "values": [<br>          "32769"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-hypervisor",<br>        "operator": "In",<br>        "values": [<br>          "nitro"<br>        ]<br>      }<br>    ]<br>  },<br>  "airflow-kubernetes-pod-operator": {<br>    "disruption": {<br>      "consolidateAfter": "1m",<br>      "consolidationPolicy": "WhenEmpty"<br>    },<br>    "limits": {<br>      "cpu": "100",<br>      "memory": "400Gi"<br>    },<br>    "nodeClassRef": "default",<br>    "requirements": [<br>      {<br>        "key": "karpenter.k8s.aws/instance-family",<br>        "operator": "In",<br>        "values": [<br>          "m7i",<br>          "m6i",<br>          "m5",<br>          "t3",<br>          "c7i",<br>          "c6i",<br>          "c5",<br>          "r7i",<br>          "r6i",<br>          "r5"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Gt",<br>        "values": [<br>          "1"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Lt",<br>        "values": [<br>          "17"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Gt",<br>        "values": [<br>          "8191"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Lt",<br>        "values": [<br>          "32769"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-hypervisor",<br>        "operator": "In",<br>        "values": [<br>          "nitro"<br>        ]<br>      }<br>    ]<br>  },<br>  "airflow-kubernetes-pod-operator-high-workload": {<br>    "disruption": {<br>      "consolidateAfter": "1m",<br>      "consolidationPolicy": "WhenEmpty"<br>    },<br>    "limits": {<br>      "cpu": "528",<br>      "memory": "1056Gi"<br>    },<br>    "nodeClassRef": "airflow-kubernetes-pod-operator-high-workload",<br>    "requirements": [<br>      {<br>        "key": "karpenter.k8s.aws/instance-family",<br>        "operator": "In",<br>        "values": [<br>          "m7i",<br>          "m6i",<br>          "m5",<br>          "t3",<br>          "c7i",<br>          "c6i",<br>          "c5",<br>          "r7i",<br>          "r6i",<br>          "r5"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Gt",<br>        "values": [<br>          "1"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Lt",<br>        "values": [<br>          "49"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Gt",<br>        "values": [<br>          "8191"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Lt",<br>        "values": [<br>          "98305"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-hypervisor",<br>        "operator": "In",<br>        "values": [<br>          "nitro"<br>        ]<br>      }<br>    ]<br>  }<br>}</pre> | no |
 | <a name="input_kubeconfig_filepath"></a> [kubeconfig\_filepath](#input\_kubeconfig\_filepath) | The path to the kubeconfig file for the Kubernetes cluster. | `string` | n/a | yes |
 | <a name="input_mcp_ami_owner_id"></a> [mcp\_ami\_owner\_id](#input\_mcp\_ami\_owner\_id) | The owner ID of the MCP AMIs | `string` | `"794625662971"` | no |

terraform-unity/variables.tf

@@ -73,7 +73,7 @@ variable "airflow_docker_images" {
   default = {
     airflow = {
       name = "ghcr.io/unity-sds/unity-sps/sps-airflow"
-      tag  = "2.1.2"
+      tag  = "2.2.0-beta-1"
     }
   }
 }
@@ -127,7 +127,7 @@ variable "karpenter_node_classes" {
       volume_size = "30Gi"
     }
     "airflow-kubernetes-pod-operator-high-workload" = {
-      volume_size = "200Gi"
+      volume_size = "300Gi"
     }
   }
 }