diff --git a/airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py b/airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py index 80b3ba7c..f670704b 100644 --- a/airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py +++ b/airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py @@ -10,6 +10,7 @@ from datetime import datetime import boto3 +from airflow.models import Variable from airflow.models.baseoperator import chain from airflow.models.param import Param from airflow.operators.python import PythonOperator @@ -30,7 +31,8 @@ # SSM keys for venue-dependent parameters UNITY_DAPA_CLIENT_ID = "/unity/shared-services/dapa/client-id" UNITY_DAPA_API_URL = "/unity/shared-services/dapa/api-url" -UNITY_OUTPUT_DATA_BUCKET = "/unity/ds/data/bucket/primary-data-bucket" +UNITY_IDENTIFIER = f"""{Variable.get("unity_deployment_name")}/{Variable.get("unity_venue")}-{Variable.get("unity_counter")}""" +UNITY_OUTPUT_DATA_BUCKET = f"/unity/ds/data/bucket/{UNITY_IDENTIFIER}/primary-data-bucket" # Resources needed by each Task # EC2 r6a.xlarge 4vCPU 32GiB diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/main.tf b/terraform-unity/modules/terraform-unity-sps-airflow/main.tf index 39009264..dd1e0c64 100644 --- a/terraform-unity/modules/terraform-unity-sps-airflow/main.tf +++ b/terraform-unity/modules/terraform-unity-sps-airflow/main.tf @@ -856,6 +856,18 @@ resource "aws_ssm_parameter" "ogc_processes_api_url" { }) } +resource "aws_ssm_parameter" "unity_output_data_bucket" { + name = format("/unity/ds/data/bucket/%s/%s-%s/primary-data-bucket", var.deployment_name, var.venue, var.counter) + description = "The S3 Address of the primary DS data bucket." + type = "String" + value = "sps-${var.venue}-ds-storage" + tags = merge(local.common_tags, { + Name = format(local.resource_name_prefix, "endpoints-output_data_bucket") + Component = "SSM" + Stack = "SSM" + }) +} + resource "kubernetes_manifest" "karpenter_node_class" { manifest = { apiVersion = "karpenter.k8s.aws/v1beta1"