diff --git a/airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py b/airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py index 0e07a7f7..b2ec7bbd 100644 --- a/airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py +++ b/airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py @@ -11,6 +11,7 @@ from datetime import datetime import boto3 +from airflow.models import Variable from airflow.models.baseoperator import chain from airflow.models.param import Param from airflow.operators.python import PythonOperator @@ -35,7 +36,8 @@ # SSM keys for venue-dependent parameters UNITY_DAPA_CLIENT_ID = "/unity/shared-services/dapa/client-id" UNITY_DAPA_API_URL = "/unity/shared-services/dapa/api-url" -UNITY_OUTPUT_DATA_BUCKET = "/unity/ds/data/bucket/primary-data-bucket" +UNITY_IDENTIFIER = f"""{Variable.get("unity_deployment_name")}/{Variable.get("unity_venue")}-{Variable.get("unity_counter")}""" +UNITY_OUTPUT_DATA_BUCKET = f"/unity/ds/data/bucket/{UNITY_IDENTIFIER}/primary-data-bucket" # Resources needed by each Task # EC2 r6a.xlarge 4vCPU 32GiB