From b61debd6b2ac2be05943fa10848787259ec9ad3e Mon Sep 17 00:00:00 2001 From: Bradley Lunsford Date: Thu, 16 May 2024 10:47:05 -0700 Subject: [PATCH 1/2] Modifying to recognized deployment & venue --- airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py b/airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py index 80b3ba7c..f670704b 100644 --- a/airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py +++ b/airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py @@ -10,6 +10,7 @@ from datetime import datetime import boto3 +from airflow.models import Variable from airflow.models.baseoperator import chain from airflow.models.param import Param from airflow.operators.python import PythonOperator @@ -30,7 +31,8 @@ # SSM keys for venue-dependent parameters UNITY_DAPA_CLIENT_ID = "/unity/shared-services/dapa/client-id" UNITY_DAPA_API_URL = "/unity/shared-services/dapa/api-url" -UNITY_OUTPUT_DATA_BUCKET = "/unity/ds/data/bucket/primary-data-bucket" +UNITY_IDENTIFIER = f"""{Variable.get("unity_deployment_name")}/{Variable.get("unity_venue")}-{Variable.get("unity_counter")}""" +UNITY_OUTPUT_DATA_BUCKET = f"/unity/ds/data/bucket/{UNITY_IDENTIFIER}/primary-data-bucket" # Resources needed by each Task # EC2 r6a.xlarge 4vCPU 32GiB From 44db833c08b9a43b1223a32db7b9dd59ad867e8c Mon Sep 17 00:00:00 2001 From: Bradley Lunsford Date: Sun, 16 Jun 2024 14:21:51 -0700 Subject: [PATCH 2/2] adding data bucket ssm param creation... guessing at value for now --- .../modules/terraform-unity-sps-airflow/main.tf | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/main.tf b/terraform-unity/modules/terraform-unity-sps-airflow/main.tf index 39009264..dd1e0c64 100644 --- a/terraform-unity/modules/terraform-unity-sps-airflow/main.tf +++ b/terraform-unity/modules/terraform-unity-sps-airflow/main.tf @@ -856,6 +856,18 @@ resource "aws_ssm_parameter" "ogc_processes_api_url" { }) } +resource "aws_ssm_parameter" "unity_output_data_bucket" { + name = format("/unity/ds/data/bucket/%s/%s-%s/primary-data-bucket", var.deployment_name, var.venue, var.counter) + description = "The S3 Address of the primary DS data bucket." + type = "String" + value = "sps-${var.venue}-ds-storage" + tags = merge(local.common_tags, { + Name = format(local.resource_name_prefix, "endpoints-output_data_bucket") + Component = "SSM" + Stack = "SSM" + }) +} + resource "kubernetes_manifest" "karpenter_node_class" { manifest = { apiVersion = "karpenter.k8s.aws/v1beta1"