From c10c3f6ff2755caf758ce2ee3a5edd33db4171db Mon Sep 17 00:00:00 2001
From: Michal Charemza <michal@charemza.name>
Date: Tue, 6 Aug 2024 09:03:57 +0100
Subject: [PATCH] feat: credentials for Airflow to accept dynamic/derived/SQL
 pipeline requests

We've configured Airflow to accept incoming requests from Data Workspace for
some cases of creating DAGs dynamically. These need credentials on the Airflow
side.
---
 infra/airflow_webserver.tf                         | 4 +++-
 infra/airflow_webserver_container_definitions.json | 6 ++++++
 infra/main.tf                                      | 8 ++++++++
 3 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/infra/airflow_webserver.tf b/infra/airflow_webserver.tf
index fed9fe1..2402846 100644
--- a/infra/airflow_webserver.tf
+++ b/infra/airflow_webserver.tf
@@ -137,7 +137,9 @@ resource "aws_ecs_task_definition" "airflow_webserver" {
 
       cloudwatch_log_group_arn = "${aws_cloudwatch_log_group.airflow_dag_tasks_airflow_logging[0].arn}"
 
-      dag_sync_github_key = "${var.dag_sync_github_key}"
+      dag_sync_github_key               = "${var.dag_sync_github_key}"
+      data_workspace_s3_import_hawk_id  = "${var.airflow_data_workspace_s3_import_hawk_id}"
+      data_workspace_s3_import_hawk_key = "${var.airflow_data_workspace_s3_import_hawk_key}"
     }
   )
   execution_role_arn       = aws_iam_role.airflow_webserver_execution[count.index].arn
diff --git a/infra/airflow_webserver_container_definitions.json b/infra/airflow_webserver_container_definitions.json
index 2b3d723..3924132 100644
--- a/infra/airflow_webserver_container_definitions.json
+++ b/infra/airflow_webserver_container_definitions.json
@@ -74,6 +74,12 @@
     },{
       "name": "DAG_SYNC_GITHUB_KEY",
       "value": "${dag_sync_github_key}"
+    },{
+      "name": "DATA_WORKSPACE_S3_IMPORT_HAWK_ID",
+      "value": "${data_workspace_s3_import_hawk_id}"
+    },{
+      "name": "DATA_WORKSPACE_S3_IMPORT_HAWK_KEY",
+      "value": "${data_workspace_s3_import_hawk_key}"
     }],
     "essential": true,
     "image": "${container_image}",
diff --git a/infra/main.tf b/infra/main.tf
index c35e834..843ba51 100644
--- a/infra/main.tf
+++ b/infra/main.tf
@@ -92,6 +92,14 @@ variable "sentry_environment" {}
 variable "airflow_authbroker_client_id" {}
 variable "airflow_authbroker_client_secret" {}
 variable "airflow_authbroker_url" {}
+variable "airflow_data_workspace_s3_import_hawk_id" {
+  type    = string
+  default = ""
+}
+variable "airflow_data_workspace_s3_import_hawk_key" {
+  type    = string
+  default = ""
+}
 
 variable "notebook_task_role_prefix" {}
 variable "notebook_task_role_policy_name" {}