feat: credentials for Airflow to accept dynamic/derived/SQL pipeline …

…requests

We've configured Airflow to accept incoming requests from Data Workspace for
some cases of creating DAGs dynamically. These need credentials on the Airflow
side.

infra/airflow_webserver.tf

@@ -137,7 +137,9 @@ resource "aws_ecs_task_definition" "airflow_webserver" {
 
       cloudwatch_log_group_arn = "${aws_cloudwatch_log_group.airflow_dag_tasks_airflow_logging[0].arn}"
 
-      dag_sync_github_key = "${var.dag_sync_github_key}"
+      dag_sync_github_key               = "${var.dag_sync_github_key}"
+      data_workspace_s3_import_hawk_id  = "${var.airflow_data_workspace_s3_import_hawk_id}"
+      data_workspace_s3_import_hawk_key = "${var.airflow_data_workspace_s3_import_hawk_key}"
     }
   )
   execution_role_arn       = aws_iam_role.airflow_webserver_execution[count.index].arn

infra/airflow_webserver_container_definitions.json

@@ -74,6 +74,12 @@
     },{
       "name": "DAG_SYNC_GITHUB_KEY",
       "value": "${dag_sync_github_key}"
+    },{
+      "name": "DATA_WORKSPACE_S3_IMPORT_HAWK_ID",
+      "value": "${data_workspace_s3_import_hawk_id}"
+    },{
+      "name": "DATA_WORKSPACE_S3_IMPORT_HAWK_KEY",
+      "value": "${data_workspace_s3_import_hawk_key}"
     }],
     "essential": true,
     "image": "${container_image}",

infra/main.tf

@@ -92,6 +92,14 @@ variable "sentry_environment" {}
 variable "airflow_authbroker_client_id" {}
 variable "airflow_authbroker_client_secret" {}
 variable "airflow_authbroker_url" {}
+variable "airflow_data_workspace_s3_import_hawk_id" {
+  type    = string
+  default = ""
+}
+variable "airflow_data_workspace_s3_import_hawk_key" {
+  type    = string
+  default = ""
+}
 
 variable "notebook_task_role_prefix" {}
 variable "notebook_task_role_policy_name" {}