From 51b99e8627cd744a0e8e31c738b8bb07b58aaf45 Mon Sep 17 00:00:00 2001 From: Bradley Lunsford Date: Mon, 23 Sep 2024 11:17:54 -0700 Subject: [PATCH 1/3] Adding additional internal alb/ingress for access by venue-services proxy --- .../terraform-unity-sps-airflow/README.md | 3 ++ .../terraform-unity-sps-airflow/data.tf | 7 +++ .../terraform-unity-sps-airflow/main.tf | 52 ++++++++++++++++++- .../README.md | 3 ++ .../data.tf | 7 +++ .../main.tf | 50 +++++++++++++++++- 6 files changed, 118 insertions(+), 4 deletions(-) diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/README.md b/terraform-unity/modules/terraform-unity-sps-airflow/README.md index 2314b4b8..8b9ff07d 100644 --- a/terraform-unity/modules/terraform-unity-sps-airflow/README.md +++ b/terraform-unity/modules/terraform-unity-sps-airflow/README.md @@ -42,6 +42,7 @@ No modules. | [aws_s3_bucket.airflow_logs](https://registry.terraform.io/providers/hashicorp/aws/5.50.0/docs/resources/s3_bucket) | resource | | [aws_security_group.airflow_efs](https://registry.terraform.io/providers/hashicorp/aws/5.50.0/docs/resources/security_group) | resource | | [aws_security_group.airflow_ingress_sg](https://registry.terraform.io/providers/hashicorp/aws/5.50.0/docs/resources/security_group) | resource | +| [aws_security_group.airflow_ingress_sg_internal](https://registry.terraform.io/providers/hashicorp/aws/5.50.0/docs/resources/security_group) | resource | | [aws_security_group_rule.airflow_efs](https://registry.terraform.io/providers/hashicorp/aws/5.50.0/docs/resources/security_group_rule) | resource | | [aws_ssm_parameter.airflow_api_health_check_endpoint](https://registry.terraform.io/providers/hashicorp/aws/5.50.0/docs/resources/ssm_parameter) | resource | | [aws_ssm_parameter.airflow_api_url](https://registry.terraform.io/providers/hashicorp/aws/5.50.0/docs/resources/ssm_parameter) | resource | @@ -54,6 +55,7 @@ No modules. | [helm_release.airflow](https://registry.terraform.io/providers/hashicorp/helm/2.13.1/docs/resources/release) | resource | | [helm_release.keda](https://registry.terraform.io/providers/hashicorp/helm/2.13.1/docs/resources/release) | resource | | [kubernetes_ingress_v1.airflow_ingress](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/ingress_v1) | resource | +| [kubernetes_ingress_v1.airflow_ingress_internal](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/ingress_v1) | resource | | [kubernetes_namespace.keda](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/namespace) | resource | | [kubernetes_persistent_volume.airflow_deployed_dags](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/persistent_volume) | resource | | [kubernetes_persistent_volume.airflow_kpo](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/persistent_volume) | resource | @@ -77,6 +79,7 @@ No modules. | [aws_ssm_parameter.subnet_ids](https://registry.terraform.io/providers/hashicorp/aws/5.50.0/docs/data-sources/ssm_parameter) | data source | | [aws_vpc.cluster_vpc](https://registry.terraform.io/providers/hashicorp/aws/5.50.0/docs/data-sources/vpc) | data source | | [kubernetes_ingress_v1.airflow_ingress](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/data-sources/ingress_v1) | data source | +| [kubernetes_ingress_v1.airflow_ingress_internal](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/data-sources/ingress_v1) | data source | | [kubernetes_namespace.service_area](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/data-sources/namespace) | data source | ## Inputs diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/data.tf b/terraform-unity/modules/terraform-unity-sps-airflow/data.tf index 5584e16a..eb80736a 100644 --- a/terraform-unity/modules/terraform-unity-sps-airflow/data.tf +++ b/terraform-unity/modules/terraform-unity-sps-airflow/data.tf @@ -25,6 +25,13 @@ data "kubernetes_ingress_v1" "airflow_ingress" { } } +data "kubernetes_ingress_v1" "airflow_ingress_internal" { + metadata { + name = kubernetes_ingress_v1.airflow_ingress_internal.metadata[0].name + namespace = data.kubernetes_namespace.service_area.metadata[0].name + } +} + data "aws_db_instance" "db" { db_instance_identifier = var.db_instance_identifier } diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/main.tf b/terraform-unity/modules/terraform-unity-sps-airflow/main.tf index 6bfa1810..d213d944 100644 --- a/terraform-unity/modules/terraform-unity-sps-airflow/main.tf +++ b/terraform-unity/modules/terraform-unity-sps-airflow/main.tf @@ -414,6 +414,17 @@ resource "aws_security_group" "airflow_ingress_sg" { }) } +resource "aws_security_group" "airflow_ingress_sg_internal" { + name = "${var.project}-${var.venue}-airflow-internal-ingress-sg" + description = "SecurityGroup for Airflow LoadBalancer internal ingress" + vpc_id = data.aws_vpc.cluster_vpc.id + tags = merge(local.common_tags, { + Name = format(local.resource_name_prefix, "AirflowLBSg") + Component = "airflow" + Stack = "airflow" + }) +} + #tfsec:ignore:AVD-AWS-0107 resource "aws_vpc_security_group_ingress_rule" "airflow_ingress_sg_jpl_rule" { for_each = toset(["128.149.0.0/16", "137.78.0.0/16", "137.79.0.0/16"]) @@ -437,7 +448,7 @@ data "aws_security_groups" "venue_proxy_sg" { resource "aws_vpc_security_group_ingress_rule" "airflow_ingress_sg_proxy_rule" { count = length(data.aws_security_groups.venue_proxy_sg.ids) > 0 ? 1 : 0 - security_group_id = aws_security_group.airflow_ingress_sg.id + security_group_id = aws_security_group.airflow_ingress_sg_internal.id description = "SecurityGroup ingress rule for venue-services proxy" ip_protocol = "tcp" from_port = local.load_balancer_port @@ -482,6 +493,43 @@ resource "kubernetes_ingress_v1" "airflow_ingress" { depends_on = [helm_release.airflow] } +resource "kubernetes_ingress_v1" "airflow_ingress_internal" { + metadata { + name = "airflow-ingress" + namespace = data.kubernetes_namespace.service_area.metadata[0].name + annotations = { + "alb.ingress.kubernetes.io/scheme" = "internal" + "alb.ingress.kubernetes.io/target-type" = "ip" + "alb.ingress.kubernetes.io/subnets" = join(",", jsondecode(data.aws_ssm_parameter.subnet_ids.value)["private"]) + "alb.ingress.kubernetes.io/listen-ports" = "[{\"HTTP\": ${local.load_balancer_port}}]" + "alb.ingress.kubernetes.io/security-groups" = aws_security_group.airflow_ingress_sg_internal.id + "alb.ingress.kubernetes.io/manage-backend-security-group-rules" = "true" + "alb.ingress.kubernetes.io/healthcheck-path" = "/health" + } + } + spec { + ingress_class_name = "alb" + rule { + http { + path { + path = "/" + path_type = "Prefix" + backend { + service { + name = "airflow-webserver" + port { + number = 8080 + } + } + } + } + } + } + } + wait_for_load_balancer = true + depends_on = [helm_release.airflow] +} + resource "aws_ssm_parameter" "airflow_ui_url" { name = format("/%s", join("/", compact(["", var.project, var.venue, var.service_area, "processing", "airflow", "ui_url"]))) description = "The URL of the Airflow UI." @@ -557,7 +605,7 @@ resource "aws_ssm_parameter" "unity_proxy_airflow_ui" { Redirect "/${var.project}/${var.venue}/sps/home" - ProxyPassMatch "http://${data.kubernetes_ingress_v1.airflow_ingress.status[0].load_balancer[0].ingress[0].hostname}:5000/$1" + ProxyPassMatch "http://${data.kubernetes_ingress_v1.airflow_ingress_internal.status[0].load_balancer[0].ingress[0].hostname}:5000/$1" ProxyPreserveHost On FallbackResource /management/index.html AddOutputFilterByType INFLATE;SUBSTITUTE;DEFLATE text/html diff --git a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/README.md b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/README.md index 05e6521c..ee4d572a 100644 --- a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/README.md +++ b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/README.md @@ -24,6 +24,7 @@ No modules. |------|------| | [aws_lambda_invocation.unity_proxy_lambda_invocation](https://registry.terraform.io/providers/hashicorp/aws/5.50.0/docs/resources/lambda_invocation) | resource | | [aws_security_group.ogc_ingress_sg](https://registry.terraform.io/providers/hashicorp/aws/5.50.0/docs/resources/security_group) | resource | +| [aws_security_group.ogc_ingress_sg_internal](https://registry.terraform.io/providers/hashicorp/aws/5.50.0/docs/resources/security_group) | resource | | [aws_ssm_parameter.ogc_processes_api_health_check_endpoint](https://registry.terraform.io/providers/hashicorp/aws/5.50.0/docs/resources/ssm_parameter) | resource | | [aws_ssm_parameter.ogc_processes_api_url](https://registry.terraform.io/providers/hashicorp/aws/5.50.0/docs/resources/ssm_parameter) | resource | | [aws_ssm_parameter.ogc_processes_ui_url](https://registry.terraform.io/providers/hashicorp/aws/5.50.0/docs/resources/ssm_parameter) | resource | @@ -33,6 +34,7 @@ No modules. | [kubernetes_deployment.ogc_processes_api](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/deployment) | resource | | [kubernetes_deployment.redis](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/deployment) | resource | | [kubernetes_ingress_v1.ogc_processes_api_ingress](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/ingress_v1) | resource | +| [kubernetes_ingress_v1.ogc_processes_api_ingress_internal](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/ingress_v1) | resource | | [kubernetes_service.ogc_processes_api](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/service) | resource | | [kubernetes_service.redis](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/service) | resource | | [aws_db_instance.db](https://registry.terraform.io/providers/hashicorp/aws/5.50.0/docs/data-sources/db_instance) | data source | @@ -43,6 +45,7 @@ No modules. | [aws_ssm_parameter.subnet_ids](https://registry.terraform.io/providers/hashicorp/aws/5.50.0/docs/data-sources/ssm_parameter) | data source | | [aws_vpc.cluster_vpc](https://registry.terraform.io/providers/hashicorp/aws/5.50.0/docs/data-sources/vpc) | data source | | [kubernetes_ingress_v1.ogc_processes_api_ingress](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/data-sources/ingress_v1) | data source | +| [kubernetes_ingress_v1.ogc_processes_api_ingress_internal](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/data-sources/ingress_v1) | data source | | [kubernetes_namespace.service_area](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/data-sources/namespace) | data source | | [kubernetes_persistent_volume_claim.airflow_deployed_dags](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/data-sources/persistent_volume_claim) | data source | diff --git a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/data.tf b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/data.tf index 0a9922b8..532e57f5 100644 --- a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/data.tf +++ b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/data.tf @@ -36,3 +36,10 @@ data "kubernetes_ingress_v1" "ogc_processes_api_ingress" { namespace = data.kubernetes_namespace.service_area.metadata[0].name } } + +data "kubernetes_ingress_v1" "ogc_processes_api_ingress_internal" { + metadata { + name = kubernetes_ingress_v1.ogc_processes_api_ingress_internal.metadata[0].name + namespace = data.kubernetes_namespace.service_area.metadata[0].name + } +} diff --git a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf index 48adbd4e..8756e24e 100644 --- a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf +++ b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf @@ -217,6 +217,17 @@ resource "aws_security_group" "ogc_ingress_sg" { }) } +resource "aws_security_group" "ogc_ingress_sg_internal" { + name = "${var.project}-${var.venue}-ogc-internal-ingress-sg" + description = "SecurityGroup for OGC API LoadBalancer internal ingress" + vpc_id = data.aws_vpc.cluster_vpc.id + tags = merge(local.common_tags, { + Name = format(local.resource_name_prefix, "OgcLBSg") + Component = "ogc" + Stack = "ogc" + }) +} + #tfsec:ignore:AVD-AWS-0107 resource "aws_vpc_security_group_ingress_rule" "ogc_ingress_sg_jpl_rule" { for_each = toset(["128.149.0.0/16", "137.78.0.0/16", "137.79.0.0/16"]) @@ -240,7 +251,7 @@ data "aws_security_groups" "venue_proxy_sg" { resource "aws_vpc_security_group_ingress_rule" "ogc_ingress_sg_proxy_rule" { count = length(data.aws_security_groups.venue_proxy_sg.ids) > 0 ? 1 : 0 - security_group_id = aws_security_group.ogc_ingress_sg.id + security_group_id = aws_security_group.ogc_ingress_sg_internal.id description = "SecurityGroup ingress rule for venue-services proxy" ip_protocol = "tcp" from_port = local.load_balancer_port @@ -284,6 +295,41 @@ resource "kubernetes_ingress_v1" "ogc_processes_api_ingress" { wait_for_load_balancer = true } +resource "kubernetes_ingress_v1" "ogc_processes_api_ingress_internal" { + metadata { + name = "ogc-processes-api-ingress" + namespace = data.kubernetes_namespace.service_area.metadata[0].name + annotations = { + "alb.ingress.kubernetes.io/scheme" = "internal" + "alb.ingress.kubernetes.io/target-type" = "ip" + "alb.ingress.kubernetes.io/subnets" = join(",", jsondecode(data.aws_ssm_parameter.subnet_ids.value)["private"]) + "alb.ingress.kubernetes.io/listen-ports" = "[{\"HTTP\": ${local.load_balancer_port}}]" + "alb.ingress.kubernetes.io/security-groups" = aws_security_group.ogc_ingress_sg_internal.id + "alb.ingress.kubernetes.io/manage-backend-security-group-rules" = "true" + "alb.ingress.kubernetes.io/healthcheck-path" = "/health" + } + } + spec { + ingress_class_name = "alb" + rule { + http { + path { + path = "/" + path_type = "Prefix" + backend { + service { + name = kubernetes_service.ogc_processes_api.metadata[0].name + port { + number = 80 + } + } + } + } + } + } + } + wait_for_load_balancer = true +} resource "aws_ssm_parameter" "ogc_processes_ui_url" { name = format("/%s", join("/", compact(["", var.project, var.venue, var.service_area, "processing", "ogc_processes", "ui_url"]))) @@ -338,7 +384,7 @@ resource "aws_ssm_parameter" "unity_proxy_ogc_api" { ProxyPassReverse "/" - ProxyPassMatch "http://${data.kubernetes_ingress_v1.ogc_processes_api_ingress.status[0].load_balancer[0].ingress[0].hostname}:5001/$1" + ProxyPassMatch "http://${data.kubernetes_ingress_v1.ogc_processes_api_ingress_internal.status[0].load_balancer[0].ingress[0].hostname}:5001/$1" ProxyPreserveHost On FallbackResource /management/index.html AddOutputFilterByType INFLATE;SUBSTITUTE;DEFLATE text/html From fb7c967e7c1b22ce99d48d769f7397ac8d6d7522 Mon Sep 17 00:00:00 2001 From: Bradley Lunsford Date: Mon, 23 Sep 2024 11:53:59 -0700 Subject: [PATCH 2/3] fix name collision --- terraform-unity/modules/terraform-unity-sps-airflow/main.tf | 2 +- .../modules/terraform-unity-sps-ogc-processes-api/main.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/main.tf b/terraform-unity/modules/terraform-unity-sps-airflow/main.tf index d213d944..c9b100b8 100644 --- a/terraform-unity/modules/terraform-unity-sps-airflow/main.tf +++ b/terraform-unity/modules/terraform-unity-sps-airflow/main.tf @@ -495,7 +495,7 @@ resource "kubernetes_ingress_v1" "airflow_ingress" { resource "kubernetes_ingress_v1" "airflow_ingress_internal" { metadata { - name = "airflow-ingress" + name = "airflow-ingress-internal" namespace = data.kubernetes_namespace.service_area.metadata[0].name annotations = { "alb.ingress.kubernetes.io/scheme" = "internal" diff --git a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf index 8756e24e..671d2e1b 100644 --- a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf +++ b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf @@ -297,7 +297,7 @@ resource "kubernetes_ingress_v1" "ogc_processes_api_ingress" { resource "kubernetes_ingress_v1" "ogc_processes_api_ingress_internal" { metadata { - name = "ogc-processes-api-ingress" + name = "ogc-processes-api-ingress-internal" namespace = data.kubernetes_namespace.service_area.metadata[0].name annotations = { "alb.ingress.kubernetes.io/scheme" = "internal" From 9676edc67a9f96c2927487ea41f64037e61f6e10 Mon Sep 17 00:00:00 2001 From: Bradley Lunsford Date: Mon, 23 Sep 2024 13:39:20 -0700 Subject: [PATCH 3/3] updating internal service endpoints --- .../modules/terraform-unity-sps-airflow/main.tf | 8 ++++---- .../modules/terraform-unity-sps-ogc-processes-api/main.tf | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/main.tf b/terraform-unity/modules/terraform-unity-sps-airflow/main.tf index c9b100b8..1c9ef2d5 100644 --- a/terraform-unity/modules/terraform-unity-sps-airflow/main.tf +++ b/terraform-unity/modules/terraform-unity-sps-airflow/main.tf @@ -548,8 +548,8 @@ resource "aws_ssm_parameter" "airflow_ui_health_check_endpoint" { type = "String" value = jsonencode({ "componentName" : "Airflow UI" - "healthCheckUrl" : "http://${data.kubernetes_ingress_v1.airflow_ingress.status[0].load_balancer[0].ingress[0].hostname}:5000/health" - "landingPageUrl" : "http://${data.kubernetes_ingress_v1.airflow_ingress.status[0].load_balancer[0].ingress[0].hostname}:5000" + "healthCheckUrl" : "http://${data.kubernetes_ingress_v1.airflow_ingress_internal.status[0].load_balancer[0].ingress[0].hostname}:5000/health" + "landingPageUrl" : "http://${data.kubernetes_ingress_v1.airflow_ingress_internal.status[0].load_balancer[0].ingress[0].hostname}:5000" }) tags = merge(local.common_tags, { Name = format(local.resource_name_prefix, "health-check-endpoints-airflow_ui") @@ -579,8 +579,8 @@ resource "aws_ssm_parameter" "airflow_api_health_check_endpoint" { type = "String" value = jsonencode({ "componentName" : "Airflow API" - "healthCheckUrl" : "http://${data.kubernetes_ingress_v1.airflow_ingress.status[0].load_balancer[0].ingress[0].hostname}:5000/api/v1/health" - "landingPageUrl" : "http://${data.kubernetes_ingress_v1.airflow_ingress.status[0].load_balancer[0].ingress[0].hostname}:5000/api/v1" + "healthCheckUrl" : "http://${data.kubernetes_ingress_v1.airflow_ingress_internal.status[0].load_balancer[0].ingress[0].hostname}:5000/api/v1/health" + "landingPageUrl" : "http://${data.kubernetes_ingress_v1.airflow_ingress_internal.status[0].load_balancer[0].ingress[0].hostname}:5000/api/v1" }) tags = merge(local.common_tags, { Name = format(local.resource_name_prefix, "health-check-endpoints-airflow_api") diff --git a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf index 671d2e1b..41974460 100644 --- a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf +++ b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf @@ -361,8 +361,8 @@ resource "aws_ssm_parameter" "ogc_processes_api_health_check_endpoint" { type = "String" value = jsonencode({ "componentName" : "OGC API" - "healthCheckUrl" : "http://${data.kubernetes_ingress_v1.ogc_processes_api_ingress.status[0].load_balancer[0].ingress[0].hostname}:5001/health" - "landingPageUrl" : "http://${data.kubernetes_ingress_v1.ogc_processes_api_ingress.status[0].load_balancer[0].ingress[0].hostname}:5001" + "healthCheckUrl" : "http://${data.kubernetes_ingress_v1.ogc_processes_api_ingress_internal.status[0].load_balancer[0].ingress[0].hostname}:5001/health" + "landingPageUrl" : "http://${data.kubernetes_ingress_v1.ogc_processes_api_ingress_internal.status[0].load_balancer[0].ingress[0].hostname}:5001" }) tags = merge(local.common_tags, { Name = format(local.resource_name_prefix, "health-check-endpoints-ogc_processes_api")