Merge pull request #245 from unity-sds/59-marketplace-adjustments-edge

Bring forward SPS marketplace adjustments

airflow/helm/values.tmpl.yaml

@@ -250,6 +250,8 @@ config:
     encrypt_s3_logs: false
   celery:
     worker_concurrency: 16
+  webserver:
+    enable_proxy_fix: 'True'
 
 dags:
   persistence:

terraform-unity/README.md

@@ -191,7 +191,9 @@ terraform apply -no-color 2>&1 | tee apply_output.txt
 | <a name="input_airflow_webserver_password"></a> [airflow\_webserver\_password](#input\_airflow\_webserver\_password) | The password for the Airflow webserver and UI. | `string` | n/a | yes |
 | <a name="input_airflow_webserver_username"></a> [airflow\_webserver\_username](#input\_airflow\_webserver\_username) | The username for the Airflow webserver and UI. | `string` | `"admin"` | no |
 | <a name="input_dag_catalog_repo"></a> [dag\_catalog\_repo](#input\_dag\_catalog\_repo) | Git repository that stores the catalog of Airflow DAGs. | <pre>object({<br>    url                 = string<br>    ref                 = string<br>    dags_directory_path = string<br>  })</pre> | <pre>{<br>  "dags_directory_path": "airflow/dags",<br>  "ref": "2.2.0",<br>  "url": "https://github.com/unity-sds/unity-sps.git"<br>}</pre> | no |
+| <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | The name of the deployment. | `string` | `""` | no |
 | <a name="input_helm_charts"></a> [helm\_charts](#input\_helm\_charts) | Helm charts for the associated services. | <pre>map(object({<br>    repository = string<br>    chart      = string<br>    version    = string<br>  }))</pre> | <pre>{<br>  "airflow": {<br>    "chart": "airflow",<br>    "repository": "https://airflow.apache.org",<br>    "version": "1.15.0"<br>  },<br>  "keda": {<br>    "chart": "keda",<br>    "repository": "https://kedacore.github.io/charts",<br>    "version": "v2.15.1"<br>  }<br>}</pre> | no |
+| <a name="input_installprefix"></a> [installprefix](#input\_installprefix) | The install prefix for the service area (unused) | `string` | `""` | no |
 | <a name="input_karpenter_node_classes"></a> [karpenter\_node\_classes](#input\_karpenter\_node\_classes) | Configuration for karpenter\_node\_classes | <pre>map(object({<br>    volume_size = string<br>  }))</pre> | <pre>{<br>  "airflow-kubernetes-pod-operator-high-workload": {<br>    "volume_size": "300Gi"<br>  },<br>  "default": {<br>    "volume_size": "30Gi"<br>  }<br>}</pre> | no |
 | <a name="input_karpenter_node_pools"></a> [karpenter\_node\_pools](#input\_karpenter\_node\_pools) | Configuration for Karpenter node pools | <pre>map(object({<br>    requirements : list(object({<br>      key : string<br>      operator : string<br>      values : list(string)<br>    }))<br>    nodeClassRef : string<br>    limits : object({<br>      cpu : string<br>      memory : string<br>    })<br>    disruption : object({<br>      consolidationPolicy : string<br>      consolidateAfter : string<br>    })<br>  }))</pre> | <pre>{<br>  "airflow-celery-workers": {<br>    "disruption": {<br>      "consolidateAfter": "1m",<br>      "consolidationPolicy": "WhenEmpty"<br>    },<br>    "limits": {<br>      "cpu": "80",<br>      "memory": "320Gi"<br>    },<br>    "nodeClassRef": "default",<br>    "requirements": [<br>      {<br>        "key": "karpenter.k8s.aws/instance-family",<br>        "operator": "In",<br>        "values": [<br>          "m7i",<br>          "m6i",<br>          "m5",<br>          "t3",<br>          "c7i",<br>          "c6i",<br>          "c5",<br>          "r7i",<br>          "r6i",<br>          "r5"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Gt",<br>        "values": [<br>          "1"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Lt",<br>        "values": [<br>          "9"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Gt",<br>        "values": [<br>          "8191"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Lt",<br>        "values": [<br>          "32769"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-hypervisor",<br>        "operator": "In",<br>        "values": [<br>          "nitro"<br>        ]<br>      }<br>    ]<br>  },<br>  "airflow-core-components": {<br>    "disruption": {<br>      "consolidateAfter": "1m",<br>      "consolidationPolicy": "WhenEmpty"<br>    },<br>    "limits": {<br>      "cpu": "40",<br>      "memory": "160Gi"<br>    },<br>    "nodeClassRef": "default",<br>    "requirements": [<br>      {<br>        "key": "karpenter.k8s.aws/instance-family",<br>        "operator": "In",<br>        "values": [<br>          "m7i",<br>          "m6i",<br>          "m5",<br>          "t3",<br>          "c7i",<br>          "c6i",<br>          "c5",<br>          "r7i",<br>          "r6i",<br>          "r5"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Gt",<br>        "values": [<br>          "1"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Lt",<br>        "values": [<br>          "17"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Gt",<br>        "values": [<br>          "8191"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Lt",<br>        "values": [<br>          "32769"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-hypervisor",<br>        "operator": "In",<br>        "values": [<br>          "nitro"<br>        ]<br>      }<br>    ]<br>  },<br>  "airflow-kubernetes-pod-operator": {<br>    "disruption": {<br>      "consolidateAfter": "1m",<br>      "consolidationPolicy": "WhenEmpty"<br>    },<br>    "limits": {<br>      "cpu": "100",<br>      "memory": "400Gi"<br>    },<br>    "nodeClassRef": "default",<br>    "requirements": [<br>      {<br>        "key": "karpenter.k8s.aws/instance-family",<br>        "operator": "In",<br>        "values": [<br>          "m7i",<br>          "m6i",<br>          "m5",<br>          "t3",<br>          "c7i",<br>          "c6i",<br>          "c5",<br>          "r7i",<br>          "r6i",<br>          "r5"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Gt",<br>        "values": [<br>          "1"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Lt",<br>        "values": [<br>          "17"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Gt",<br>        "values": [<br>          "8191"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Lt",<br>        "values": [<br>          "32769"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-hypervisor",<br>        "operator": "In",<br>        "values": [<br>          "nitro"<br>        ]<br>      }<br>    ]<br>  },<br>  "airflow-kubernetes-pod-operator-high-workload": {<br>    "disruption": {<br>      "consolidateAfter": "1m",<br>      "consolidationPolicy": "WhenEmpty"<br>    },<br>    "limits": {<br>      "cpu": "528",<br>      "memory": "1056Gi"<br>    },<br>    "nodeClassRef": "airflow-kubernetes-pod-operator-high-workload",<br>    "requirements": [<br>      {<br>        "key": "karpenter.k8s.aws/instance-family",<br>        "operator": "In",<br>        "values": [<br>          "m7i",<br>          "m6i",<br>          "m5",<br>          "t3",<br>          "c7i",<br>          "c6i",<br>          "c5",<br>          "r7i",<br>          "r6i",<br>          "r5"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Gt",<br>        "values": [<br>          "1"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Lt",<br>        "values": [<br>          "49"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Gt",<br>        "values": [<br>          "8191"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Lt",<br>        "values": [<br>          "98305"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-hypervisor",<br>        "operator": "In",<br>        "values": [<br>          "nitro"<br>        ]<br>      }<br>    ]<br>  }<br>}</pre> | no |
 | <a name="input_kubeconfig_filepath"></a> [kubeconfig\_filepath](#input\_kubeconfig\_filepath) | The path to the kubeconfig file for the Kubernetes cluster. | `string` | n/a | yes |
@@ -200,6 +202,7 @@ terraform apply -no-color 2>&1 | tee apply_output.txt
 | <a name="input_project"></a> [project](#input\_project) | The project or mission deploying Unity SPS. | `string` | `"unity"` | no |
 | <a name="input_release"></a> [release](#input\_release) | The software release version. | `string` | `"24.3"` | no |
 | <a name="input_service_area"></a> [service\_area](#input\_service\_area) | The service area owner of the resources being deployed. | `string` | `"sps"` | no |
+| <a name="input_tags"></a> [tags](#input\_tags) | Tags for the deployment (unused) | `map(string)` | <pre>{<br>  "empty": ""<br>}</pre> | no |
 | <a name="input_venue"></a> [venue](#input\_venue) | The MCP venue in which the resources will be deployed. | `string` | n/a | yes |
 
 ## Outputs

terraform-unity/modules/terraform-unity-sps-airflow/README.md

@@ -69,6 +69,7 @@ No modules.
 | [kubernetes_storage_class.efs](https://registry.terraform.io/providers/hashicorp/kubernetes/2.32.0/docs/resources/storage_class) | resource |
 | [null_resource.remove_keda_finalizers](https://registry.terraform.io/providers/hashicorp/null/3.2.3/docs/resources/resource) | resource |
 | [random_id.airflow_webserver_secret](https://registry.terraform.io/providers/hashicorp/random/3.6.1/docs/resources/id) | resource |
+| [time_sleep.wait_after_ssm](https://registry.terraform.io/providers/hashicorp/time/0.12.1/docs/resources/sleep) | resource |
 | [time_sleep.wait_for_efs_mount_target_dns_propagation](https://registry.terraform.io/providers/hashicorp/time/0.12.1/docs/resources/sleep) | resource |
 | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/5.67.0/docs/data-sources/caller_identity) | data source |
 | [aws_db_instance.db](https://registry.terraform.io/providers/hashicorp/aws/5.67.0/docs/data-sources/db_instance) | data source |

terraform-unity/modules/terraform-unity-sps-airflow/main.tf

@@ -635,7 +635,7 @@ resource "aws_ssm_parameter" "unity_proxy_airflow_ui" {
       Redirect "/${var.project}/${var.venue}/sps/home"
     </Location>
     <LocationMatch "^/${var.project}/${var.venue}/sps/(.*)$">
-      ProxyPassMatch "http://${data.kubernetes_ingress_v1.airflow_ingress_internal.status[0].load_balancer[0].ingress[0].hostname}:5000/$1"
+      ProxyPassMatch "http://${data.kubernetes_ingress_v1.airflow_ingress_internal.status[0].load_balancer[0].ingress[0].hostname}:5000/$1" retry=5 disablereuse=On
       ProxyPreserveHost On
       FallbackResource /management/index.html
       AddOutputFilterByType INFLATE;SUBSTITUTE;DEFLATE text/html
@@ -653,8 +653,8 @@ EOT
 data "aws_lambda_functions" "lambda_check_all" {}
 
 resource "aws_lambda_invocation" "unity_proxy_lambda_invocation" {
-  count         = contains(data.aws_lambda_functions.lambda_check_all.function_names, "unity-${var.venue}-httpdproxymanagement") ? 1 : 0
-  function_name = "unity-${var.venue}-httpdproxymanagement"
+  count         = contains(data.aws_lambda_functions.lambda_check_all.function_names, "${var.project}-${var.venue}-httpdproxymanagement") ? 1 : 0
+  function_name = "${var.project}-${var.venue}-httpdproxymanagement"
   input         = "{}"
   triggers = {
     redeployment = sha1(jsonencode([

terraform-unity/modules/terraform-unity-sps-eks/README.md

@@ -18,7 +18,7 @@
 
 | Name | Source | Version |
 |------|--------|---------|
-| <a name="module_unity-eks"></a> [unity-eks](#module\_unity-eks) | git@github.com:unity-sds/unity-cs-infra.git//terraform-unity-eks_module | unity-sps-2.2.0-hotfix |
+| <a name="module_unity-eks"></a> [unity-eks](#module\_unity-eks) | git::https://github.com/unity-sds/unity-cs-infra.git//terraform-unity-eks_module | 59-sps-eks-marketplace-adjustments |
 
 ## Resources
 
@@ -31,10 +31,13 @@
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
+| <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | The name of the deployment. | `string` | n/a | yes |
+| <a name="input_installprefix"></a> [installprefix](#input\_installprefix) | The install prefix for the service area (unused) | `string` | `""` | no |
 | <a name="input_nodegroups"></a> [nodegroups](#input\_nodegroups) | A map of node group configurations | <pre>map(object({<br>    create_iam_role            = optional(bool)<br>    iam_role_arn               = optional(string)<br>    ami_id                     = optional(string)<br>    min_size                   = optional(number)<br>    max_size                   = optional(number)<br>    desired_size               = optional(number)<br>    instance_types             = optional(list(string))<br>    capacity_type              = optional(string)<br>    enable_bootstrap_user_data = optional(bool)<br>    metadata_options           = optional(map(any))<br>    block_device_mappings = optional(map(object({<br>      device_name = string<br>      ebs = object({<br>        volume_size           = number<br>        volume_type           = string<br>        encrypted             = bool<br>        delete_on_termination = bool<br>      })<br>    })))<br>  }))</pre> | <pre>{<br>  "defaultGroup": {<br>    "block_device_mappings": {<br>      "xvda": {<br>        "device_name": "/dev/xvda",<br>        "ebs": {<br>          "delete_on_termination": true,<br>          "encrypted": true,<br>          "volume_size": 100,<br>          "volume_type": "gp2"<br>        }<br>      }<br>    },<br>    "desired_size": 1,<br>    "instance_types": [<br>      "t3.xlarge"<br>    ],<br>    "max_size": 1,<br>    "metadata_options": {<br>      "http_endpoint": "enabled",<br>      "http_put_response_hop_limit": 3<br>    },<br>    "min_size": 1<br>  }<br>}</pre> | no |
 | <a name="input_project"></a> [project](#input\_project) | The project or mission deploying Unity SPS | `string` | `"unity"` | no |
 | <a name="input_release"></a> [release](#input\_release) | The software release version. | `string` | `"24.3"` | no |
 | <a name="input_service_area"></a> [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | `"sps"` | no |
+| <a name="input_tags"></a> [tags](#input\_tags) | Tags for the deployment (unused) | `map(string)` | <pre>{<br>  "empty": ""<br>}</pre> | no |
 | <a name="input_venue"></a> [venue](#input\_venue) | The MCP venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes |
 
 ## Outputs

terraform-unity/modules/terraform-unity-sps-eks/main.tf

@@ -9,8 +9,10 @@ terraform {
 }
 
 module "unity-eks" {
-  source          = "git@github.com:unity-sds/unity-cs-infra.git//terraform-unity-eks_module?ref=unity-sps-2.2.0-hotfix"
+  source          = "git::https://github.com/unity-sds/unity-cs-infra.git//terraform-unity-eks_module?ref=unity-sps-2.4.0"
   deployment_name = local.cluster_name
+  project         = var.project
+  venue           = var.venue
   nodegroups      = var.nodegroups
   aws_auth_roles = [{
     rolearn  = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/mcp-tenantOperator"

terraform-unity/modules/terraform-unity-sps-eks/variables.tf

@@ -21,6 +21,26 @@ variable "release" {
   default     = "24.3"
 }
 
+# tflint-ignore: terraform_unused_declarations
+variable "deployment_name" {
+  description = "The name of the deployment."
+  type        = string
+}
+
+# tflint-ignore: terraform_unused_declarations
+variable "tags" {
+  description = "Tags for the deployment (unused)"
+  type        = map(string)
+  default     = { empty = "" }
+}
+
+# tflint-ignore: terraform_unused_declarations
+variable "installprefix" {
+  description = "The install prefix for the service area (unused)"
+  type        = string
+  default     = ""
+}
+
 variable "nodegroups" {
   description = "A map of node group configurations"
   type = map(object({

terraform-unity/modules/terraform-unity-sps-initiators/main.tf

@@ -78,7 +78,7 @@ resource "aws_s3_object" "router_config" {
 }
 
 module "unity_initiator" {
-  source        = "git@github.com:unity-sds/unity-initiator.git//terraform-unity/initiator?ref=unity-sps-2.2.0"
+  source        = "git::https://github.com/unity-sds/unity-initiator.git//terraform-unity/initiator?ref=unity-sps-2.2.0"
   code_bucket   = aws_s3_bucket.code.id
   project       = var.project
   router_config = "s3://${aws_s3_bucket.config.id}/${aws_s3_object.router_config.key}"
@@ -91,7 +91,7 @@ resource "aws_s3_object" "isl_stacam_rawdp_folder" {
 }
 
 module "s3_bucket_notification" {
-  source              = "git@github.com:unity-sds/unity-initiator.git//terraform-unity/triggers/s3-bucket-notification?ref=unity-sps-2.2.0"
+  source              = "git::https://github.com/unity-sds/unity-initiator.git//terraform-unity/triggers/s3-bucket-notification?ref=unity-sps-2.2.0"
   initiator_topic_arn = module.unity_initiator.initiator_topic_arn
   isl_bucket          = aws_s3_bucket.inbound_staging_location.id
   isl_bucket_prefix   = "STACAM/RawDP/"

terraform-unity/modules/terraform-unity-sps-karpenter/README.md

@@ -35,10 +35,13 @@
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
+| <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | The name of the deployment. | `string` | n/a | yes |
 | <a name="input_helm_charts"></a> [helm\_charts](#input\_helm\_charts) | Helm charts for the associated services. | <pre>map(object({<br>    repository = string<br>    chart      = string<br>    version    = string<br>  }))</pre> | <pre>{<br>  "karpenter": {<br>    "chart": "karpenter",<br>    "repository": "oci://public.ecr.aws/karpenter",<br>    "version": "1.0.2"<br>  }<br>}</pre> | no |
+| <a name="input_installprefix"></a> [installprefix](#input\_installprefix) | The install prefix for the service area (unused) | `string` | `""` | no |
 | <a name="input_project"></a> [project](#input\_project) | The project or mission deploying Unity SPS | `string` | `"unity"` | no |
 | <a name="input_release"></a> [release](#input\_release) | The software release version. | `string` | `"24.3"` | no |
 | <a name="input_service_area"></a> [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | `"sps"` | no |
+| <a name="input_tags"></a> [tags](#input\_tags) | Tags for the deployment (unused) | `map(string)` | <pre>{<br>  "empty": ""<br>}</pre> | no |
 | <a name="input_venue"></a> [venue](#input\_venue) | The MCP venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes |
 
 ## Outputs

terraform-unity/modules/terraform-unity-sps-karpenter/variables.tf

@@ -21,6 +21,26 @@ variable "release" {
   default     = "24.3"
 }
 
+# tflint-ignore: terraform_unused_declarations
+variable "deployment_name" {
+  description = "The name of the deployment."
+  type        = string
+}
+
+# tflint-ignore: terraform_unused_declarations
+variable "tags" {
+  description = "Tags for the deployment (unused)"
+  type        = map(string)
+  default     = { empty = "" }
+}
+
+# tflint-ignore: terraform_unused_declarations
+variable "installprefix" {
+  description = "The install prefix for the service area (unused)"
+  type        = string
+  default     = ""
+}
+
 variable "helm_charts" {
   description = "Helm charts for the associated services."
   type = map(object({