diff --git a/airflow/helm/values.tmpl.yaml b/airflow/helm/values.tmpl.yaml
index 449b57a2..d87fa1f9 100644
--- a/airflow/helm/values.tmpl.yaml
+++ b/airflow/helm/values.tmpl.yaml
@@ -250,6 +250,8 @@ config:
encrypt_s3_logs: false
celery:
worker_concurrency: 16
+ webserver:
+ enable_proxy_fix: 'True'
dags:
persistence:
diff --git a/terraform-unity/README.md b/terraform-unity/README.md
index 419b6baa..742bfdab 100644
--- a/terraform-unity/README.md
+++ b/terraform-unity/README.md
@@ -191,7 +191,9 @@ terraform apply -no-color 2>&1 | tee apply_output.txt
| [airflow\_webserver\_password](#input\_airflow\_webserver\_password) | The password for the Airflow webserver and UI. | `string` | n/a | yes |
| [airflow\_webserver\_username](#input\_airflow\_webserver\_username) | The username for the Airflow webserver and UI. | `string` | `"admin"` | no |
| [dag\_catalog\_repo](#input\_dag\_catalog\_repo) | Git repository that stores the catalog of Airflow DAGs. | object({
url = string
ref = string
dags_directory_path = string
}) | {
"dags_directory_path": "airflow/dags",
"ref": "2.2.0",
"url": "https://github.com/unity-sds/unity-sps.git"
} | no |
+| [deployment\_name](#input\_deployment\_name) | The name of the deployment. | `string` | `""` | no |
| [helm\_charts](#input\_helm\_charts) | Helm charts for the associated services. | map(object({
repository = string
chart = string
version = string
})) | {
"airflow": {
"chart": "airflow",
"repository": "https://airflow.apache.org",
"version": "1.15.0"
},
"keda": {
"chart": "keda",
"repository": "https://kedacore.github.io/charts",
"version": "v2.15.1"
}
} | no |
+| [installprefix](#input\_installprefix) | The install prefix for the service area (unused) | `string` | `""` | no |
| [karpenter\_node\_classes](#input\_karpenter\_node\_classes) | Configuration for karpenter\_node\_classes | map(object({
volume_size = string
})) | {
"airflow-kubernetes-pod-operator-high-workload": {
"volume_size": "300Gi"
},
"default": {
"volume_size": "30Gi"
}
} | no |
| [karpenter\_node\_pools](#input\_karpenter\_node\_pools) | Configuration for Karpenter node pools | map(object({
requirements : list(object({
key : string
operator : string
values : list(string)
}))
nodeClassRef : string
limits : object({
cpu : string
memory : string
})
disruption : object({
consolidationPolicy : string
consolidateAfter : string
})
})) | {
"airflow-celery-workers": {
"disruption": {
"consolidateAfter": "1m",
"consolidationPolicy": "WhenEmpty"
},
"limits": {
"cpu": "80",
"memory": "320Gi"
},
"nodeClassRef": "default",
"requirements": [
{
"key": "karpenter.k8s.aws/instance-family",
"operator": "In",
"values": [
"m7i",
"m6i",
"m5",
"t3",
"c7i",
"c6i",
"c5",
"r7i",
"r6i",
"r5"
]
},
{
"key": "karpenter.k8s.aws/instance-cpu",
"operator": "Gt",
"values": [
"1"
]
},
{
"key": "karpenter.k8s.aws/instance-cpu",
"operator": "Lt",
"values": [
"9"
]
},
{
"key": "karpenter.k8s.aws/instance-memory",
"operator": "Gt",
"values": [
"8191"
]
},
{
"key": "karpenter.k8s.aws/instance-memory",
"operator": "Lt",
"values": [
"32769"
]
},
{
"key": "karpenter.k8s.aws/instance-hypervisor",
"operator": "In",
"values": [
"nitro"
]
}
]
},
"airflow-core-components": {
"disruption": {
"consolidateAfter": "1m",
"consolidationPolicy": "WhenEmpty"
},
"limits": {
"cpu": "40",
"memory": "160Gi"
},
"nodeClassRef": "default",
"requirements": [
{
"key": "karpenter.k8s.aws/instance-family",
"operator": "In",
"values": [
"m7i",
"m6i",
"m5",
"t3",
"c7i",
"c6i",
"c5",
"r7i",
"r6i",
"r5"
]
},
{
"key": "karpenter.k8s.aws/instance-cpu",
"operator": "Gt",
"values": [
"1"
]
},
{
"key": "karpenter.k8s.aws/instance-cpu",
"operator": "Lt",
"values": [
"17"
]
},
{
"key": "karpenter.k8s.aws/instance-memory",
"operator": "Gt",
"values": [
"8191"
]
},
{
"key": "karpenter.k8s.aws/instance-memory",
"operator": "Lt",
"values": [
"32769"
]
},
{
"key": "karpenter.k8s.aws/instance-hypervisor",
"operator": "In",
"values": [
"nitro"
]
}
]
},
"airflow-kubernetes-pod-operator": {
"disruption": {
"consolidateAfter": "1m",
"consolidationPolicy": "WhenEmpty"
},
"limits": {
"cpu": "100",
"memory": "400Gi"
},
"nodeClassRef": "default",
"requirements": [
{
"key": "karpenter.k8s.aws/instance-family",
"operator": "In",
"values": [
"m7i",
"m6i",
"m5",
"t3",
"c7i",
"c6i",
"c5",
"r7i",
"r6i",
"r5"
]
},
{
"key": "karpenter.k8s.aws/instance-cpu",
"operator": "Gt",
"values": [
"1"
]
},
{
"key": "karpenter.k8s.aws/instance-cpu",
"operator": "Lt",
"values": [
"17"
]
},
{
"key": "karpenter.k8s.aws/instance-memory",
"operator": "Gt",
"values": [
"8191"
]
},
{
"key": "karpenter.k8s.aws/instance-memory",
"operator": "Lt",
"values": [
"32769"
]
},
{
"key": "karpenter.k8s.aws/instance-hypervisor",
"operator": "In",
"values": [
"nitro"
]
}
]
},
"airflow-kubernetes-pod-operator-high-workload": {
"disruption": {
"consolidateAfter": "1m",
"consolidationPolicy": "WhenEmpty"
},
"limits": {
"cpu": "528",
"memory": "1056Gi"
},
"nodeClassRef": "airflow-kubernetes-pod-operator-high-workload",
"requirements": [
{
"key": "karpenter.k8s.aws/instance-family",
"operator": "In",
"values": [
"m7i",
"m6i",
"m5",
"t3",
"c7i",
"c6i",
"c5",
"r7i",
"r6i",
"r5"
]
},
{
"key": "karpenter.k8s.aws/instance-cpu",
"operator": "Gt",
"values": [
"1"
]
},
{
"key": "karpenter.k8s.aws/instance-cpu",
"operator": "Lt",
"values": [
"49"
]
},
{
"key": "karpenter.k8s.aws/instance-memory",
"operator": "Gt",
"values": [
"8191"
]
},
{
"key": "karpenter.k8s.aws/instance-memory",
"operator": "Lt",
"values": [
"98305"
]
},
{
"key": "karpenter.k8s.aws/instance-hypervisor",
"operator": "In",
"values": [
"nitro"
]
}
]
}
} | no |
| [kubeconfig\_filepath](#input\_kubeconfig\_filepath) | The path to the kubeconfig file for the Kubernetes cluster. | `string` | n/a | yes |
@@ -200,6 +202,7 @@ terraform apply -no-color 2>&1 | tee apply_output.txt
| [project](#input\_project) | The project or mission deploying Unity SPS. | `string` | `"unity"` | no |
| [release](#input\_release) | The software release version. | `string` | `"24.3"` | no |
| [service\_area](#input\_service\_area) | The service area owner of the resources being deployed. | `string` | `"sps"` | no |
+| [tags](#input\_tags) | Tags for the deployment (unused) | `map(string)` | {
"empty": ""
} | no |
| [venue](#input\_venue) | The MCP venue in which the resources will be deployed. | `string` | n/a | yes |
## Outputs
diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/README.md b/terraform-unity/modules/terraform-unity-sps-airflow/README.md
index f8db6278..bab93ee0 100644
--- a/terraform-unity/modules/terraform-unity-sps-airflow/README.md
+++ b/terraform-unity/modules/terraform-unity-sps-airflow/README.md
@@ -69,6 +69,7 @@ No modules.
| [kubernetes_storage_class.efs](https://registry.terraform.io/providers/hashicorp/kubernetes/2.32.0/docs/resources/storage_class) | resource |
| [null_resource.remove_keda_finalizers](https://registry.terraform.io/providers/hashicorp/null/3.2.3/docs/resources/resource) | resource |
| [random_id.airflow_webserver_secret](https://registry.terraform.io/providers/hashicorp/random/3.6.1/docs/resources/id) | resource |
+| [time_sleep.wait_after_ssm](https://registry.terraform.io/providers/hashicorp/time/0.12.1/docs/resources/sleep) | resource |
| [time_sleep.wait_for_efs_mount_target_dns_propagation](https://registry.terraform.io/providers/hashicorp/time/0.12.1/docs/resources/sleep) | resource |
| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/5.67.0/docs/data-sources/caller_identity) | data source |
| [aws_db_instance.db](https://registry.terraform.io/providers/hashicorp/aws/5.67.0/docs/data-sources/db_instance) | data source |
diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/main.tf b/terraform-unity/modules/terraform-unity-sps-airflow/main.tf
index b9acf1b5..80658129 100644
--- a/terraform-unity/modules/terraform-unity-sps-airflow/main.tf
+++ b/terraform-unity/modules/terraform-unity-sps-airflow/main.tf
@@ -635,7 +635,7 @@ resource "aws_ssm_parameter" "unity_proxy_airflow_ui" {
Redirect "/${var.project}/${var.venue}/sps/home"
- ProxyPassMatch "http://${data.kubernetes_ingress_v1.airflow_ingress_internal.status[0].load_balancer[0].ingress[0].hostname}:5000/$1"
+ ProxyPassMatch "http://${data.kubernetes_ingress_v1.airflow_ingress_internal.status[0].load_balancer[0].ingress[0].hostname}:5000/$1" retry=5 disablereuse=On
ProxyPreserveHost On
FallbackResource /management/index.html
AddOutputFilterByType INFLATE;SUBSTITUTE;DEFLATE text/html
@@ -653,8 +653,8 @@ EOT
data "aws_lambda_functions" "lambda_check_all" {}
resource "aws_lambda_invocation" "unity_proxy_lambda_invocation" {
- count = contains(data.aws_lambda_functions.lambda_check_all.function_names, "unity-${var.venue}-httpdproxymanagement") ? 1 : 0
- function_name = "unity-${var.venue}-httpdproxymanagement"
+ count = contains(data.aws_lambda_functions.lambda_check_all.function_names, "${var.project}-${var.venue}-httpdproxymanagement") ? 1 : 0
+ function_name = "${var.project}-${var.venue}-httpdproxymanagement"
input = "{}"
triggers = {
redeployment = sha1(jsonencode([
diff --git a/terraform-unity/modules/terraform-unity-sps-eks/README.md b/terraform-unity/modules/terraform-unity-sps-eks/README.md
index 24bc32aa..6030c029 100644
--- a/terraform-unity/modules/terraform-unity-sps-eks/README.md
+++ b/terraform-unity/modules/terraform-unity-sps-eks/README.md
@@ -18,7 +18,7 @@
| Name | Source | Version |
|------|--------|---------|
-| [unity-eks](#module\_unity-eks) | git@github.com:unity-sds/unity-cs-infra.git//terraform-unity-eks_module | unity-sps-2.2.0-hotfix |
+| [unity-eks](#module\_unity-eks) | git::https://github.com/unity-sds/unity-cs-infra.git//terraform-unity-eks_module | 59-sps-eks-marketplace-adjustments |
## Resources
@@ -31,10 +31,13 @@
| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
+| [deployment\_name](#input\_deployment\_name) | The name of the deployment. | `string` | n/a | yes |
+| [installprefix](#input\_installprefix) | The install prefix for the service area (unused) | `string` | `""` | no |
| [nodegroups](#input\_nodegroups) | A map of node group configurations | map(object({
create_iam_role = optional(bool)
iam_role_arn = optional(string)
ami_id = optional(string)
min_size = optional(number)
max_size = optional(number)
desired_size = optional(number)
instance_types = optional(list(string))
capacity_type = optional(string)
enable_bootstrap_user_data = optional(bool)
metadata_options = optional(map(any))
block_device_mappings = optional(map(object({
device_name = string
ebs = object({
volume_size = number
volume_type = string
encrypted = bool
delete_on_termination = bool
})
})))
})) | {
"defaultGroup": {
"block_device_mappings": {
"xvda": {
"device_name": "/dev/xvda",
"ebs": {
"delete_on_termination": true,
"encrypted": true,
"volume_size": 100,
"volume_type": "gp2"
}
}
},
"desired_size": 1,
"instance_types": [
"t3.xlarge"
],
"max_size": 1,
"metadata_options": {
"http_endpoint": "enabled",
"http_put_response_hop_limit": 3
},
"min_size": 1
}
} | no |
| [project](#input\_project) | The project or mission deploying Unity SPS | `string` | `"unity"` | no |
| [release](#input\_release) | The software release version. | `string` | `"24.3"` | no |
| [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | `"sps"` | no |
+| [tags](#input\_tags) | Tags for the deployment (unused) | `map(string)` | {
"empty": ""
} | no |
| [venue](#input\_venue) | The MCP venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes |
## Outputs
diff --git a/terraform-unity/modules/terraform-unity-sps-eks/main.tf b/terraform-unity/modules/terraform-unity-sps-eks/main.tf
index 777fd649..20fae0fa 100644
--- a/terraform-unity/modules/terraform-unity-sps-eks/main.tf
+++ b/terraform-unity/modules/terraform-unity-sps-eks/main.tf
@@ -9,8 +9,10 @@ terraform {
}
module "unity-eks" {
- source = "git@github.com:unity-sds/unity-cs-infra.git//terraform-unity-eks_module?ref=unity-sps-2.2.0-hotfix"
+ source = "git::https://github.com/unity-sds/unity-cs-infra.git//terraform-unity-eks_module?ref=unity-sps-2.4.0"
deployment_name = local.cluster_name
+ project = var.project
+ venue = var.venue
nodegroups = var.nodegroups
aws_auth_roles = [{
rolearn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/mcp-tenantOperator"
diff --git a/terraform-unity/modules/terraform-unity-sps-eks/variables.tf b/terraform-unity/modules/terraform-unity-sps-eks/variables.tf
index 3b36d1c3..9fe2e106 100644
--- a/terraform-unity/modules/terraform-unity-sps-eks/variables.tf
+++ b/terraform-unity/modules/terraform-unity-sps-eks/variables.tf
@@ -21,6 +21,26 @@ variable "release" {
default = "24.3"
}
+# tflint-ignore: terraform_unused_declarations
+variable "deployment_name" {
+ description = "The name of the deployment."
+ type = string
+}
+
+# tflint-ignore: terraform_unused_declarations
+variable "tags" {
+ description = "Tags for the deployment (unused)"
+ type = map(string)
+ default = { empty = "" }
+}
+
+# tflint-ignore: terraform_unused_declarations
+variable "installprefix" {
+ description = "The install prefix for the service area (unused)"
+ type = string
+ default = ""
+}
+
variable "nodegroups" {
description = "A map of node group configurations"
type = map(object({
diff --git a/terraform-unity/modules/terraform-unity-sps-initiators/main.tf b/terraform-unity/modules/terraform-unity-sps-initiators/main.tf
index 56893c70..0e9f6e8e 100644
--- a/terraform-unity/modules/terraform-unity-sps-initiators/main.tf
+++ b/terraform-unity/modules/terraform-unity-sps-initiators/main.tf
@@ -78,7 +78,7 @@ resource "aws_s3_object" "router_config" {
}
module "unity_initiator" {
- source = "git@github.com:unity-sds/unity-initiator.git//terraform-unity/initiator?ref=unity-sps-2.2.0"
+ source = "git::https://github.com/unity-sds/unity-initiator.git//terraform-unity/initiator?ref=unity-sps-2.2.0"
code_bucket = aws_s3_bucket.code.id
project = var.project
router_config = "s3://${aws_s3_bucket.config.id}/${aws_s3_object.router_config.key}"
@@ -91,7 +91,7 @@ resource "aws_s3_object" "isl_stacam_rawdp_folder" {
}
module "s3_bucket_notification" {
- source = "git@github.com:unity-sds/unity-initiator.git//terraform-unity/triggers/s3-bucket-notification?ref=unity-sps-2.2.0"
+ source = "git::https://github.com/unity-sds/unity-initiator.git//terraform-unity/triggers/s3-bucket-notification?ref=unity-sps-2.2.0"
initiator_topic_arn = module.unity_initiator.initiator_topic_arn
isl_bucket = aws_s3_bucket.inbound_staging_location.id
isl_bucket_prefix = "STACAM/RawDP/"
diff --git a/terraform-unity/modules/terraform-unity-sps-karpenter/README.md b/terraform-unity/modules/terraform-unity-sps-karpenter/README.md
index c8d232be..05ff4958 100644
--- a/terraform-unity/modules/terraform-unity-sps-karpenter/README.md
+++ b/terraform-unity/modules/terraform-unity-sps-karpenter/README.md
@@ -35,10 +35,13 @@
| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
+| [deployment\_name](#input\_deployment\_name) | The name of the deployment. | `string` | n/a | yes |
| [helm\_charts](#input\_helm\_charts) | Helm charts for the associated services. | map(object({
repository = string
chart = string
version = string
})) | {
"karpenter": {
"chart": "karpenter",
"repository": "oci://public.ecr.aws/karpenter",
"version": "1.0.2"
}
} | no |
+| [installprefix](#input\_installprefix) | The install prefix for the service area (unused) | `string` | `""` | no |
| [project](#input\_project) | The project or mission deploying Unity SPS | `string` | `"unity"` | no |
| [release](#input\_release) | The software release version. | `string` | `"24.3"` | no |
| [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | `"sps"` | no |
+| [tags](#input\_tags) | Tags for the deployment (unused) | `map(string)` | {
"empty": ""
} | no |
| [venue](#input\_venue) | The MCP venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes |
## Outputs
diff --git a/terraform-unity/modules/terraform-unity-sps-karpenter/variables.tf b/terraform-unity/modules/terraform-unity-sps-karpenter/variables.tf
index 84221fcd..a726aa87 100644
--- a/terraform-unity/modules/terraform-unity-sps-karpenter/variables.tf
+++ b/terraform-unity/modules/terraform-unity-sps-karpenter/variables.tf
@@ -21,6 +21,26 @@ variable "release" {
default = "24.3"
}
+# tflint-ignore: terraform_unused_declarations
+variable "deployment_name" {
+ description = "The name of the deployment."
+ type = string
+}
+
+# tflint-ignore: terraform_unused_declarations
+variable "tags" {
+ description = "Tags for the deployment (unused)"
+ type = map(string)
+ default = { empty = "" }
+}
+
+# tflint-ignore: terraform_unused_declarations
+variable "installprefix" {
+ description = "The install prefix for the service area (unused)"
+ type = string
+ default = ""
+}
+
variable "helm_charts" {
description = "Helm charts for the associated services."
type = map(object({
diff --git a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf
index 7803b985..defca7d8 100644
--- a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf
+++ b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf
@@ -387,7 +387,7 @@ resource "aws_ssm_parameter" "unity_proxy_ogc_api" {
ProxyPassReverse "/"
- ProxyPassMatch "http://${data.kubernetes_ingress_v1.ogc_processes_api_ingress_internal.status[0].load_balancer[0].ingress[0].hostname}:5001/$1"
+ ProxyPassMatch "http://${data.kubernetes_ingress_v1.ogc_processes_api_ingress_internal.status[0].load_balancer[0].ingress[0].hostname}:5001/$1" retry=5 disablereuse=On
ProxyPreserveHost On
FallbackResource /management/index.html
AddOutputFilterByType INFLATE;SUBSTITUTE;DEFLATE text/html
@@ -405,8 +405,8 @@ EOT
data "aws_lambda_functions" "lambda_check_all" {}
resource "aws_lambda_invocation" "unity_proxy_lambda_invocation" {
- count = contains(data.aws_lambda_functions.lambda_check_all.function_names, "unity-${var.venue}-httpdproxymanagement") ? 1 : 0
- function_name = "unity-${var.venue}-httpdproxymanagement"
+ count = contains(data.aws_lambda_functions.lambda_check_all.function_names, "${var.project}-${var.venue}-httpdproxymanagement") ? 1 : 0
+ function_name = "${var.project}-${var.venue}-httpdproxymanagement"
input = "{}"
triggers = {
redeployment = sha1(jsonencode([
diff --git a/terraform-unity/variables.tf b/terraform-unity/variables.tf
index dfe3d4e5..d0843269 100644
--- a/terraform-unity/variables.tf
+++ b/terraform-unity/variables.tf
@@ -338,3 +338,24 @@ variable "dag_catalog_repo" {
dags_directory_path = "airflow/dags"
}
}
+
+# tflint-ignore: terraform_unused_declarations
+variable "deployment_name" {
+ description = "The name of the deployment."
+ type = string
+ default = ""
+}
+
+# tflint-ignore: terraform_unused_declarations
+variable "tags" {
+ description = "Tags for the deployment (unused)"
+ type = map(string)
+ default = { empty = "" }
+}
+
+# tflint-ignore: terraform_unused_declarations
+variable "installprefix" {
+ description = "The install prefix for the service area (unused)"
+ type = string
+ default = ""
+}