External Deployments Operator IAM Role (#258)

dominodatalab · Aug 28, 2024 · 792c0bd · 792c0bd
1 parent 1466c67
commit 792c0bd
Show file tree

Hide file tree

Showing 17 changed files with 142 additions and 3 deletions.
diff --git a/.gitignore b/.gitignore
@@ -53,4 +53,5 @@ k8s-proxy-tunnel.sh
 
 # local files
 .DS_Store
-*/.idea/*
+.idea/*
+*.iml
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -67,6 +67,7 @@ repos:
           - "--args=--severity=HIGH,CRITICAL"
           - "--args=--ignorefile=__GIT_WORKING_DIR__/.trivyignore"
           - "--args=--exit-code=1"
+          - "--hook-config=--parallelism-limit=1"
   - repo: local
     hooks:
       - id: check_aws_partition

diff --git a/examples/deploy/meta.sh b/examples/deploy/meta.sh
@@ -11,12 +11,13 @@ declare -a MOD_DIRS=(
 
 declare -A COMP_MODS
 COMP_MODS["infra"]="infra"
-COMP_MODS["cluster"]="eks irsa_external_dns irsa_policies"
+COMP_MODS["cluster"]="eks irsa_external_dns irsa_policies irsa_external_deployments_operator"
 COMP_MODS["nodes"]="nodes"
 
 declare -A MOD_ADD
 MOD_ADD["irsa_external_dns"]="irsa"
 MOD_ADD["irsa_policies"]="irsa"
+MOD_ADD["irsa_external_deployments_operator"]="irsa"
 
 INFRA_DIR="${MOD_DIRS[0]}"
 CLUSTER_DIR="${MOD_DIRS[1]}"

diff --git a/examples/deploy/terraform/cluster/README.md b/examples/deploy/terraform/cluster/README.md
@@ -21,6 +21,7 @@
 | Name | Source | Version |
 |------|--------|---------|
 | <a name="module_eks"></a> [eks](#module\_eks) | ./../../../../modules/eks | n/a |
+| <a name="module_irsa_external_deployments_operator"></a> [irsa\_external\_deployments\_operator](#module\_irsa\_external\_deployments\_operator) | ./../../../../modules/irsa | n/a |
 | <a name="module_irsa_external_dns"></a> [irsa\_external\_dns](#module\_irsa\_external\_dns) | ./../../../../modules/irsa | n/a |
 | <a name="module_irsa_policies"></a> [irsa\_policies](#module\_irsa\_policies) | ./../../../../modules/irsa | n/a |
 
@@ -37,6 +38,7 @@
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_eks"></a> [eks](#input\_eks) | service\_ipv4\_cidr = CIDR for EKS cluster kubernetes\_network\_config.<br>    creation\_role\_name = Name of the role to import.<br>    k8s\_version = EKS cluster k8s version.<br>    kubeconfig = {<br>      extra\_args = Optional extra args when generating kubeconfig.<br>      path       = Fully qualified path name to write the kubeconfig file.<br>    }<br>    public\_access = {<br>      enabled = Enable EKS API public endpoint.<br>      cidrs   = List of CIDR ranges permitted for accessing the EKS public endpoint.<br>    }<br>    Custom role maps for aws auth configmap<br>    custom\_role\_maps = {<br>      rolearn  = string<br>      username = string<br>      groups   = list(string)<br>    }<br>    master\_role\_names  = IAM role names to be added as masters in eks.<br>    cluster\_addons     = EKS cluster addons. vpc-cni is installed separately.<br>    vpc\_cni            = Configuration for AWS VPC CNI<br>    ssm\_log\_group\_name = CloudWatch log group to send the SSM session logs to.<br>    identity\_providers = Configuration for IDP(Identity Provider).<br>  } | <pre>object({<br>    service_ipv4_cidr  = optional(string)<br>    creation_role_name = optional(string, null)<br>    k8s_version        = optional(string)<br>    kubeconfig = optional(object({<br>      extra_args = optional(string)<br>      path       = optional(string)<br>    }), {})<br>    public_access = optional(object({<br>      enabled = optional(bool)<br>      cidrs   = optional(list(string))<br>    }), {})<br>    custom_role_maps = optional(list(object({<br>      rolearn  = string<br>      username = string<br>      groups   = list(string)<br>    })))<br>    master_role_names  = optional(list(string))<br>    cluster_addons     = optional(list(string))<br>    ssm_log_group_name = optional(string)<br>    vpc_cni = optional(object({<br>      prefix_delegation = optional(bool)<br>      annotate_pod_ip   = optional(bool)<br>    }))<br>    identity_providers = optional(list(object({<br>      client_id                     = string<br>      groups_claim                  = optional(string)<br>      groups_prefix                 = optional(string)<br>      identity_provider_config_name = string<br>      issuer_url                    = optional(string)<br>      required_claims               = optional(string)<br>      username_claim                = optional(string)<br>      username_prefix               = optional(string)<br>    })))<br>  })</pre> | `{}` | no |
+| <a name="input_irsa_external_deployments_operator"></a> [irsa\_external\_deployments\_operator](#input\_irsa\_external\_deployments\_operator) | Config to create IRSA role for the external deployments operator. | <pre>object({<br>    enabled              = optional(bool, false)<br>    namespace            = optional(string, "domino-compute")<br>    service_account_name = optional(string, "pham-juno-operator")<br>  })</pre> | `{}` | no |
 | <a name="input_irsa_external_dns"></a> [irsa\_external\_dns](#input\_irsa\_external\_dns) | Mappings for custom IRSA configurations. | <pre>object({<br>    enabled             = optional(bool, false)<br>    hosted_zone_name    = optional(string, null)<br>    namespace           = optional(string, null)<br>    serviceaccount_name = optional(string, null)<br>    rm_role_policy = optional(object({<br>      remove           = optional(bool, false)<br>      detach_from_role = optional(bool, false)<br>      policy_name      = optional(string, "")<br>    }), {})<br>  })</pre> | `{}` | no |
 | <a name="input_irsa_policies"></a> [irsa\_policies](#input\_irsa\_policies) | Mappings for custom IRSA configurations. | <pre>list(object({<br>    name                = string<br>    namespace           = string<br>    serviceaccount_name = string<br>    policy              = string #json<br>  }))</pre> | `[]` | no |
 | <a name="input_kms_info"></a> [kms\_info](#input\_kms\_info) | Overrides the KMS key information. Meant for migrated configurations.<br>    {<br>      key\_id  = KMS key id.<br>      key\_arn = KMS key arn.<br>      enabled = KMS key is enabled.<br>    } | <pre>object({<br>    key_id  = string<br>    key_arn = string<br>    enabled = bool<br>  })</pre> | `null` | no |
@@ -47,6 +49,7 @@
 | Name | Description |
 |------|-------------|
 | <a name="output_eks"></a> [eks](#output\_eks) | EKS details. |
+| <a name="output_external_deployments_operator"></a> [external\_deployments\_operator](#output\_external\_deployments\_operator) | "External\_deployments\_operator info"<br>  {<br>    irsa\_role = irsa role arn<br>    service\_account\_name = service account name<br>  } |
 | <a name="output_external_dns_irsa_role_arn"></a> [external\_dns\_irsa\_role\_arn](#output\_external\_dns\_irsa\_role\_arn) | "External\_dns info"<br>  {<br>    irsa\_role = irsa role arn.<br>    zone\_id   = hosted zone id for external\_dns Iam policy<br>    zone\_name = hosted zone name for external\_dns Iam policy<br>  } |
 | <a name="output_infra"></a> [infra](#output\_infra) | Infra details. |
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
diff --git a/examples/deploy/terraform/cluster/main.tf b/examples/deploy/terraform/cluster/main.tf
@@ -73,6 +73,18 @@ module "irsa_policies" {
   additional_irsa_configs = var.irsa_policies
 }
 
+# If you are enabling the IRSA configuration for external-deployments-operator
+module "irsa_external_deployments_operator" {
+  source                        = "./../../../../modules/irsa"
+  use_cluster_odc_idp           = local.is_eks_account_same
+  eks_info                      = module.eks.info
+  external_deployments_operator = var.irsa_external_deployments_operator
+
+  providers = {
+    aws = aws.global
+  }
+}
+
 # Provider configuration for the account where the hosted zone is defined.
 # Useful in configurations where accounts do not have a public hosted zone(i.e us-gov regions) and internet routing(public DNS)
 # is instead defined in a different account. Configure the `global` aws alias accordingly,

diff --git a/examples/deploy/terraform/cluster/outputs.tf b/examples/deploy/terraform/cluster/outputs.tf
@@ -19,3 +19,14 @@ output "external_dns_irsa_role_arn" {
   EOF
   value       = module.irsa_external_dns
 }
+
+output "external_deployments_operator" {
+  description = <<EOF
+  "External_deployments_operator info"
+  {
+    irsa_role = irsa role arn
+    service_account_name = service account name
+  }
+  EOF
+  value       = module.irsa_external_deployments_operator
+}
diff --git a/examples/deploy/terraform/cluster/variables.tf b/examples/deploy/terraform/cluster/variables.tf
@@ -117,3 +117,15 @@ variable "use_fips_endpoint" {
   type        = bool
   default     = false
 }
+
+variable "irsa_external_deployments_operator" {
+  description = "Config to create IRSA role for the external deployments operator."
+
+  type = object({
+    enabled              = optional(bool, false)
+    namespace            = optional(string, "domino-compute")
+    service_account_name = optional(string, "pham-juno-operator")
+  })
+
+  default = {}
+}
diff --git a/examples/tfvars/external-deployments.tfvars b/examples/tfvars/external-deployments.tfvars
@@ -0,0 +1,25 @@
+deploy_id        = "ed-test-001"
+region           = "us-west-2"
+ssh_pvt_key_path = "domino.pem"
+
+default_node_groups = {
+  compute = {
+    availability_zone_ids = ["usw2-az1", "usw2-az2"]
+  }
+  gpu = {
+    availability_zone_ids = ["usw2-az1", "usw2-az2"]
+  }
+  platform = {
+    "availability_zone_ids" = ["usw2-az1", "usw2-az2"]
+  }
+}
+
+bastion = {
+  enabled = true
+}
+
+irsa_external_deployments_operator = {
+  enabled              = "true",
+  namespace            = "domino-compute",
+  service_account_name = "test-operator-account"
+}
diff --git a/modules/irsa/README.md b/modules/irsa/README.md
@@ -33,6 +33,7 @@ No modules.
 | [aws_iam_policy.external_dns](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_policy.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_policy.trident_operator](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
+| [aws_iam_role.external_deployments_operator](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
 | [aws_iam_role.external_dns](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
 | [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
 | [aws_iam_role.trident_operator](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
@@ -52,6 +53,7 @@ No modules.
 |------|-------------|------|---------|:--------:|
 | <a name="input_additional_irsa_configs"></a> [additional\_irsa\_configs](#input\_additional\_irsa\_configs) | Input for additional irsa configurations | <pre>list(object({<br>    name                = string<br>    namespace           = string<br>    serviceaccount_name = string<br>    policy              = string #json<br>  }))</pre> | `[]` | no |
 | <a name="input_eks_info"></a> [eks\_info](#input\_eks\_info) | cluster = {<br>      specs {<br>        name            = Cluster name.<br>        account\_id      = AWS account id where the cluster resides.<br>      }<br>      oidc = {<br>        arn = OIDC provider ARN.<br>        url = OIDC provider url.<br>        cert = {<br>          thumbprint\_list = OIDC cert thumbprints.<br>          url             = OIDC cert URL.<br>      }<br>    } | <pre>object({<br>    nodes = object({<br>      roles = list(object({<br>        arn  = string<br>        name = string<br>      }))<br>    })<br>    cluster = object({<br>      specs = object({<br>        name       = string<br>        account_id = string<br>      })<br>      oidc = object({<br>        arn = string<br>        url = string<br>        cert = object({<br>          thumbprint_list = list(string)<br>          url             = string<br>        })<br>      })<br>    })<br>  })</pre> | n/a | yes |
+| <a name="input_external_deployments_operator"></a> [external\_deployments\_operator](#input\_external\_deployments\_operator) | Config to create IRSA role for the external deployments operator. | <pre>object({<br>    enabled              = optional(bool, false)<br>    namespace            = optional(string, "domino-compute")<br>    service_account_name = optional(string, "pham-juno-operator")<br>  })</pre> | `{}` | no |
 | <a name="input_external_dns"></a> [external\_dns](#input\_external\_dns) | Config to enable irsa for external-dns | <pre>object({<br>    enabled             = optional(bool, false)<br>    hosted_zone_name    = optional(string, null)<br>    hosted_zone_private = optional(string, false)<br>    namespace           = optional(string, "domino-platform")<br>    serviceaccount_name = optional(string, "external-dns")<br>    rm_role_policy = optional(object({<br>      remove           = optional(bool, false)<br>      detach_from_role = optional(bool, false)<br>      policy_name      = optional(string, "")<br>    }), {})<br>  })</pre> | `{}` | no |
 | <a name="input_netapp_trident_operator"></a> [netapp\_trident\_operator](#input\_netapp\_trident\_operator) | Config to create IRSA role for the netapp-trident-operator. | <pre>object({<br>    enabled             = optional(bool, false)<br>    namespace           = optional(string, "trident")<br>    serviceaccount_name = optional(string, "trident-controller")<br>    region              = optional(string)<br>  })</pre> | `{}` | no |
 | <a name="input_use_cluster_odc_idp"></a> [use\_cluster\_odc\_idp](#input\_use\_cluster\_odc\_idp) | Toogle to uset the oidc idp connector in the trust policy.<br>    Set to `true` if the cluster and the hosted zone are in different aws accounts.<br>    `rm_role_policy` used to facilitiate the cleanup if a node attached policy was used previously. | `bool` | `true` | no |
@@ -61,6 +63,7 @@ No modules.
 
 | Name | Description |
 |------|-------------|
+| <a name="output_external_deployments_operator"></a> [external\_deployments\_operator](#output\_external\_deployments\_operator) | External deployments operator role info |
 | <a name="output_external_dns"></a> [external\_dns](#output\_external\_dns) | External\_dns info |
 | <a name="output_netapp_trident_operator"></a> [netapp\_trident\_operator](#output\_netapp\_trident\_operator) | NetApp Astra Trident NETAPP Operator role info |
 | <a name="output_roles"></a> [roles](#output\_roles) | Roles mapping info |

diff --git a/modules/irsa/external-deployments-operator.tf b/modules/irsa/external-deployments-operator.tf
@@ -0,0 +1,23 @@
+resource "aws_iam_role" "external_deployments_operator" {
+  count = var.external_deployments_operator.enabled ? 1 : 0
+
+  name = "${local.name_prefix}-external-deployments-operator"
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = "sts:AssumeRoleWithWebIdentity"
+        Effect = "Allow"
+        Principal = {
+          Federated = local.oidc_provider_arn
+        }
+        Condition : {
+          StringEquals : {
+            "${trimprefix(local.oidc_provider_url, "https://")}:aud" : "sts.amazonaws.com",
+            "${trimprefix(local.oidc_provider_url, "https://")}:sub" : "system:serviceaccount:${var.external_deployments_operator.namespace}:${var.external_deployments_operator.service_account_name}"
+          }
+        }
+      },
+    ]
+  })
+}
diff --git a/modules/irsa/outputs.tf b/modules/irsa/outputs.tf
@@ -19,3 +19,11 @@ output "netapp_trident_operator" {
     irsa_role = aws_iam_role.trident_operator[0].arn
   } : null
 }
+
+output "external_deployments_operator" {
+  description = "External deployments operator role info"
+  value = var.external_deployments_operator.enabled ? {
+    irsa_role            = aws_iam_role.external_deployments_operator[0].arn
+    service_account_name = var.external_deployments_operator.service_account_name
+  } : null
+}
diff --git a/modules/irsa/variables.tf b/modules/irsa/variables.tf
@@ -107,3 +107,14 @@ variable "netapp_trident_operator" {
 
   default = {}
 }
+variable "external_deployments_operator" {
+  description = "Config to create IRSA role for the external deployments operator."
+
+  type = object({
+    enabled              = optional(bool, false)
+    namespace            = optional(string, "domino-compute")
+    service_account_name = optional(string, "pham-juno-operator")
+  })
+
+  default = {}
+}
diff --git a/tests/deploy/cluster-ci.tfvars.tftpl b/tests/deploy/cluster-ci.tfvars.tftpl
@@ -3,4 +3,10 @@ irsa_external_dns = {
   hosted_zone_name = "deploys-delta.domino.tech"
 }
 
+irsa_external_deployments_operator = {
+  enabled = true
+  namespace = "domino-compute"
+  service_account_name = "test-operator-account"
+}
+
 use_fips_endpoint = true
diff --git a/tests/deploy/meta.sh b/tests/deploy/meta.sh
@@ -12,12 +12,13 @@ NODES_VARS_TPL="${SH_DIR}/nodes-ci.tfvars.tftpl"
 
 declare -A COMP_MODS
 COMP_MODS["infra"]="infra"
-COMP_MODS["cluster"]="eks irsa_external_dns irsa_policies"
+COMP_MODS["cluster"]="eks irsa_external_dns irsa_policies irsa_external_deployments_operator"
 COMP_MODS["nodes"]="nodes"
 
 declare -A MOD_ADD
 MOD_ADD["irsa_external_dns"]="irsa"
 MOD_ADD["irsa_policies"]="irsa"
+MOD_ADD["irsa_external_deployments_operator"]="irsa"
 
 export SH_DIR \
   CI_DEPLOY \