From 792c0bd100b794cffb8bee32f1654cb123d6fd86 Mon Sep 17 00:00:00 2001 From: David McWhorter <105815369+dmcwhorter-ddl@users.noreply.github.com> Date: Wed, 28 Aug 2024 16:42:57 -0400 Subject: [PATCH] External Deployments Operator IAM Role (#258) --- .gitignore | 3 ++- .pre-commit-config.yaml | 1 + examples/deploy/meta.sh | 3 ++- examples/deploy/terraform/cluster/README.md | 3 +++ examples/deploy/terraform/cluster/main.tf | 12 +++++++++ examples/deploy/terraform/cluster/outputs.tf | 11 ++++++++ .../deploy/terraform/cluster/variables.tf | 12 +++++++++ examples/tfvars/external-deployments.tfvars | 25 +++++++++++++++++++ modules/irsa/README.md | 3 +++ modules/irsa/external-deployments-operator.tf | 23 +++++++++++++++++ modules/irsa/outputs.tf | 8 ++++++ modules/irsa/variables.tf | 11 ++++++++ tests/deploy/cluster-ci.tfvars.tftpl | 6 +++++ tests/deploy/meta.sh | 3 ++- tests/plan/terraform/README.md | 2 ++ tests/plan/terraform/main.tf | 7 ++++++ tests/plan/terraform/variables.tf | 12 +++++++++ 17 files changed, 142 insertions(+), 3 deletions(-) create mode 100644 examples/tfvars/external-deployments.tfvars create mode 100644 modules/irsa/external-deployments-operator.tf diff --git a/.gitignore b/.gitignore index eef08482..7fcbb6f0 100644 --- a/.gitignore +++ b/.gitignore @@ -53,4 +53,5 @@ k8s-proxy-tunnel.sh # local files .DS_Store -*/.idea/* +.idea/* +*.iml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 36cf374b..0299f05e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -67,6 +67,7 @@ repos: - "--args=--severity=HIGH,CRITICAL" - "--args=--ignorefile=__GIT_WORKING_DIR__/.trivyignore" - "--args=--exit-code=1" + - "--hook-config=--parallelism-limit=1" - repo: local hooks: - id: check_aws_partition diff --git a/examples/deploy/meta.sh b/examples/deploy/meta.sh index 759e08e0..a69d87f5 100644 --- a/examples/deploy/meta.sh +++ b/examples/deploy/meta.sh @@ -11,12 +11,13 @@ declare -a MOD_DIRS=( declare -A COMP_MODS COMP_MODS["infra"]="infra" -COMP_MODS["cluster"]="eks irsa_external_dns irsa_policies" +COMP_MODS["cluster"]="eks irsa_external_dns irsa_policies irsa_external_deployments_operator" COMP_MODS["nodes"]="nodes" declare -A MOD_ADD MOD_ADD["irsa_external_dns"]="irsa" MOD_ADD["irsa_policies"]="irsa" +MOD_ADD["irsa_external_deployments_operator"]="irsa" INFRA_DIR="${MOD_DIRS[0]}" CLUSTER_DIR="${MOD_DIRS[1]}" diff --git a/examples/deploy/terraform/cluster/README.md b/examples/deploy/terraform/cluster/README.md index b200df3c..f69575e8 100644 --- a/examples/deploy/terraform/cluster/README.md +++ b/examples/deploy/terraform/cluster/README.md @@ -21,6 +21,7 @@ | Name | Source | Version | |------|--------|---------| | [eks](#module\_eks) | ./../../../../modules/eks | n/a | +| [irsa\_external\_deployments\_operator](#module\_irsa\_external\_deployments\_operator) | ./../../../../modules/irsa | n/a | | [irsa\_external\_dns](#module\_irsa\_external\_dns) | ./../../../../modules/irsa | n/a | | [irsa\_policies](#module\_irsa\_policies) | ./../../../../modules/irsa | n/a | @@ -37,6 +38,7 @@ | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [eks](#input\_eks) | service\_ipv4\_cidr = CIDR for EKS cluster kubernetes\_network\_config.
creation\_role\_name = Name of the role to import.
k8s\_version = EKS cluster k8s version.
kubeconfig = {
extra\_args = Optional extra args when generating kubeconfig.
path = Fully qualified path name to write the kubeconfig file.
}
public\_access = {
enabled = Enable EKS API public endpoint.
cidrs = List of CIDR ranges permitted for accessing the EKS public endpoint.
}
Custom role maps for aws auth configmap
custom\_role\_maps = {
rolearn = string
username = string
groups = list(string)
}
master\_role\_names = IAM role names to be added as masters in eks.
cluster\_addons = EKS cluster addons. vpc-cni is installed separately.
vpc\_cni = Configuration for AWS VPC CNI
ssm\_log\_group\_name = CloudWatch log group to send the SSM session logs to.
identity\_providers = Configuration for IDP(Identity Provider).
} |
object({
service_ipv4_cidr = optional(string)
creation_role_name = optional(string, null)
k8s_version = optional(string)
kubeconfig = optional(object({
extra_args = optional(string)
path = optional(string)
}), {})
public_access = optional(object({
enabled = optional(bool)
cidrs = optional(list(string))
}), {})
custom_role_maps = optional(list(object({
rolearn = string
username = string
groups = list(string)
})))
master_role_names = optional(list(string))
cluster_addons = optional(list(string))
ssm_log_group_name = optional(string)
vpc_cni = optional(object({
prefix_delegation = optional(bool)
annotate_pod_ip = optional(bool)
}))
identity_providers = optional(list(object({
client_id = string
groups_claim = optional(string)
groups_prefix = optional(string)
identity_provider_config_name = string
issuer_url = optional(string)
required_claims = optional(string)
username_claim = optional(string)
username_prefix = optional(string)
})))
})
| `{}` | no | +| [irsa\_external\_deployments\_operator](#input\_irsa\_external\_deployments\_operator) | Config to create IRSA role for the external deployments operator. |
object({
enabled = optional(bool, false)
namespace = optional(string, "domino-compute")
service_account_name = optional(string, "pham-juno-operator")
})
| `{}` | no | | [irsa\_external\_dns](#input\_irsa\_external\_dns) | Mappings for custom IRSA configurations. |
object({
enabled = optional(bool, false)
hosted_zone_name = optional(string, null)
namespace = optional(string, null)
serviceaccount_name = optional(string, null)
rm_role_policy = optional(object({
remove = optional(bool, false)
detach_from_role = optional(bool, false)
policy_name = optional(string, "")
}), {})
})
| `{}` | no | | [irsa\_policies](#input\_irsa\_policies) | Mappings for custom IRSA configurations. |
list(object({
name = string
namespace = string
serviceaccount_name = string
policy = string #json
}))
| `[]` | no | | [kms\_info](#input\_kms\_info) | Overrides the KMS key information. Meant for migrated configurations.
{
key\_id = KMS key id.
key\_arn = KMS key arn.
enabled = KMS key is enabled.
} |
object({
key_id = string
key_arn = string
enabled = bool
})
| `null` | no | @@ -47,6 +49,7 @@ | Name | Description | |------|-------------| | [eks](#output\_eks) | EKS details. | +| [external\_deployments\_operator](#output\_external\_deployments\_operator) | "External\_deployments\_operator info"
{
irsa\_role = irsa role arn
service\_account\_name = service account name
} | | [external\_dns\_irsa\_role\_arn](#output\_external\_dns\_irsa\_role\_arn) | "External\_dns info"
{
irsa\_role = irsa role arn.
zone\_id = hosted zone id for external\_dns Iam policy
zone\_name = hosted zone name for external\_dns Iam policy
} | | [infra](#output\_infra) | Infra details. | diff --git a/examples/deploy/terraform/cluster/main.tf b/examples/deploy/terraform/cluster/main.tf index cf90583d..4e4c89ed 100644 --- a/examples/deploy/terraform/cluster/main.tf +++ b/examples/deploy/terraform/cluster/main.tf @@ -73,6 +73,18 @@ module "irsa_policies" { additional_irsa_configs = var.irsa_policies } +# If you are enabling the IRSA configuration for external-deployments-operator +module "irsa_external_deployments_operator" { + source = "./../../../../modules/irsa" + use_cluster_odc_idp = local.is_eks_account_same + eks_info = module.eks.info + external_deployments_operator = var.irsa_external_deployments_operator + + providers = { + aws = aws.global + } +} + # Provider configuration for the account where the hosted zone is defined. # Useful in configurations where accounts do not have a public hosted zone(i.e us-gov regions) and internet routing(public DNS) # is instead defined in a different account. Configure the `global` aws alias accordingly, diff --git a/examples/deploy/terraform/cluster/outputs.tf b/examples/deploy/terraform/cluster/outputs.tf index 85d6b918..274ff13e 100644 --- a/examples/deploy/terraform/cluster/outputs.tf +++ b/examples/deploy/terraform/cluster/outputs.tf @@ -19,3 +19,14 @@ output "external_dns_irsa_role_arn" { EOF value = module.irsa_external_dns } + +output "external_deployments_operator" { + description = < [additional\_irsa\_configs](#input\_additional\_irsa\_configs) | Input for additional irsa configurations |
list(object({
name = string
namespace = string
serviceaccount_name = string
policy = string #json
}))
| `[]` | no | | [eks\_info](#input\_eks\_info) | cluster = {
specs {
name = Cluster name.
account\_id = AWS account id where the cluster resides.
}
oidc = {
arn = OIDC provider ARN.
url = OIDC provider url.
cert = {
thumbprint\_list = OIDC cert thumbprints.
url = OIDC cert URL.
}
} |
object({
nodes = object({
roles = list(object({
arn = string
name = string
}))
})
cluster = object({
specs = object({
name = string
account_id = string
})
oidc = object({
arn = string
url = string
cert = object({
thumbprint_list = list(string)
url = string
})
})
})
})
| n/a | yes | +| [external\_deployments\_operator](#input\_external\_deployments\_operator) | Config to create IRSA role for the external deployments operator. |
object({
enabled = optional(bool, false)
namespace = optional(string, "domino-compute")
service_account_name = optional(string, "pham-juno-operator")
})
| `{}` | no | | [external\_dns](#input\_external\_dns) | Config to enable irsa for external-dns |
object({
enabled = optional(bool, false)
hosted_zone_name = optional(string, null)
hosted_zone_private = optional(string, false)
namespace = optional(string, "domino-platform")
serviceaccount_name = optional(string, "external-dns")
rm_role_policy = optional(object({
remove = optional(bool, false)
detach_from_role = optional(bool, false)
policy_name = optional(string, "")
}), {})
})
| `{}` | no | | [netapp\_trident\_operator](#input\_netapp\_trident\_operator) | Config to create IRSA role for the netapp-trident-operator. |
object({
enabled = optional(bool, false)
namespace = optional(string, "trident")
serviceaccount_name = optional(string, "trident-controller")
region = optional(string)
})
| `{}` | no | | [use\_cluster\_odc\_idp](#input\_use\_cluster\_odc\_idp) | Toogle to uset the oidc idp connector in the trust policy.
Set to `true` if the cluster and the hosted zone are in different aws accounts.
`rm_role_policy` used to facilitiate the cleanup if a node attached policy was used previously. | `bool` | `true` | no | @@ -61,6 +63,7 @@ No modules. | Name | Description | |------|-------------| +| [external\_deployments\_operator](#output\_external\_deployments\_operator) | External deployments operator role info | | [external\_dns](#output\_external\_dns) | External\_dns info | | [netapp\_trident\_operator](#output\_netapp\_trident\_operator) | NetApp Astra Trident NETAPP Operator role info | | [roles](#output\_roles) | Roles mapping info | diff --git a/modules/irsa/external-deployments-operator.tf b/modules/irsa/external-deployments-operator.tf new file mode 100644 index 00000000..493e135a --- /dev/null +++ b/modules/irsa/external-deployments-operator.tf @@ -0,0 +1,23 @@ +resource "aws_iam_role" "external_deployments_operator" { + count = var.external_deployments_operator.enabled ? 1 : 0 + + name = "${local.name_prefix}-external-deployments-operator" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRoleWithWebIdentity" + Effect = "Allow" + Principal = { + Federated = local.oidc_provider_arn + } + Condition : { + StringEquals : { + "${trimprefix(local.oidc_provider_url, "https://")}:aud" : "sts.amazonaws.com", + "${trimprefix(local.oidc_provider_url, "https://")}:sub" : "system:serviceaccount:${var.external_deployments_operator.namespace}:${var.external_deployments_operator.service_account_name}" + } + } + }, + ] + }) +} diff --git a/modules/irsa/outputs.tf b/modules/irsa/outputs.tf index d4954572..f3edc24e 100644 --- a/modules/irsa/outputs.tf +++ b/modules/irsa/outputs.tf @@ -19,3 +19,11 @@ output "netapp_trident_operator" { irsa_role = aws_iam_role.trident_operator[0].arn } : null } + +output "external_deployments_operator" { + description = "External deployments operator role info" + value = var.external_deployments_operator.enabled ? { + irsa_role = aws_iam_role.external_deployments_operator[0].arn + service_account_name = var.external_deployments_operator.service_account_name + } : null +} diff --git a/modules/irsa/variables.tf b/modules/irsa/variables.tf index a0be1022..d8852122 100644 --- a/modules/irsa/variables.tf +++ b/modules/irsa/variables.tf @@ -107,3 +107,14 @@ variable "netapp_trident_operator" { default = {} } +variable "external_deployments_operator" { + description = "Config to create IRSA role for the external deployments operator." + + type = object({ + enabled = optional(bool, false) + namespace = optional(string, "domino-compute") + service_account_name = optional(string, "pham-juno-operator") + }) + + default = {} +} diff --git a/tests/deploy/cluster-ci.tfvars.tftpl b/tests/deploy/cluster-ci.tfvars.tftpl index f8623d77..fc0e5ef4 100644 --- a/tests/deploy/cluster-ci.tfvars.tftpl +++ b/tests/deploy/cluster-ci.tfvars.tftpl @@ -3,4 +3,10 @@ irsa_external_dns = { hosted_zone_name = "deploys-delta.domino.tech" } +irsa_external_deployments_operator = { + enabled = true + namespace = "domino-compute" + service_account_name = "test-operator-account" +} + use_fips_endpoint = true diff --git a/tests/deploy/meta.sh b/tests/deploy/meta.sh index c5d6bcc9..e54a95e2 100644 --- a/tests/deploy/meta.sh +++ b/tests/deploy/meta.sh @@ -12,12 +12,13 @@ NODES_VARS_TPL="${SH_DIR}/nodes-ci.tfvars.tftpl" declare -A COMP_MODS COMP_MODS["infra"]="infra" -COMP_MODS["cluster"]="eks irsa_external_dns irsa_policies" +COMP_MODS["cluster"]="eks irsa_external_dns irsa_policies irsa_external_deployments_operator" COMP_MODS["nodes"]="nodes" declare -A MOD_ADD MOD_ADD["irsa_external_dns"]="irsa" MOD_ADD["irsa_policies"]="irsa" +MOD_ADD["irsa_external_deployments_operator"]="irsa" export SH_DIR \ CI_DEPLOY \ diff --git a/tests/plan/terraform/README.md b/tests/plan/terraform/README.md index 980e5c3a..59e254ec 100644 --- a/tests/plan/terraform/README.md +++ b/tests/plan/terraform/README.md @@ -22,6 +22,7 @@ |------|--------|---------| | [eks](#module\_eks) | ./../../../modules/eks | n/a | | [infra](#module\_infra) | ./../../../modules/infra/ | n/a | +| [irsa\_external\_deployments\_operator](#module\_irsa\_external\_deployments\_operator) | ./../../../modules/irsa | n/a | | [irsa\_external\_dns](#module\_irsa\_external\_dns) | ./../../../modules/irsa | n/a | | [irsa\_policies](#module\_irsa\_policies) | ./../../../modules/irsa | n/a | | [nodes](#module\_nodes) | ./../../../modules/nodes | n/a | @@ -45,6 +46,7 @@ | [eks](#input\_eks) | k8s\_version = EKS cluster k8s version.
nodes\_master Grants the nodes role system:master access. NOT recomended
kubeconfig = {
extra\_args = Optional extra args when generating kubeconfig.
path = Fully qualified path name to write the kubeconfig file.
}
public\_access = {
enabled = Enable EKS API public endpoint.
cidrs = List of CIDR ranges permitted for accessing the EKS public endpoint.
}
Custom role maps for aws auth configmap
custom\_role\_maps = {
rolearn = string
username = string
groups = list(string)
}
master\_role\_names = IAM role names to be added as masters in eks.
cluster\_addons = EKS cluster addons. vpc-cni is installed separately.
vpc\_cni = Configuration for AWS VPC CNI
ssm\_log\_group\_name = CloudWatch log group to send the SSM session logs to.
identity\_providers = Configuration for IDP(Identity Provider).
} |
object({
k8s_version = optional(string, "1.27")
nodes_master = optional(bool, false)
kubeconfig = optional(object({
extra_args = optional(string, "")
path = optional(string, null)
}), {})
public_access = optional(object({
enabled = optional(bool, false)
cidrs = optional(list(string), [])
}), {})
custom_role_maps = optional(list(object({
rolearn = string
username = string
groups = list(string)
})), [])
master_role_names = optional(list(string), [])
cluster_addons = optional(list(string), ["kube-proxy", "coredns"])
ssm_log_group_name = optional(string, "session-manager")
vpc_cni = optional(object({
prefix_delegation = optional(bool)
annotate_pod_ip = optional(bool)
}))
identity_providers = optional(list(object({
client_id = string
groups_claim = optional(string, null)
groups_prefix = optional(string, null)
identity_provider_config_name = string
issuer_url = optional(string, null)
required_claims = optional(string, null)
username_claim = optional(string, null)
username_prefix = optional(string, null)
})), [])
})
| `{}` | no | | [enable\_private\_link](#input\_enable\_private\_link) | Enable Private Link connections | `bool` | `false` | no | | [ignore\_tags](#input\_ignore\_tags) | Tag keys to be ignored by the aws provider. | `list(string)` | `[]` | no | +| [irsa\_external\_deployments\_operator](#input\_irsa\_external\_deployments\_operator) | Config to create IRSA role for the external deployments operator. |
object({
enabled = optional(bool, false)
namespace = optional(string, "domino-compute")
service_account_name = optional(string, "pham-juno-operator")
})
| `{}` | no | | [kms](#input\_kms) | enabled = Toggle,if set use either the specified KMS key\_id or a Domino-generated one.
key\_id = optional(string, null)
additional\_policies = "Allows setting additional KMS key policies when using a Domino-generated key" |
object({
enabled = optional(bool, true)
key_id = optional(string, null)
additional_policies = optional(list(string), [])
})
| `{}` | no | | [network](#input\_network) | vpc = {
id = Existing vpc id, it will bypass creation by this module.
subnets = {
private = Existing private subnets.
public = Existing public subnets.
pod = Existing pod subnets.
}), {})
}), {})
network\_bits = {
public = Number of network bits to allocate to the public subnet. i.e /27 -> 32 IPs.
private = Number of network bits to allocate to the private subnet. i.e /19 -> 8,192 IPs.
pod = Number of network bits to allocate to the private subnet. i.e /19 -> 8,192 IPs.
}
cidrs = {
vpc = The IPv4 CIDR block for the VPC.
pod = The IPv4 CIDR block for the Pod subnets.
}
use\_pod\_cidr = Use additional pod CIDR range (ie 100.64.0.0/16) for pod networking. |
object({
vpc = optional(object({
id = optional(string, null)
subnets = optional(object({
private = optional(list(string), [])
public = optional(list(string), [])
pod = optional(list(string), [])
}), {})
}), {})
network_bits = optional(object({
public = optional(number, 27)
private = optional(number, 19)
pod = optional(number, 19)
}
), {})
cidrs = optional(object({
vpc = optional(string, "10.0.0.0/16")
pod = optional(string, "100.64.0.0/16")
}), {})
use_pod_cidr = optional(bool, true)
})
| `{}` | no | | [region](#input\_region) | AWS region for the deployment | `string` | n/a | yes | diff --git a/tests/plan/terraform/main.tf b/tests/plan/terraform/main.tf index 2b38be98..c2f285d3 100644 --- a/tests/plan/terraform/main.tf +++ b/tests/plan/terraform/main.tf @@ -69,6 +69,13 @@ module "irsa_policies" { use_fips_endpoint = var.use_fips_endpoint } +module "irsa_external_deployments_operator" { + source = "./../../../modules/irsa" + eks_info = module.eks.info + external_deployments_operator = var.irsa_external_deployments_operator + use_fips_endpoint = var.use_fips_endpoint +} + module "nodes" { source = "./../../../modules/nodes" region = module.infra.region diff --git a/tests/plan/terraform/variables.tf b/tests/plan/terraform/variables.tf index 45377957..0d94118d 100644 --- a/tests/plan/terraform/variables.tf +++ b/tests/plan/terraform/variables.tf @@ -449,3 +449,15 @@ variable "use_fips_endpoint" { type = bool default = false } + +variable "irsa_external_deployments_operator" { + description = "Config to create IRSA role for the external deployments operator." + + type = object({ + enabled = optional(bool, false) + namespace = optional(string, "domino-compute") + service_account_name = optional(string, "pham-juno-operator") + }) + + default = {} +}