pull through cache for calico on quay (#245)

* pull through cache for calico on quay

* update to disable in pull through cache in non-AWS partition and in FIPS mode

* fix policy inclusion

* address PR comments

* simplify change triggers for setup scripts

examples/deploy/terraform/cluster/main.tf

@@ -27,6 +27,7 @@ module "eks" {
   tags                = local.infra.tags
   ignore_tags         = local.infra.ignore_tags
   use_fips_endpoint   = var.use_fips_endpoint
+  calico              = { image_registry = try(local.infra.storage.ecr.calico_image_registry, null) }
 }
 
 data "aws_caller_identity" "global" {

modules/eks/README.md

@@ -69,10 +69,11 @@
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_bastion_info"></a> [bastion\_info](#input\_bastion\_info) | user                = Bastion username.<br>    public\_ip           = Bastion public ip.<br>    security\_group\_id   = Bastion sg id.<br>    ssh\_bastion\_command = Command to ssh onto bastion. | <pre>object({<br>    user                = string<br>    public_ip           = string<br>    security_group_id   = string<br>    ssh_bastion_command = string<br>  })</pre> | n/a | yes |
+| <a name="input_calico"></a> [calico](#input\_calico) | calico = {<br>      version = Configure the version for Calico<br>      image\_registry = Configure the image registry for Calico<br>    } | <pre>object({<br>    image_registry = optional(string, "quay.io")<br>    version        = optional(string, "v3.27.3")<br>  })</pre> | `{}` | no |
 | <a name="input_create_eks_role_arn"></a> [create\_eks\_role\_arn](#input\_create\_eks\_role\_arn) | Role arn to assume during the EKS cluster creation. | `string` | n/a | yes |
 | <a name="input_deploy_id"></a> [deploy\_id](#input\_deploy\_id) | Domino Deployment ID | `string` | n/a | yes |
 | <a name="input_efs_security_group"></a> [efs\_security\_group](#input\_efs\_security\_group) | Security Group ID for EFS | `string` | n/a | yes |
-| <a name="input_eks"></a> [eks](#input\_eks) | service\_ipv4\_cidr = CIDR for EKS cluster kubernetes\_network\_config.<br>    creation\_role\_name = Name of the role to import.<br>    k8s\_version = EKS cluster k8s version.<br>    nodes\_master  Grants the nodes role system:master access. NOT recomended<br>    kubeconfig = {<br>      extra\_args = Optional extra args when generating kubeconfig.<br>      path       = Fully qualified path name to write the kubeconfig file.<br>    }<br>    public\_access = {<br>      enabled = Enable EKS API public endpoint.<br>      cidrs   = List of CIDR ranges permitted for accessing the EKS public endpoint.<br>    }<br>    Custom role maps for aws auth configmap<br>    custom\_role\_maps = {<br>      rolearn = string<br>      username = string<br>      groups = list(string)<br>    }<br>    master\_role\_names = IAM role names to be added as masters in eks.<br>    cluster\_addons = EKS cluster addons. vpc-cni is installed separately.<br>    vpc\_cni = Configuration for AWS VPC CNI<br>    ssm\_log\_group\_name = CloudWatch log group to send the SSM session logs to.<br>    identity\_providers = Configuration for IDP(Identity Provider).<br>  } | <pre>object({<br>    service_ipv4_cidr  = optional(string, "172.20.0.0/16")<br>    creation_role_name = optional(string, null)<br>    k8s_version        = optional(string, "1.27")<br>    nodes_master       = optional(bool, false)<br>    kubeconfig = optional(object({<br>      extra_args = optional(string, "")<br>      path       = optional(string, null)<br>    }), {})<br>    public_access = optional(object({<br>      enabled = optional(bool, false)<br>      cidrs   = optional(list(string), [])<br>    }), {})<br>    custom_role_maps = optional(list(object({<br>      rolearn  = string<br>      username = string<br>      groups   = list(string)<br>    })), [])<br>    master_role_names  = optional(list(string), [])<br>    cluster_addons     = optional(list(string), ["kube-proxy", "coredns", "vpc-cni"])<br>    ssm_log_group_name = optional(string, "session-manager")<br>    vpc_cni = optional(object({<br>      prefix_delegation = optional(bool, false)<br>      annotate_pod_ip   = optional(bool, true)<br>    }))<br>    identity_providers = optional(list(object({<br>      client_id                     = string<br>      groups_claim                  = optional(string, null)<br>      groups_prefix                 = optional(string, null)<br>      identity_provider_config_name = string<br>      issuer_url                    = optional(string, null)<br>      required_claims               = optional(string, null)<br>      username_claim                = optional(string, null)<br>      username_prefix               = optional(string, null)<br>    })), [])<br>  })</pre> | `{}` | no |
+| <a name="input_eks"></a> [eks](#input\_eks) | service\_ipv4\_cidr = CIDR for EKS cluster kubernetes\_network\_config.<br>    creation\_role\_name = Name of the role to import.<br>    k8s\_version = EKS cluster k8s version.<br>    nodes\_master  Grants the nodes role system:master access. NOT recomended<br>    kubeconfig = {<br>      extra\_args = Optional extra args when generating kubeconfig.<br>      path       = Fully qualified path name to write the kubeconfig file.<br>    }<br>    public\_access = {<br>      enabled = Enable EKS API public endpoint.<br>      cidrs   = List of CIDR ranges permitted for accessing the EKS public endpoint.<br>    }<br>    Custom role maps for aws auth configmap<br>    custom\_role\_maps = {<br>      rolearn = string<br>      username = string<br>      groups = list(string)<br>    }<br>    master\_role\_names = IAM role names to be added as masters in eks.<br>    cluster\_addons = EKS cluster addons. vpc-cni is installed separately.<br>    vpc\_cni = Configuration for AWS VPC CNI<br>    ssm\_log\_group\_name = CloudWatch log group to send the SSM session logs to.<br>    identity\_providers = Configuration for IDP(Identity Provider).<br>  } | <pre>object({<br>    service_ipv4_cidr  = optional(string, "172.20.0.0/16")<br>    creation_role_name = optional(string, null)<br>    k8s_version        = optional(string, "1.27")<br>    nodes_master       = optional(bool, false)<br>    kubeconfig = optional(object({<br>      extra_args = optional(string, "")<br>      path       = optional(string, null)<br>    }), {})<br>    public_access = optional(object({<br>      enabled = optional(bool, false)<br>      cidrs   = optional(list(string), [])<br>    }), {})<br>    custom_role_maps = optional(list(object({<br>      rolearn  = string<br>      username = string<br>      groups   = list(string)<br>    })), [])<br>    master_role_names  = optional(list(string), [])<br>    cluster_addons     = optional(list(string), ["kube-proxy", "coredns", "vpc-cni"])<br>    ssm_log_group_name = optional(string, "session-manager")<br>    vpc_cni = optional(object({<br>      prefix_delegation = optional(bool, false)<br>      annotate_pod_ip   = optional(bool, true)<br>    }))<br>    identity_providers = optional(list(object({<br>      client_id                     = string<br>      groups_claim                  = optional(string, null)<br>      groups_prefix                 = optional(string, null)<br>      identity_provider_config_name = string<br>      issuer_url                    = optional(string, null)<br>      required_claims               = optional(string, null)<br>      username_claim                = optional(string, null)<br>      username_prefix               = optional(string, null)<br>    })), []),<br>  })</pre> | `{}` | no |
 | <a name="input_ignore_tags"></a> [ignore\_tags](#input\_ignore\_tags) | Tag keys to be ignored by the aws provider. | `list(string)` | `[]` | no |
 | <a name="input_kms_info"></a> [kms\_info](#input\_kms\_info) | key\_id  = KMS key id.<br>    key\_arn = KMS key arn.<br>    enabled = KMS key is enabled | <pre>object({<br>    key_id  = string<br>    key_arn = string<br>    enabled = bool<br>  })</pre> | n/a | yes |
 | <a name="input_network_info"></a> [network\_info](#input\_network\_info) | id = VPC ID.<br>    subnets = {<br>      public = List of public Subnets.<br>      [{<br>        name = Subnet name.<br>        subnet\_id = Subnet ud<br>        az = Subnet availability\_zone<br>        az\_id = Subnet availability\_zone\_id<br>      }]<br>      private = List of private Subnets.<br>      [{<br>        name = Subnet name.<br>        subnet\_id = Subnet ud<br>        az = Subnet availability\_zone<br>        az\_id = Subnet availability\_zone\_id<br>      }]<br>      pod = List of pod Subnets.<br>      [{<br>        name = Subnet name.<br>        subnet\_id = Subnet ud<br>        az = Subnet availability\_zone<br>        az\_id = Subnet availability\_zone\_id<br>      }]<br>    } | <pre>object({<br>    vpc_id = string<br>    subnets = object({<br>      public = list(object({<br>        name      = string<br>        subnet_id = string<br>        az        = string<br>        az_id     = string<br>      }))<br>      private = list(object({<br>        name      = string<br>        subnet_id = string<br>        az        = string<br>        az_id     = string<br>      }))<br>      pod = list(object({<br>        name      = string<br>        subnet_id = string<br>        az        = string<br>        az_id     = string<br>      }))<br>    })<br>    vpc_cidrs = optional(string, "10.0.0.0/16")<br>  })</pre> | n/a | yes |

modules/eks/main.tf

@@ -252,5 +252,6 @@ locals {
       }]
     }
     kubeconfig = local.kubeconfig
+    calico     = var.calico
   }
 }

modules/eks/submodules/k8s/README.md

@@ -33,8 +33,7 @@ No modules.
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_bastion_info"></a> [bastion\_info](#input\_bastion\_info) | user                = Bastion username.<br>    public\_ip           = Bastion public ip.<br>    security\_group\_id   = Bastion sg id.<br>    ssh\_bastion\_command = Command to ssh onto bastion. | <pre>object({<br>    user                = string<br>    public_ip           = string<br>    security_group_id   = string<br>    ssh_bastion_command = string<br>  })</pre> | n/a | yes |
-| <a name="input_calico_version"></a> [calico\_version](#input\_calico\_version) | Calico operator version. | `string` | `"v3.25.2"` | no |
-| <a name="input_eks_info"></a> [eks\_info](#input\_eks\_info) | cluster = {<br>      version           = K8s version.<br>      arn               = EKS Cluster arn.<br>      security\_group\_id = EKS Cluster security group id.<br>      endpoint          = EKS Cluster API endpoint.<br>      roles             = Default IAM Roles associated with the EKS cluster. {<br>        name = string<br>        arn = string<br>      }<br>      custom\_roles      = Custom IAM Roles associated with the EKS cluster. {<br>        rolearn  = string<br>        username = string<br>        groups   = list(string)<br>      }<br>      oidc = {<br>        arn = OIDC provider ARN.<br>        url = OIDC provider url.<br>      }<br>    }<br>    nodes = {<br>      security\_group\_id = EKS Nodes security group id.<br>      roles = IAM Roles associated with the EKS Nodes.{<br>        name = string<br>        arn  = string<br>      }<br>    }<br>    kubeconfig = Kubeconfig details.{<br>      path       = string<br>      extra\_args = string<br>    } | <pre>object({<br>    cluster = object({<br>      version           = string<br>      arn               = string<br>      security_group_id = string<br>      endpoint          = string<br>      roles = list(object({<br>        name = string<br>        arn  = string<br>      }))<br>      custom_roles = list(object({<br>        rolearn  = string<br>        username = string<br>        groups   = list(string)<br>      }))<br>      oidc = object({<br>        arn = string<br>        url = string<br>      })<br>    })<br>    nodes = object({<br>      nodes_master      = bool<br>      security_group_id = string<br>      roles = list(object({<br>        name = string<br>        arn  = string<br>      }))<br>    })<br>    kubeconfig = object({<br>      path       = string<br>      extra_args = string<br>    })<br>  })</pre> | n/a | yes |
+| <a name="input_eks_info"></a> [eks\_info](#input\_eks\_info) | cluster = {<br>      version           = K8s version.<br>      arn               = EKS Cluster arn.<br>      security\_group\_id = EKS Cluster security group id.<br>      endpoint          = EKS Cluster API endpoint.<br>      roles             = Default IAM Roles associated with the EKS cluster. {<br>        name = string<br>        arn = string<br>      }<br>      custom\_roles      = Custom IAM Roles associated with the EKS cluster. {<br>        rolearn  = string<br>        username = string<br>        groups   = list(string)<br>      }<br>      oidc = {<br>        arn = OIDC provider ARN.<br>        url = OIDC provider url.<br>      }<br>    }<br>    nodes = {<br>      security\_group\_id = EKS Nodes security group id.<br>      roles = IAM Roles associated with the EKS Nodes.{<br>        name = string<br>        arn  = string<br>      }<br>    }<br>    kubeconfig = Kubeconfig details.{<br>      path       = string<br>      extra\_args = string<br>    }<br>    calico = {<br>      version = Configuration the version for Calico<br>      image\_registry = Configure the image registry for Calico<br>    } | <pre>object({<br>    cluster = object({<br>      version           = string<br>      arn               = string<br>      security_group_id = string<br>      endpoint          = string<br>      roles = list(object({<br>        name = string<br>        arn  = string<br>      }))<br>      custom_roles = list(object({<br>        rolearn  = string<br>        username = string<br>        groups   = list(string)<br>      }))<br>      oidc = object({<br>        arn = string<br>        url = string<br>      })<br>    })<br>    nodes = object({<br>      nodes_master      = bool<br>      security_group_id = string<br>      roles = list(object({<br>        name = string<br>        arn  = string<br>      }))<br>    })<br>    kubeconfig = object({<br>      path       = string<br>      extra_args = string<br>    })<br>    calico = object({<br>      version        = string<br>      image_registry = string<br>    })<br>  })</pre> | n/a | yes |
 | <a name="input_ssh_key"></a> [ssh\_key](#input\_ssh\_key) | path          = SSH private key filepath.<br>    key\_pair\_name = AWS key\_pair name. | <pre>object({<br>    path          = string<br>    key_pair_name = string<br>  })</pre> | n/a | yes |
 | <a name="input_use_fips_endpoint"></a> [use\_fips\_endpoint](#input\_use\_fips\_endpoint) | Use aws FIPS endpoints | `bool` | `false` | no |
 

modules/eks/submodules/k8s/main.tf

@@ -13,41 +13,45 @@ locals {
   resources_directory       = path.cwd
   templates_dir             = "${path.module}/templates"
 
+  k8s_functions_sh = {
+    filename = local.k8s_functions_sh_filename
+    content = templatefile("${local.templates_dir}/${local.k8s_functions_sh_template}", {
+      kubeconfig_path       = var.eks_info.kubeconfig.path
+      k8s_tunnel_port       = random_integer.port.result
+      aws_auth_yaml         = basename(local.aws_auth_filename)
+      ssh_pvt_key_path      = var.ssh_key.path
+      eks_cluster_arn       = var.eks_info.cluster.arn
+      bastion_user          = var.bastion_info != null ? var.bastion_info.user : ""
+      bastion_public_ip     = var.bastion_info != null ? var.bastion_info.public_ip : ""
+      calico_version        = var.eks_info.calico.version
+      calico_fips_mode      = var.use_fips_endpoint ? "Enabled" : "Disabled"
+      calico_image_registry = var.eks_info.calico.image_registry
+    })
+  }
+
+  aws_auth = {
+    filename = local.aws_auth_filename
+    content = templatefile("${local.templates_dir}/${local.aws_auth_template}",
+      {
+        nodes_master         = try(var.eks_info.nodes.nodes_master, false)
+        eks_node_role_arns   = toset(var.eks_info.nodes.roles[*].arn)
+        eks_master_role_arns = toset(var.eks_info.cluster.roles[*].arn)
+        eks_custom_role_maps = var.eks_info.cluster.custom_roles
+    })
+  }
+
   templates = {
-    k8s_functions_sh = {
-      filename = local.k8s_functions_sh_filename
-      content = templatefile("${local.templates_dir}/${local.k8s_functions_sh_template}", {
-        kubeconfig_path   = var.eks_info.kubeconfig.path
-        k8s_tunnel_port   = random_integer.port.result
-        aws_auth_yaml     = basename(local.aws_auth_filename)
-        ssh_pvt_key_path  = var.ssh_key.path
-        eks_cluster_arn   = var.eks_info.cluster.arn
-        calico_version    = var.calico_version
-        bastion_user      = var.bastion_info != null ? var.bastion_info.user : ""
-        bastion_public_ip = var.bastion_info != null ? var.bastion_info.public_ip : ""
-        calico_fips_mode  = var.use_fips_endpoint ? "Enabled" : "Disabled"
-      })
-    }
+    k8s_functions_sh = local.k8s_functions_sh
+    aws_auth         = local.aws_auth
 
     k8s_presetup = {
       filename = local.k8s_pre_setup_sh_file
       content = templatefile("${local.templates_dir}/${local.k8s_pre_setup_sh_template}", {
-        k8s_functions_sh_filename = local.k8s_functions_sh_filename
+        k8s_functions_sh_filename = local.k8s_functions_sh.filename
+        hash                      = join("-", [md5(local.k8s_functions_sh.content), md5(local.aws_auth.content)])
         use_fips_endpoint         = tostring(var.use_fips_endpoint)
       })
     }
-
-    aws_auth = {
-      filename = local.aws_auth_filename
-      content = templatefile("${local.templates_dir}/${local.aws_auth_template}",
-        {
-          nodes_master         = try(var.eks_info.nodes.nodes_master, false)
-          eks_node_role_arns   = toset(var.eks_info.nodes.roles[*].arn)
-          eks_master_role_arns = toset(var.eks_info.cluster.roles[*].arn)
-          eks_custom_role_maps = var.eks_info.cluster.custom_roles
-      })
-
-    }
   }
 }
 

modules/eks/submodules/k8s/templates/k8s-functions.sh.tftpl

@@ -119,8 +119,9 @@ install_calico() {
       --namespace "tigera-operator" \
       --set installation.kubernetesProvider=EKS \
       --set installation.cni.type=AmazonVPC \
-      --set installation.registry="quay.io/" \
+      --set installation.registry="${calico_image_registry}/" \
       --set installation.fipsMode="${calico_fips_mode}" \
+      --wait \
       --timeout 10m \
       --create-namespace \
       --install

modules/eks/submodules/k8s/templates/k8s-pre-setup.sh.tftpl

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+# ${hash}
+
 source ${k8s_functions_sh_filename}
 export AWS_USE_FIPS_ENDPOINT=${use_fips_endpoint}
 

modules/eks/submodules/k8s/variables.tf

@@ -1,9 +1,3 @@
-variable "calico_version" {
-  type        = string
-  description = "Calico operator version."
-  default     = "v3.25.2"
-}
-
 variable "bastion_info" {
   description = <<EOF
     user                = Bastion username.
@@ -62,6 +56,10 @@ variable "eks_info" {
       path       = string
       extra_args = string
     }
+    calico = {
+      version = Configuration the version for Calico
+      image_registry = Configure the image registry for Calico
+    }
   EOF
   type = object({
     cluster = object({
@@ -95,6 +93,10 @@ variable "eks_info" {
       path       = string
       extra_args = string
     })
+    calico = object({
+      version        = string
+      image_registry = string
+    })
   })
 }
 

modules/eks/variables.tf

@@ -180,7 +180,7 @@ variable "eks" {
       required_claims               = optional(string, null)
       username_claim                = optional(string, null)
       username_prefix               = optional(string, null)
-    })), [])
+    })), []),
   })
 
   default = {}
@@ -257,3 +257,19 @@ variable "use_fips_endpoint" {
   type        = bool
   default     = false
 }
+
+variable "calico" {
+  description = <<EOF
+    calico = {
+      version = Configure the version for Calico
+      image_registry = Configure the image registry for Calico
+    }
+  EOF
+
+  type = object({
+    image_registry = optional(string, "quay.io")
+    version        = optional(string, "v3.27.3")
+  })
+
+  default = {}
+}

modules/iam-bootstrap/bootstrap-1.json

@@ -141,6 +141,15 @@
       ],
       "Resource": ["*"]
     },
+    {
+      "Sid": "ECRUngated",
+      "Effect": "Allow",
+      "Action": [
+        "ecr:CreatePullThroughCacheRule",
+        "ecr:DescribePullThroughCacheRules"
+      ],
+      "Resource": ["*"]
+    },
     {
       "Sid": "ECRGated",
       "Effect": "Allow",
@@ -186,6 +195,7 @@
         "ec2:DeleteFlowLogs",
         "ec2:DeleteLaunchTemplate",
         "ec2:DeleteSecurityGroup",
+        "ecr:DeletePullThroughCacheRule",
         "ecr:DeleteRepository",
         "eks:DeleteCluster",
         "eks:DeleteNodegroup",