Adds NetApp FSX as a filesystem storage alternative to EFS (#254)

.circleci/config.yml

@@ -6,7 +6,7 @@ parameters:
     default: "v3.11.2"
   terraform_version:
     type: string
-    default: "1.7.3"
+    default: "1.9.3"
   hcledit_version:
     type: string
     default: "0.2.9"

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 ## NOTE: Changes(rename/add/delete) to pre-commit ids need to be replicated in .github/workflows/terraform-checks.yml(GHA).
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.6.0
     hooks:
       - id: check-merge-conflict
       - id: end-of-file-fixer
@@ -13,67 +13,67 @@ repos:
       - id: circleci-validate
         args: [--org-slug, github/cerebrotech]
   - repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: 0.20.0
+    rev: 0.29.1
     hooks:
       - id: check-github-workflows
       - id: check-dependabot
       - id: check-github-actions
   - repo: https://github.com/antonbabenko/pre-commit-terraform
-    rev: v1.88.4
+    rev: v1.92.1
     hooks:
       - id: terraform_validate
         # See #4 on https://github.com/antonbabenko/pre-commit-terraform#terraform_validate
         exclude: (modules/eks/[^/]+$|modules/infra/submodules/cost-usage-report)
         args:
-          - '--hook-config=--retry-once-with-cleanup=true'
+          - "--hook-config=--retry-once-with-cleanup=true"
       - id: terraform_providers_lock
         args:
           - --tf-init-args=-upgrade
       - id: terraform_docs
         args:
-          - '--args=--lockfile=false'
-          - '--hook-config=--path-to-file=README.md'
-          - '--hook-config=--add-to-existing-file=true'
-          - '--hook-config=--create-file-if-not-exist=true'
-          - '--hook-config=--recursive.enabled=true'
-          - '--hook-config=--recursive.path=submodules'
+          - "--args=--lockfile=false"
+          - "--hook-config=--path-to-file=README.md"
+          - "--hook-config=--add-to-existing-file=true"
+          - "--hook-config=--create-file-if-not-exist=true"
+          - "--hook-config=--recursive.enabled=true"
+          - "--hook-config=--recursive.path=submodules"
       - id: terraform_fmt
       - id: terraform_tflint
         args:
-          - '--args=--config=__GIT_WORKING_DIR__/.tflint.hcl'
-          - '--args=--only=terraform_deprecated_interpolation'
-          - '--args=--only=terraform_deprecated_index'
-          - '--args=--only=terraform_unused_declarations'
-          - '--args=--only=terraform_comment_syntax'
-          - '--args=--only=terraform_documented_outputs'
-          - '--args=--only=terraform_documented_variables'
-          - '--args=--only=terraform_typed_variables'
-          - '--args=--only=terraform_module_pinned_source'
-          - '--args=--only=terraform_naming_convention'
-          - '--args=--only=terraform_required_version'
-          - '--args=--only=terraform_required_providers'
-          - '--args=--only=terraform_standard_module_structure'
-          - '--args=--only=terraform_workspace_remote'
-          - '--args=--enable-rule=aws_iam_policy_document_gov_friendly_arns'
-          - '--args=--enable-rule=aws_iam_policy_gov_friendly_arns'
-          - '--args=--enable-rule=aws_iam_role_policy_gov_friendly_arns'
+          - "--args=--config=__GIT_WORKING_DIR__/.tflint.hcl"
+          - "--args=--only=terraform_deprecated_interpolation"
+          - "--args=--only=terraform_deprecated_index"
+          - "--args=--only=terraform_unused_declarations"
+          - "--args=--only=terraform_comment_syntax"
+          - "--args=--only=terraform_documented_outputs"
+          - "--args=--only=terraform_documented_variables"
+          - "--args=--only=terraform_typed_variables"
+          - "--args=--only=terraform_module_pinned_source"
+          - "--args=--only=terraform_naming_convention"
+          - "--args=--only=terraform_required_version"
+          - "--args=--only=terraform_required_providers"
+          - "--args=--only=terraform_standard_module_structure"
+          - "--args=--only=terraform_workspace_remote"
+          - "--args=--enable-rule=aws_iam_policy_document_gov_friendly_arns"
+          - "--args=--enable-rule=aws_iam_policy_gov_friendly_arns"
+          - "--args=--enable-rule=aws_iam_role_policy_gov_friendly_arns"
       - id: terraform_checkov
         args:
-          - '--args=--compact'
-          - '--args=--quiet'
-          - '--args=--skip-check CKV_CIRCLECIPIPELINES_2,CKV_CIRCLECIPIPELINES_6,CKV2_AWS_11,CKV2_AWS_12,CKV2_AWS_6,CKV_AWS_109,CKV_AWS_111,CKV_AWS_135,CKV_AWS_144,CKV_AWS_145,CKV_AWS_158,CKV_AWS_18,CKV_AWS_184,CKV_AWS_19,CKV_AWS_21,CKV_AWS_66,CKV_AWS_88,CKV2_GHA_1,CKV_AWS_163,CKV_AWS_39,CKV_AWS_38,CKV2_AWS_61,CKV2_AWS_62,CKV_AWS_136,CKV_AWS_329,CKV_AWS_338,CKV_AWS_339,CKV_AWS_341,CKV_AWS_356,CKV2_AWS_19,CKV2_AWS_5,CKV_AWS_150,CKV_AWS_123,CKV2_AWS_65,CKV2_AWS_67'
+          - "--args=--compact"
+          - "--args=--quiet"
+          - "--args=--skip-check CKV_CIRCLECIPIPELINES_2,CKV_CIRCLECIPIPELINES_6,CKV2_AWS_11,CKV2_AWS_12,CKV2_AWS_6,CKV_AWS_109,CKV_AWS_111,CKV_AWS_135,CKV_AWS_144,CKV_AWS_145,CKV_AWS_158,CKV_AWS_18,CKV_AWS_184,CKV_AWS_19,CKV_AWS_21,CKV_AWS_66,CKV_AWS_88,CKV2_GHA_1,CKV_AWS_163,CKV_AWS_39,CKV_AWS_38,CKV2_AWS_61,CKV2_AWS_62,CKV_AWS_136,CKV_AWS_329,CKV_AWS_338,CKV_AWS_339,CKV_AWS_341,CKV_AWS_356,CKV2_AWS_19,CKV2_AWS_5,CKV_AWS_150,CKV_AWS_123,CKV2_AWS_65,CKV2_AWS_67,CKV2_AWS_57,CKV_AWS_149"
       - id: terraform_trivy
         args:
-          - '--args=--severity=HIGH,CRITICAL'
-          - '--args=--ignorefile=__GIT_WORKING_DIR__/.trivyignore'
-          - '--args=--exit-code=1'
+          - "--args=--severity=HIGH,CRITICAL"
+          - "--args=--ignorefile=__GIT_WORKING_DIR__/.trivyignore"
+          - "--args=--exit-code=1"
   - repo: local
     hooks:
       - id: check_aws_partition
         name: Check for hard coded AWS partition
         entry: ./bin/pre-commit/check-aws-partition.sh
         language: script
-        exclude: '^(bin|examples)'
+        exclude: "^(bin|examples)"
       - id: validate_iam_bootstrap
         name: Validate IAM bootstrap
         entry: ./bin/pre-commit/validate-iam-bootstrap.py

bin/pre-commit/validate-iam-bootstrap.py

@@ -6,7 +6,5 @@
 
 for bootstrap in glob("modules/iam-bootstrap/bootstrap-*.json"):
     text = json.loads(Path(bootstrap).read_text())
-    if len(json.dumps(text, separators=(",", ":"))) > 6000:
-        raise SystemExit(
-            f"{bootstrap} is over 6k characters, make sure it's under the IAM PolicySize quota (6144)"
-        )
+    if len(json.dumps(text, separators=(",", ":"))) > 6050:
+        raise SystemExit(f"{bootstrap} is over 6k characters, make sure it's under the IAM PolicySize quota (6144)")

modules/eks/README.md

@@ -44,9 +44,9 @@
 | [aws_security_group.eks_cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource |
 | [aws_security_group.eks_nodes](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource |
 | [aws_security_group_rule.bastion_eks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
-| [aws_security_group_rule.efs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
 | [aws_security_group_rule.eks_cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
 | [aws_security_group_rule.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
+| [aws_security_group_rule.shared_storage](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
 | [null_resource.kubeconfig](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
 | [terraform_data.run_k8s_pre_setup](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource |
 | [aws_caller_identity.aws_account](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
@@ -72,8 +72,9 @@
 | <a name="input_calico"></a> [calico](#input\_calico) | calico = {<br>      version = Configure the version for Calico<br>      image\_registry = Configure the image registry for Calico<br>    } | <pre>object({<br>    image_registry = optional(string, "quay.io")<br>    version        = optional(string, "v3.27.3")<br>  })</pre> | `{}` | no |
 | <a name="input_create_eks_role_arn"></a> [create\_eks\_role\_arn](#input\_create\_eks\_role\_arn) | Role arn to assume during the EKS cluster creation. | `string` | n/a | yes |
 | <a name="input_deploy_id"></a> [deploy\_id](#input\_deploy\_id) | Domino Deployment ID | `string` | n/a | yes |
-| <a name="input_efs_security_group"></a> [efs\_security\_group](#input\_efs\_security\_group) | Security Group ID for EFS | `string` | n/a | yes |
+| <a name="input_efs_security_group"></a> [efs\_security\_group](#input\_efs\_security\_group) | Security Group ID for EFS | `string` | `null` | no |
 | <a name="input_eks"></a> [eks](#input\_eks) | service\_ipv4\_cidr = CIDR for EKS cluster kubernetes\_network\_config.<br>    creation\_role\_name = Name of the role to import.<br>    k8s\_version = EKS cluster k8s version.<br>    nodes\_master  Grants the nodes role system:master access. NOT recomended<br>    kubeconfig = {<br>      extra\_args = Optional extra args when generating kubeconfig.<br>      path       = Fully qualified path name to write the kubeconfig file.<br>    }<br>    public\_access = {<br>      enabled = Enable EKS API public endpoint.<br>      cidrs   = List of CIDR ranges permitted for accessing the EKS public endpoint.<br>    }<br>    Custom role maps for aws auth configmap<br>    custom\_role\_maps = {<br>      rolearn = string<br>      username = string<br>      groups = list(string)<br>    }<br>    master\_role\_names = IAM role names to be added as masters in eks.<br>    cluster\_addons = EKS cluster addons. vpc-cni is installed separately.<br>    vpc\_cni = Configuration for AWS VPC CNI<br>    ssm\_log\_group\_name = CloudWatch log group to send the SSM session logs to.<br>    identity\_providers = Configuration for IDP(Identity Provider).<br>  } | <pre>object({<br>    service_ipv4_cidr  = optional(string, "172.20.0.0/16")<br>    creation_role_name = optional(string, null)<br>    k8s_version        = optional(string, "1.27")<br>    nodes_master       = optional(bool, false)<br>    kubeconfig = optional(object({<br>      extra_args = optional(string, "")<br>      path       = optional(string, null)<br>    }), {})<br>    public_access = optional(object({<br>      enabled = optional(bool, false)<br>      cidrs   = optional(list(string), [])<br>    }), {})<br>    custom_role_maps = optional(list(object({<br>      rolearn  = string<br>      username = string<br>      groups   = list(string)<br>    })), [])<br>    master_role_names  = optional(list(string), [])<br>    cluster_addons     = optional(list(string), ["kube-proxy", "coredns", "vpc-cni"])<br>    ssm_log_group_name = optional(string, "session-manager")<br>    vpc_cni = optional(object({<br>      prefix_delegation = optional(bool, false)<br>      annotate_pod_ip   = optional(bool, true)<br>    }))<br>    identity_providers = optional(list(object({<br>      client_id                     = string<br>      groups_claim                  = optional(string, null)<br>      groups_prefix                 = optional(string, null)<br>      identity_provider_config_name = string<br>      issuer_url                    = optional(string, null)<br>      required_claims               = optional(string, null)<br>      username_claim                = optional(string, null)<br>      username_prefix               = optional(string, null)<br>    })), []),<br>  })</pre> | `{}` | no |
+| <a name="input_fsx"></a> [fsx](#input\_fsx) | Configuration for FSX | <pre>object({<br>    astra_trident_operator_role = optional(string, null)<br>    svm = optional(object({<br>      id            = optional(string, null)<br>      management_ip = optional(string, null)<br>      nfs_ip        = optional(string, null)<br>    }), null)<br>    filesystem = optional(object({<br>      id                = optional(string, null)<br>      security_group_id = optional(string, null)<br>    }), null)<br>  })</pre> | `null` | no |
 | <a name="input_ignore_tags"></a> [ignore\_tags](#input\_ignore\_tags) | Tag keys to be ignored by the aws provider. | `list(string)` | `[]` | no |
 | <a name="input_kms_info"></a> [kms\_info](#input\_kms\_info) | key\_id  = KMS key id.<br>    key\_arn = KMS key arn.<br>    enabled = KMS key is enabled | <pre>object({<br>    key_id  = string<br>    key_arn = string<br>    enabled = bool<br>  })</pre> | n/a | yes |
 | <a name="input_network_info"></a> [network\_info](#input\_network\_info) | id = VPC ID.<br>    subnets = {<br>      public = List of public Subnets.<br>      [{<br>        name = Subnet name.<br>        subnet\_id = Subnet ud<br>        az = Subnet availability\_zone<br>        az\_id = Subnet availability\_zone\_id<br>      }]<br>      private = List of private Subnets.<br>      [{<br>        name = Subnet name.<br>        subnet\_id = Subnet ud<br>        az = Subnet availability\_zone<br>        az\_id = Subnet availability\_zone\_id<br>      }]<br>      pod = List of pod Subnets.<br>      [{<br>        name = Subnet name.<br>        subnet\_id = Subnet ud<br>        az = Subnet availability\_zone<br>        az\_id = Subnet availability\_zone\_id<br>      }]<br>    } | <pre>object({<br>    vpc_id = string<br>    subnets = object({<br>      public = list(object({<br>        name      = string<br>        subnet_id = string<br>        az        = string<br>        az_id     = string<br>      }))<br>      private = list(object({<br>        name      = string<br>        subnet_id = string<br>        az        = string<br>        az_id     = string<br>      }))<br>      pod = list(object({<br>        name      = string<br>        subnet_id = string<br>        az        = string<br>        az_id     = string<br>      }))<br>    })<br>    vpc_cidrs = optional(string, "10.0.0.0/16")<br>  })</pre> | n/a | yes |

modules/eks/node-group.tf

@@ -51,12 +51,36 @@ resource "aws_security_group_rule" "node" {
   )
 }
 
-resource "aws_security_group_rule" "efs" {
-  security_group_id        = var.efs_security_group
-  protocol                 = "tcp"
-  from_port                = 2049
-  to_port                  = 2049
-  type                     = "ingress"
-  description              = "EFS access"
+moved {
+  from = aws_security_group_rule.efs
+  to   = aws_security_group_rule.shared_storage["efs_2049_2049"]
+}
+
+### FSX
+
+locals {
+  shared_storage_type = var.fsx != null ? "fsx" : "efs"
+  inbound_rules = local.shared_storage_type == "fsx" ? {
+    rules = [
+      { protocol = "all", from_port = 0, to_port = 65535, description = "All traffic from EKS nodes." },
+    ]
+    security_group_id = var.fsx.filesystem.security_group_id
+    } : {
+    rules = [
+      { protocol = "tcp", from_port = 2049, to_port = 2049, description = "EFS access" }
+    ]
+    security_group_id = var.efs_security_group
+  }
+}
+
+resource "aws_security_group_rule" "shared_storage" {
+  for_each = { for r in local.inbound_rules.rules : "${local.shared_storage_type}_${r.from_port}_${r.to_port}" => r }
+
+  security_group_id        = local.inbound_rules.security_group_id
   source_security_group_id = aws_security_group.eks_nodes.id
+  protocol                 = each.value.protocol
+  from_port                = each.value.from_port
+  to_port                  = each.value.to_port
+  type                     = "ingress"
+  description              = each.value.description
 }

modules/eks/submodules/k8s/README.md

@@ -34,6 +34,7 @@ No modules.
 |------|-------------|------|---------|:--------:|
 | <a name="input_bastion_info"></a> [bastion\_info](#input\_bastion\_info) | user                = Bastion username.<br>    public\_ip           = Bastion public ip.<br>    security\_group\_id   = Bastion sg id.<br>    ssh\_bastion\_command = Command to ssh onto bastion. | <pre>object({<br>    user                = string<br>    public_ip           = string<br>    security_group_id   = string<br>    ssh_bastion_command = string<br>  })</pre> | n/a | yes |
 | <a name="input_eks_info"></a> [eks\_info](#input\_eks\_info) | cluster = {<br>      version           = K8s version.<br>      arn               = EKS Cluster arn.<br>      security\_group\_id = EKS Cluster security group id.<br>      endpoint          = EKS Cluster API endpoint.<br>      roles             = Default IAM Roles associated with the EKS cluster. {<br>        name = string<br>        arn = string<br>      }<br>      custom\_roles      = Custom IAM Roles associated with the EKS cluster. {<br>        rolearn  = string<br>        username = string<br>        groups   = list(string)<br>      }<br>      oidc = {<br>        arn = OIDC provider ARN.<br>        url = OIDC provider url.<br>      }<br>    }<br>    nodes = {<br>      security\_group\_id = EKS Nodes security group id.<br>      roles = IAM Roles associated with the EKS Nodes.{<br>        name = string<br>        arn  = string<br>      }<br>    }<br>    kubeconfig = Kubeconfig details.{<br>      path       = string<br>      extra\_args = string<br>    }<br>    calico = {<br>      version = Configuration the version for Calico<br>      image\_registry = Configure the image registry for Calico<br>    } | <pre>object({<br>    cluster = object({<br>      version           = string<br>      arn               = string<br>      security_group_id = string<br>      endpoint          = string<br>      roles = list(object({<br>        name = string<br>        arn  = string<br>      }))<br>      custom_roles = list(object({<br>        rolearn  = string<br>        username = string<br>        groups   = list(string)<br>      }))<br>      oidc = object({<br>        arn = string<br>        url = string<br>      })<br>    })<br>    nodes = object({<br>      nodes_master      = bool<br>      security_group_id = string<br>      roles = list(object({<br>        name = string<br>        arn  = string<br>      }))<br>    })<br>    kubeconfig = object({<br>      path       = string<br>      extra_args = string<br>    })<br>    calico = object({<br>      version        = string<br>      image_registry = string<br>    })<br>  })</pre> | n/a | yes |
+| <a name="input_fsx"></a> [fsx](#input\_fsx) | Configuration for FSx | <pre>object({<br>    astra_trident_operator_role = optional(string, null)<br>    svm = optional(object({<br>      id            = optional(string, null)<br>      management_ip = optional(string, null)<br>      nfs_ip        = optional(string, null)<br>    }), null)<br>    filesystem = optional(object({<br>      id = optional(string, null)<br>    }), null)<br>  })</pre> | `null` | no |
 | <a name="input_ssh_key"></a> [ssh\_key](#input\_ssh\_key) | path          = SSH private key filepath.<br>    key\_pair\_name = AWS key\_pair name. | <pre>object({<br>    path          = string<br>    key_pair_name = string<br>  })</pre> | n/a | yes |
 | <a name="input_use_fips_endpoint"></a> [use\_fips\_endpoint](#input\_use\_fips\_endpoint) | Use aws FIPS endpoints | `bool` | `false` | no |
 

modules/eks/submodules/k8s/main.tf

@@ -26,6 +26,7 @@ locals {
       calico_version        = var.eks_info.calico.version
       calico_fips_mode      = var.use_fips_endpoint ? "Enabled" : "Disabled"
       calico_image_registry = var.eks_info.calico.image_registry
+      fsx                   = var.fsx
     })
   }
 

modules/eks/submodules/k8s/variables.tf

@@ -105,3 +105,19 @@ variable "use_fips_endpoint" {
   type        = bool
   default     = false
 }
+
+variable "fsx" {
+  description = "Configuration for FSx"
+  type = object({
+    astra_trident_operator_role = optional(string, null)
+    svm = optional(object({
+      id            = optional(string, null)
+      management_ip = optional(string, null)
+      nfs_ip        = optional(string, null)
+    }), null)
+    filesystem = optional(object({
+      id = optional(string, null)
+    }), null)
+  })
+  default = null
+}