Refactor split submodule scripts (#79)

* Refactor the split bash script orchestration * precommit * Add descriptions to new outputs * pwd/ls to debug weird path problem in circle * Sorry, this one * Make example easy to run w/unique name * Less silly hash generation * Switch to terraform_data
dominodatalab · Apr 25, 2023 · 35f0247 · 35f0247
1 parent 80f8bfc
commit 35f0247
Show file tree

Hide file tree

Showing 11 changed files with 74 additions and 63 deletions.
diff --git a/examples/minimal-with-bastion/README.md b/examples/minimal-with-bastion/README.md
@@ -43,6 +43,7 @@ No resources.
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
+| <a name="input_deploy_id"></a> [deploy\_id](#input\_deploy\_id) | Unique name for deployment | `string` | `"dominoeks004"` | no |
 | <a name="input_region"></a> [region](#input\_region) | AWS region for deployment. | `string` | `"us-west-2"` | no |
 
 ## Outputs

diff --git a/examples/minimal-with-bastion/main.tf b/examples/minimal-with-bastion/main.tf
@@ -3,7 +3,7 @@ module "domino_eks" {
   source           = "./../.."
   region           = var.region
   ssh_pvt_key_path = "./../examples.pem"
-  deploy_id        = "dominoeks004"
+  deploy_id        = var.deploy_id
   default_node_groups = {
     compute = {
       availability_zone_ids = ["usw2-az1", "usw2-az2"]

diff --git a/examples/minimal-with-bastion/variables.tf b/examples/minimal-with-bastion/variables.tf
@@ -3,3 +3,9 @@ variable "region" {
   type        = string
   default     = "us-west-2"
 }
+
+variable "deploy_id" {
+  description = "Unique name for deployment"
+  type        = string
+  default     = "dominoeks004"
+}
diff --git a/submodules/eks/README.md b/submodules/eks/README.md
@@ -15,12 +15,12 @@
 |------|---------|
 | <a name="provider_aws"></a> [aws](#provider\_aws) | >= 4.0 |
 | <a name="provider_null"></a> [null](#provider\_null) | >= 3.1.0 |
+| <a name="provider_terraform"></a> [terraform](#provider\_terraform) | n/a |
 
 ## Modules
 
 | Name | Source | Version |
 |------|--------|---------|
-| <a name="module_calico_setup"></a> [calico\_setup](#module\_calico\_setup) | ../k8s | n/a |
 | <a name="module_k8s_setup"></a> [k8s\_setup](#module\_k8s\_setup) | ../k8s | n/a |
 
 ## Resources
@@ -48,6 +48,8 @@
 | [aws_security_group_rule.eks_cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
 | [aws_security_group_rule.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
 | [null_resource.kubeconfig](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
+| [terraform_data.calico_setup](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource |
+| [terraform_data.run_k8s_pre_setup](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource |
 | [aws_ami.custom](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source |
 | [aws_caller_identity.aws_account](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
 | [aws_ec2_instance_type_offerings.nodes](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type_offerings) | data source |

diff --git a/submodules/eks/k8s.tf b/submodules/eks/k8s.tf
@@ -1,25 +1,47 @@
+locals {
+  run_setup = var.bastion_info != null || var.eks.public_access.enabled ? 1 : 0
+}
+
 module "k8s_setup" {
-  count = var.bastion_info != null || var.eks.public_access.enabled ? 1 : 0
+  count = local.run_setup
 
-  source        = "../k8s"
-  ssh_key       = var.ssh_key
-  bastion_info  = var.bastion_info
-  network_info  = var.network_info
-  eks_info      = local.eks_info
-  cluster_setup = true
+  source       = "../k8s"
+  ssh_key      = var.ssh_key
+  bastion_info = var.bastion_info
+  network_info = var.network_info
+  eks_info     = local.eks_info
 
   depends_on = [aws_eks_addon.vpc_cni, null_resource.kubeconfig]
 }
 
-module "calico_setup" {
-  count = var.bastion_info != null || var.eks.public_access.enabled ? 1 : 0
+resource "terraform_data" "run_k8s_pre_setup" {
+  count = local.run_setup
+
+  triggers_replace = [
+    module.k8s_setup[0].change_hash
+  ]
+
+  provisioner "local-exec" {
+    command     = "./${module.k8s_setup[0].filename} set_k8s_auth set_eniconfig"
+    interpreter = ["bash", "-c"]
+    working_dir = module.k8s_setup[0].resources_directory
+  }
+
+  depends_on = [module.k8s_setup]
+}
+
+resource "terraform_data" "calico_setup" {
+  count = local.run_setup
+
+  triggers_replace = [
+    module.k8s_setup[0].change_hash
+  ]
 
-  source         = "../k8s"
-  ssh_key        = var.ssh_key
-  bastion_info   = var.bastion_info
-  network_info   = var.network_info
-  eks_info       = local.eks_info
-  install_calico = true
+  provisioner "local-exec" {
+    command     = "./${module.k8s_setup[0].filename} install_calico"
+    interpreter = ["bash", "-c"]
+    working_dir = module.k8s_setup[0].resources_directory
+  }
 
-  depends_on = [aws_eks_node_group.node_groups, module.k8s_setup]
+  depends_on = [aws_eks_node_group.node_groups, terraform_data.run_k8s_pre_setup]
 }
diff --git a/submodules/eks/node-group.tf b/submodules/eks/node-group.tf
@@ -179,7 +179,7 @@ data "aws_ssm_parameter" "eks_gpu_ami_release_version" {
 }
 
 resource "aws_eks_node_group" "node_groups" {
-  depends_on           = [module.k8s_setup]
+  depends_on           = [terraform_data.run_k8s_pre_setup]
   for_each             = local.node_groups_by_name
   cluster_name         = aws_eks_cluster.this.name
   version              = each.value.node_group.ami != null ? null : aws_eks_cluster.this.version

diff --git a/submodules/k8s/README.md b/submodules/k8s/README.md
@@ -16,7 +16,6 @@
 | Name | Version |
 |------|---------|
 | <a name="provider_local"></a> [local](#provider\_local) | >= 2.2.0 |
-| <a name="provider_null"></a> [null](#provider\_null) | >= 3.1.0 |
 | <a name="provider_random"></a> [random](#provider\_random) | >= 3.4.3 |
 
 ## Modules
@@ -28,7 +27,6 @@ No modules.
 | Name | Type |
 |------|------|
 | [local_file.templates](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
-| [null_resource.run_k8s_pre_setup](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
 | [random_integer.port](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/integer) | resource |
 
 ## Inputs
@@ -37,13 +35,15 @@ No modules.
 |------|-------------|------|---------|:--------:|
 | <a name="input_bastion_info"></a> [bastion\_info](#input\_bastion\_info) | user                = Bastion username.<br>    public\_ip           = Bastion public ip.<br>    security\_group\_id   = Bastion sg id.<br>    ssh\_bastion\_command = Command to ssh onto bastion. | <pre>object({<br>    user                = string<br>    public_ip           = string<br>    security_group_id   = string<br>    ssh_bastion_command = string<br>  })</pre> | n/a | yes |
 | <a name="input_calico_version"></a> [calico\_version](#input\_calico\_version) | Calico operator version. | `string` | `"v3.25.0"` | no |
-| <a name="input_cluster_setup"></a> [cluster\_setup](#input\_cluster\_setup) | Perform cluster setup functions | `bool` | `false` | no |
 | <a name="input_eks_info"></a> [eks\_info](#input\_eks\_info) | cluster = {<br>      arn               = EKS Cluster arn.<br>      security\_group\_id = EKS Cluster security group id.<br>      endpoint          = EKS Cluster API endpoint.<br>      roles             = Default IAM Roles associated with the EKS cluster. {<br>        name = string<br>        arn = string<br>      }<br>      custom\_roles      = Custom IAM Roles associated with the EKS cluster. {<br>        rolearn  = string<br>        username = string<br>        groups   = list(string)<br>      }<br>    }<br>    nodes = {<br>      security\_group\_id = EKS Nodes security group id.<br>      roles = IAM Roles associated with the EKS Nodes.{<br>        name = string<br>        arn  = string<br>      }<br>    }<br>    kubeconfig = Kubeconfig details.{<br>      path       = string<br>      extra\_args = string<br>    } | <pre>object({<br>    cluster = object({<br>      arn               = string<br>      security_group_id = string<br>      endpoint          = string<br>      roles = list(object({<br>        name = string<br>        arn  = string<br>      }))<br>      custom_roles = list(object({<br>        rolearn  = string<br>        username = string<br>        groups   = list(string)<br>      }))<br>    })<br>    nodes = object({<br>      security_group_id = string<br>      roles = list(object({<br>        name = string<br>        arn  = string<br>      }))<br>    })<br>    kubeconfig = object({<br>      path       = string<br>      extra_args = string<br>    })<br>  })</pre> | n/a | yes |
-| <a name="input_install_calico"></a> [install\_calico](#input\_install\_calico) | Perform calico install functions | `bool` | `false` | no |
 | <a name="input_network_info"></a> [network\_info](#input\_network\_info) | id = VPC ID.<br>    subnets = {<br>      public = List of public Subnets.<br>      [{<br>        name = Subnet name.<br>        subnet\_id = Subnet ud<br>        az = Subnet availability\_zone<br>        az\_id = Subnet availability\_zone\_id<br>      }]<br>      private = List of private Subnets.<br>      [{<br>        name = Subnet name.<br>        subnet\_id = Subnet ud<br>        az = Subnet availability\_zone<br>        az\_id = Subnet availability\_zone\_id<br>      }]<br>      pod = List of pod Subnets.<br>      [{<br>        name = Subnet name.<br>        subnet\_id = Subnet ud<br>        az = Subnet availability\_zone<br>        az\_id = Subnet availability\_zone\_id<br>      }]<br>    } | <pre>object({<br>    vpc_id = string<br>    subnets = object({<br>      public = list(object({<br>        name      = string<br>        subnet_id = string<br>        az        = string<br>        az_id     = string<br>      }))<br>      private = optional(list(object({<br>        name      = string<br>        subnet_id = string<br>        az        = string<br>        az_id     = string<br>      })), [])<br>      pod = optional(list(object({<br>        name      = string<br>        subnet_id = string<br>        az        = string<br>        az_id     = string<br>      })), [])<br>    })<br>  })</pre> | n/a | yes |
 | <a name="input_ssh_key"></a> [ssh\_key](#input\_ssh\_key) | path          = SSH private key filepath.<br>    key\_pair\_name = AWS key\_pair name. | <pre>object({<br>    path          = string<br>    key_pair_name = string<br>  })</pre> | n/a | yes |
 
 ## Outputs
 
-No outputs.
+| Name | Description |
+|------|-------------|
+| <a name="output_change_hash"></a> [change\_hash](#output\_change\_hash) | Hash of all templated files |
+| <a name="output_filename"></a> [filename](#output\_filename) | Filename of primary script |
+| <a name="output_resources_directory"></a> [resources\_directory](#output\_resources\_directory) | Directory for provisioned scripts and templated files |
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
diff --git a/submodules/k8s/main.tf b/submodules/k8s/main.tf
@@ -35,8 +35,6 @@ locals {
       filename = local.k8s_pre_setup_sh_filename
       content = templatefile("${local.templates_dir}/${local.k8s_pre_setup_sh_template}", {
         k8s_functions_sh_filename = local.k8s_functions_sh_filename
-        cluster_setup             = var.cluster_setup
-        install_calico            = var.install_calico
       })
     }
 
@@ -70,21 +68,6 @@ resource "local_file" "templates" {
   file_permission      = "0744"
 }
 
-resource "null_resource" "run_k8s_pre_setup" {
-  triggers = {
-    k8s_presetup_hash     = md5(local_file.templates["k8s_presetup"].content)
-    k8s_functions_sh_hash = md5(local_file.templates["k8s_functions_sh"].content)
-    aws_auth_hash         = md5(local_file.templates["aws_auth"].content)
-    eni_config_hash       = try(md5(local_file.templates["eni_config"].content), "none")
-  }
-
-  provisioner "local-exec" {
-    command     = basename(local_file.templates["k8s_presetup"].filename)
-    interpreter = ["bash"]
-    working_dir = local.resources_directory
-  }
-
-  depends_on = [
-    local_file.templates,
-  ]
+locals {
+  change_hash = "${join("-", [for file in ["k8s_presetup", "k8s_functions_sh", "aws_auth"] : md5(local_file.templates[file].content)])}-${try(md5(local_file.templates["eni_config"].content), "none")}"
 }
diff --git a/submodules/k8s/outputs.tf b/submodules/k8s/outputs.tf
@@ -0,0 +1,14 @@
+output "change_hash" {
+  description = "Hash of all templated files"
+  value       = local.change_hash
+}
+
+output "filename" {
+  description = "Filename of primary script"
+  value       = basename(local_file.templates["k8s_presetup"].filename)
+}
+
+output "resources_directory" {
+  description = "Directory for provisioned scripts and templated files"
+  value       = local.resources_directory
+}
diff --git a/submodules/k8s/templates/k8s-pre-setup.sh.tftpl b/submodules/k8s/templates/k8s-pre-setup.sh.tftpl
@@ -8,11 +8,6 @@ trap close_ssh_tunnel_to_k8s_api EXIT
 open_ssh_tunnel_to_k8s_api
 check_kubeconfig
 
-if ${cluster_setup}; then
-set_k8s_auth
-set_eniconfig
-fi
-
-if ${install_calico}; then
-install_calico
-fi
+for arg in "$@"; do
+  "$arg"
+done
diff --git a/submodules/k8s/variables.tf b/submodules/k8s/variables.tf
@@ -1,15 +1,3 @@
-variable "cluster_setup" {
-  type        = bool
-  description = "Perform cluster setup functions"
-  default     = false
-}
-
-variable "install_calico" {
-  type        = bool
-  description = "Perform calico install functions"
-  default     = false
-}
-
 variable "calico_version" {
   type        = string
   description = "Calico operator version."