PLAT-5622 k8s provision through socks proxy

conditional kubeconfig cleanup on destroy

.gitignore

@@ -35,7 +35,6 @@ domino.pem.pub
 k8s-functions.sh
 k8s-pre-setup.sh
 kubeconfig
-mallory.json
 domino.yml
 terraform_tflint.txt
 terraform_tfsec.txt

main.tf

@@ -155,10 +155,9 @@ data "aws_iam_role" "eks_master_roles" {
 module "k8s_setup" {
   count                = var.create_bastion ? 1 : 0
   source               = "./submodules/k8s"
-  ssh_pvt_key_path     = abspath(local.ssh_pvt_key_path)
+  ssh_pvt_key_path     = local.ssh_pvt_key_path
   bastion_user         = local.bastion_user
   bastion_public_ip    = try(module.bastion[0].public_ip, "")
-  k8s_cluster_endpoint = module.eks.cluster_endpoint
   eks_node_role_arns   = [for r in module.eks.eks_node_roles : r.arn]
   eks_master_role_arns = [for r in concat(values(data.aws_iam_role.eks_master_roles), module.eks.eks_master_roles) : r.arn]
   kubeconfig_path      = local.kubeconfig_path

outputs.tf

@@ -8,11 +8,6 @@ output "bastion_ip" {
   value       = var.create_bastion ? module.bastion[0].public_ip : ""
 }
 
-output "k8s_tunnel_command" {
-  description = "Command to run the k8s tunnel mallory."
-  value       = try(module.k8s_setup.k8s_tunnel_command, "")
-}
-
 output "hostname" {
   description = "Domino instance URL."
   value       = "${var.deploy_id}.${var.route53_hosted_zone_name}"

submodules/eks/cluster.tf

@@ -117,10 +117,25 @@ resource "aws_eks_addon" "this" {
 
 resource "null_resource" "kubeconfig" {
   provisioner "local-exec" {
-    command = "aws eks update-kubeconfig --kubeconfig ${var.kubeconfig_path} --region ${var.region} --name ${aws_eks_cluster.this.name}"
+    when    = create
+    command = "aws eks update-kubeconfig --kubeconfig ${self.triggers.kubeconfig_file} --region ${self.triggers.region} --name ${self.triggers.cluster_name}"
+  }
+  provisioner "local-exec" {
+    when    = destroy
+    command = <<EOT
+      if (($(kubectl config --kubeconfig ${self.triggers.kubeconfig_file} get-contexts -o name | grep -v ${self.triggers.cluster_name}| wc -l) > 0 )); then
+        kubectl config --kubeconfig ${self.triggers.kubeconfig_file} delete-cluster ${self.triggers.cluster_name}
+        kubectl config --kubeconfig ${self.triggers.kubeconfig_file} delete-context ${self.triggers.cluster_name}
+      else
+        rm -f ${self.triggers.kubeconfig_file}
+      fi
+    EOT
   }
   triggers = {
     domino_eks_cluster_ca = aws_eks_cluster.this.certificate_authority[0].data
+    cluster_name          = aws_eks_cluster.this.name
+    kubeconfig_file       = var.kubeconfig_path
+    region                = var.region
   }
   depends_on = [aws_eks_cluster.this]
 }

submodules/k8s/README.md

@@ -37,15 +37,11 @@ No modules.
 | <a name="input_calico_version"></a> [calico\_version](#input\_calico\_version) | Calico operator version. | `string` | `"v1.11.0"` | no |
 | <a name="input_eks_master_role_arns"></a> [eks\_master\_role\_arns](#input\_eks\_master\_role\_arns) | IAM role arns to be added as masters in eks. | `list(string)` | `[]` | no |
 | <a name="input_eks_node_role_arns"></a> [eks\_node\_role\_arns](#input\_eks\_node\_role\_arns) | Roles arns for EKS nodes to be added to aws-auth for api auth. | `list(string)` | n/a | yes |
-| <a name="input_k8s_cluster_endpoint"></a> [k8s\_cluster\_endpoint](#input\_k8s\_cluster\_endpoint) | EKS cluster API endpoint. | `string` | n/a | yes |
+| <a name="input_k8s_tunnel_port"></a> [k8s\_tunnel\_port](#input\_k8s\_tunnel\_port) | K8s ssh tunnel port | `string` | `"1080"` | no |
 | <a name="input_kubeconfig_path"></a> [kubeconfig\_path](#input\_kubeconfig\_path) | Kubeconfig filename. | `string` | `"kubeconfig"` | no |
-| <a name="input_mallory_local_normal_port"></a> [mallory\_local\_normal\_port](#input\_mallory\_local\_normal\_port) | Mallory k8s tunnel normal port. | `string` | `"1315"` | no |
-| <a name="input_mallory_local_smart_port"></a> [mallory\_local\_smart\_port](#input\_mallory\_local\_smart\_port) | Mallory k8s tunnel smart(filters based on the blocked list) port. | `string` | `"1316"` | no |
 | <a name="input_ssh_pvt_key_path"></a> [ssh\_pvt\_key\_path](#input\_ssh\_pvt\_key\_path) | SSH private key filepath. | `string` | n/a | yes |
 
 ## Outputs
 
-| Name | Description |
-|------|-------------|
-| <a name="output_k8s_tunnel_command"></a> [k8s\_tunnel\_command](#output\_k8s\_tunnel\_command) | Command to run the k8s tunnel mallory. |
+No outputs.
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

submodules/k8s/main.tf

@@ -1,21 +1,15 @@
 locals {
-  mallory_config_filename       = "mallory.json"
-  mallory_container_name        = "mallory_k8s_tunnel"
-  mallory_config_path_container = "/root/.config/${local.mallory_config_filename}"
-  pvt_key_path_container        = "/root/${basename(var.ssh_pvt_key_path)}"
-  k8s_functions_sh_filename     = "k8s-functions.sh"
-  k8s_functions_sh_template     = "k8s-functions.sh.tftpl"
-  k8s_pre_setup_sh_filename     = "k8s-pre-setup.sh"
-  k8s_pre_setup_sh_template     = "k8s-pre-setup.sh.tftpl"
-  aws_auth_filename             = "aws-auth.yaml"
-  aws_auth_template             = "aws-auth.yaml.tftpl"
+  k8s_functions_sh_filename = "k8s-functions.sh"
+  k8s_functions_sh_template = "k8s-functions.sh.tftpl"
+  k8s_pre_setup_sh_filename = "k8s-pre-setup.sh"
+  k8s_pre_setup_sh_template = "k8s-pre-setup.sh.tftpl"
+  aws_auth_filename         = "aws-auth.yaml"
+  aws_auth_template         = "aws-auth.yaml.tftpl"
   calico = {
     operator_url         = "https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/${var.calico_version}/config/master/calico-operator.yaml"
     custom_resources_url = "https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/${var.calico_version}/config/master/calico-crs.yaml"
   }
 
-  k8s_tunnel_command = "docker run --rm --name ${local.mallory_container_name} -d -v $PWD/${local.mallory_config_filename}:${local.mallory_config_path_container} -p ${var.mallory_local_normal_port}:${var.mallory_local_normal_port} -p ${var.mallory_local_smart_port}:${var.mallory_local_smart_port} -v ${var.ssh_pvt_key_path}:${local.pvt_key_path_container} zoobab/mallory"
-
   resources_directory = path.cwd
   templates_dir       = "${path.module}/templates"
 
@@ -24,13 +18,13 @@ locals {
       filename = local.k8s_functions_sh_filename
       content = templatefile("${local.templates_dir}/${local.k8s_functions_sh_template}", {
         kubeconfig_path             = var.kubeconfig_path
-        k8s_tunnel_command          = local.k8s_tunnel_command
-        mallory_port                = var.mallory_local_smart_port
-        mallory_container_name      = local.mallory_container_name
-        mallory_config_file         = local.mallory_config_filename
+        k8s_tunnel_port             = var.k8s_tunnel_port
         aws_auth_yaml               = basename(local.aws_auth_filename)
         calico_operator_url         = local.calico.operator_url
         calico_custom_resources_url = local.calico.custom_resources_url
+        bastion_user                = var.bastion_user
+        bastion_public_ip           = var.bastion_public_ip
+        ssh_pvt_key_path            = var.ssh_pvt_key_path
       })
     }
 
@@ -41,17 +35,6 @@ locals {
       })
     }
 
-    mallory_k8s_tunnel = {
-      filename = local.mallory_config_filename
-      content = jsonencode(
-        {
-          "id_rsa"       = local.pvt_key_path_container
-          "local_smart"  = ":${var.mallory_local_smart_port}"
-          "local_normal" = ":${var.mallory_local_normal_port}"
-          "remote"       = "ssh://${var.bastion_user}@${var.bastion_public_ip}:22"
-          "blocked"      = [var.k8s_cluster_endpoint]
-    }) }
-
     aws_auth = {
       filename = local.aws_auth_filename
       content = templatefile("${local.templates_dir}/${local.aws_auth_template}",
@@ -75,7 +58,7 @@ resource "local_file" "templates" {
 resource "null_resource" "run_k8s_pre_setup" {
   provisioner "local-exec" {
     command     = basename(local_file.templates["k8s_presetup"].filename)
-    interpreter = ["bash", "-ex"]
+    interpreter = ["bash"]
     working_dir = local.resources_directory
   }
 

submodules/k8s/outputs.tf

@@ -1,4 +0,0 @@
-output "k8s_tunnel_command" {
-  description = "Command to run the k8s tunnel mallory."
-  value       = local.k8s_tunnel_command
-}

submodules/k8s/templates/k8s-functions.sh.tftpl

@@ -5,25 +5,13 @@ GREEN="\e[32m"
 EC="\e[0m"
 
 open_ssh_tunnel_to_k8s_api() {
-  printf "$GREEN Setting up mallory k8s tunnel... $EC \n"
-  if [[ -z "$https_proxy" ]]; then
-    MALLORY_PORT=${mallory_port}
-    https_proxy=http://localhost:$MALLORY_PORT
-    export https_proxy
-  fi
-
-  if test -f "${mallory_config_file}"; then
-    { docker kill ${mallory_container_name} && docker rm ${mallory_container_name}; } || true
-    eval ${k8s_tunnel_command} && sleep 3
-  else
-    echo "Mallory config file ${mallory_config_file} does not exist." && exit 1
-  fi
-
-  echo
+  TUNNEL_SOCKET_FILE=$${TUNNEL_SOCKET_FILE:-/tmp/k8s-tunnel-socket}
+  printf "$GREEN Openning k8s tunnel ... $EC \n"
+  ssh -q -N -f -M -o "StrictHostKeyChecking no" -o "ExitOnForwardFailure=yes" -i "${ssh_pvt_key_path}" -D ${k8s_tunnel_port} -S "$TUNNEL_SOCKET_FILE" ${bastion_user}@${bastion_public_ip}
 }
 
 check_kubeconfig() {
-  KUBECONFIG="${kubeconfig_path}"
+  local KUBECONFIG="${kubeconfig_path}"
   printf "$GREEN Checking if $KUBECONFIG exists... $EC \n"
   if test -f "$KUBECONFIG"; then
     echo "$KUBECONFIG exists." && export KUBECONFIG
@@ -34,42 +22,48 @@ check_kubeconfig() {
 }
 
 set_k8s_auth() {
-  AWS_AUTH_YAML="${aws_auth_yaml}"
+  local AWS_AUTH_YAML="${aws_auth_yaml}"
   if test -f "$AWS_AUTH_YAML"; then
     printf "$GREEN Updating $AWS_AUTH_YAML... $EC \n"
-    kubectl apply -f "$AWS_AUTH_YAML"
+    kubectl_apply "$AWS_AUTH_YAML"
   else
     printf "$RED $AWS_AUTH_YAML does not exist. $EC \n" && exit 1
   fi
   echo
 }
 
 install_calico() {
-  CALICO_OPERATOR_YAML_URL=${calico_operator_url}
+  local CALICO_OPERATOR_YAML_URL=${calico_operator_url}
   printf "$GREEN Installing Calico Operator $EC \n"
-  kubectl apply -f $CALICO_OPERATOR_YAML_URL || printf "$RED There was an error installing the calico operator"
+  kubectl_apply $CALICO_OPERATOR_YAML_URL || printf "$RED There was an error installing the calico operator"
   echo
-  CALICO_CRD_YAML_URL=${calico_custom_resources_url}
+  local CALICO_CRD_YAML_URL=${calico_custom_resources_url}
   printf "$GREEN Installing Calico Custom resources $EC \n" || printf "$RED There was an error installing the calico CRD"
-  kubectl apply -f $CALICO_CRD_YAML_URL
+  kubectl_apply $CALICO_CRD_YAML_URL
   echo
 }
 
+validate_url() {
+  local url="$1"
+  local log_file="validate-url.log"
+  if curl --head --fail --max-time 10 --output "$log_file" --stderr "$log_file" "$url"; then
+    rm "$log_file" && return 0
+  else
+    cat "$log_file" && return 1
+  fi
+}
+
 kubectl_apply() {
-  k8s_manifest="$1"
-  if test -f "$k8s_manifest"; then
+  local k8s_manifest="$1"
+  if test -f "$k8s_manifest" || validate_url "$k8s_manifest"; then
     echo "Applying $k8s_manifest..."
-    kubectl apply -f $k8s_manifest
+    HTTPS_PROXY=socks5://127.0.0.1:${k8s_tunnel_port} kubectl --kubeconfig "${kubeconfig_path}" apply -f "$k8s_manifest"
   else
     printf "$RED $k8s_manifest does not exist. $EC \n" && exit 1
   fi
-
-  kubectl apply -f "$k8s_manifest"
 }
 
 close_ssh_tunnel_to_k8s_api() {
-  printf "$GREEN Shutting down mallory k8s tunnel ${mallory_container_name} ... $EC"
-  docker kill "${mallory_container_name}"
-  docker rm "${mallory_container_name}" || true
-  echo
+  printf "$GREEN Shutting down k8s tunnel ... $EC"
+  ssh -S $TUNNEL_SOCKET_FILE -O exit ${bastion_user}@${bastion_public_ip}
 }

submodules/k8s/templates/k8s-pre-setup.sh.tftpl

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#! /usr/bin/env bash -ex
 
 RED="\e[31m"
 GREEN="\e[32m"

submodules/k8s/variables.tf

@@ -21,10 +21,6 @@ variable "bastion_public_ip" {
   type        = string
   description = "Bastion host public ip."
 }
-variable "k8s_cluster_endpoint" {
-  type        = string
-  description = "EKS cluster API endpoint."
-}
 
 variable "eks_node_role_arns" {
   type        = list(string)
@@ -37,16 +33,10 @@ variable "eks_master_role_arns" {
   default     = []
 }
 
-variable "mallory_local_normal_port" {
-  type        = string
-  description = "Mallory k8s tunnel normal port."
-  default     = "1315"
-}
-
-variable "mallory_local_smart_port" {
+variable "k8s_tunnel_port" {
   type        = string
-  description = "Mallory k8s tunnel smart(filters based on the blocked list) port."
-  default     = "1316"
+  description = "K8s ssh tunnel port"
+  default     = "1080"
 }
 
 variable "calico_version" {

tests/README.md

@@ -25,10 +25,10 @@ No resources.
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_deploy_id"></a> [deploy\_id](#input\_deploy\_id) | Domino Deployment ID. | `string` | `"domino-eks-test"` | no |
+| <a name="input_deploy_id"></a> [deploy\_id](#input\_deploy\_id) | Domino Deployment ID. | `string` | `"mhtest3"` | no |
 | <a name="input_k8s_version"></a> [k8s\_version](#input\_k8s\_version) | EKS cluster k8s version. | `string` | `"1.23"` | no |
 | <a name="input_region"></a> [region](#input\_region) | AWS region for the deployment | `string` | `"us-west-2"` | no |
-| <a name="input_tags"></a> [tags](#input\_tags) | Deployment tags. | `map(string)` | <pre>{<br>  "deploy_id": "domino-eks-test",<br>  "deploy_tag": "domino-eks-test",<br>  "deploy_type": "terraform-aws-eks",<br>  "domino-deploy-id": "domino-eks-test"<br>}</pre> | no |
+| <a name="input_tags"></a> [tags](#input\_tags) | Deployment tags. | `map(string)` | <pre>{<br>  "deploy_id": "mhtest3",<br>  "deploy_tag": "mhtest3",<br>  "deploy_type": "terraform-aws-eks",<br>  "domino-deploy-id": "mhtest3"<br>}</pre> | no |
 
 ## Outputs
 

tests/variables.tf

@@ -1,7 +1,7 @@
 variable "deploy_id" {
   type        = string
   description = "Domino Deployment ID."
-  default     = "domino-eks-test"
+  default     = "mhtest3"
 }
 
 
@@ -16,10 +16,10 @@ variable "tags" {
   type        = map(string)
   description = "Deployment tags."
   default = {
-    deploy_id        = "domino-eks-test"
-    deploy_tag       = "domino-eks-test"
+    deploy_id        = "mhtest3"
+    deploy_tag       = "mhtest3"
     deploy_type      = "terraform-aws-eks"
-    domino-deploy-id = "domino-eks-test"
+    domino-deploy-id = "mhtest3"
   }
 }