Releases v3 (#134)

* PLAT-6822: Node pool,cluster, infra isolation (#113) * PLAT-6823: Node pool,cluster, infra segregation * [DOM-49677] Initial commit submodule (#126) * PLAT-7142: CDK to TF migration support. --------- Co-authored-by: Luis De Bello <[email protected]>
dominodatalab · Sep 19, 2023 · 42d64b4 · 42d64b4
1 parent 6ff5ab2
commit 42d64b4
Show file tree

Hide file tree

Showing 168 changed files with 4,687 additions and 1,913 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,15 +1,15 @@
 version: 2.1
 
 parameters:
-  workspace:
-    type: string
-    default: circleci-<< pipeline.number >>
   helm_version:
     type: string
     default: "v3.11.2"
   terraform_version:
     type: string
     default: "1.4.4"
+  hcledit_version:
+    type: string
+    default: "0.2.9"
   GHA_Actor:
     type: string
     default: ""
@@ -26,14 +26,6 @@ orbs:
   envsubst: sawadashota/[email protected]
 
 commands:
-  set_tf_vars:
-    description: "Sets Terraform variables"
-    steps:
-      - run:
-          name: Configure terraform vars
-          working_directory: tests
-          command: |
-            envsubst < ci.tfvars.tftpl | tee terraform.tfvars
   install_tf:
     description: "Install Terraform"
     parameters:
@@ -42,6 +34,45 @@ commands:
     steps:
       - terraform/install:
           terraform_version: << parameters.terraform_version >>
+  install_hcledit:
+    description: "Install HCL edit"
+    parameters:
+      hcledit_version:
+        type: string
+        default: "0.2.9"
+    steps:
+      - run:
+          name: Install HCL edit
+          working_directory: tests/deploy
+          environment:
+            HCLEDIT_VERSION: << parameters.hcledit_version >>
+          command: bash ci-deploy.sh install_hcledit
+  set_current_mod_source:
+    description: "Set up module source to current branch."
+    steps:
+      - run:
+          name: "Set module source to current branch"
+          working_directory: tests/deploy
+          command: bash ci-deploy.sh set_mod_src_circle_branch
+  set_aws_creds:
+    description: "Sets short-lived creds"
+    steps:
+      - aws-cli/setup:
+          role-arn: "${AWS_IAM_ROLE}"
+          session-duration: "900"
+  set_tf_vars:
+    description: "Sets Terraform variables"
+    steps:
+      - envsubst/install
+      - install_hcledit
+      - set_aws_creds
+      - run:
+          name: Bootstrap terraform module/vars.
+          working_directory: tests/deploy
+          command: |
+            bash ci-deploy.sh setup_modules
+            bash ci-deploy.sh set_tf_vars
+      - set_current_mod_source
   install_helm:
     description: "Install Helm"
     parameters:
@@ -50,71 +81,67 @@ commands:
     steps:
       - run:
           name: Install Helm
+          working_directory: tests/deploy
           environment:
             HELM_VERSION: << parameters.helm_version >>
-          command: |
-            if [ -z "$HELM_VERSION" ]; then
-              echo "HELM_VERSION environment variable not set, exiting."
-              exit 1
-            fi
-            echo "Installing Helm version: ${HELM_VERSION}"
-            curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
-            chmod +x get_helm.sh
-            ./get_helm.sh --version "${HELM_VERSION}"
-            rm ./get_helm.sh
-            helm version --short
-  gen_pvt_key:
-    description: "Generates ssh key"
-    steps:
-      - run:
-          name: Generate pvt key
-          command: ssh-keygen -q -P '' -t rsa -b 4096 -m PEM -f tests/domino.pem
+          command: bash ci-deploy.sh install_helm
   tf_init_apply:
     description: "Terraform init"
     steps:
-      - aws-cli/setup:
-         role-arn: "${AWS_IAM_ROLE}"
-         session-duration: "43200"
       - run:
           name: Terraform init/validate/apply
-          working_directory: tests
+          working_directory: tests/deploy
           command: |
-            echo "Current dir: $(pwd)"
-            terraform init
-            terraform validate
-            terraform workspace new ${WORKSPACE}
-            terraform apply -auto-approve
+            ## Most of this verbiage will go away after release.
+            set -e
+            echo "Current dir: $PWD"
+
+            if [ -f "migrated.txt" ]; then
+              echo "This legacy deployment has been migrated"
+              bash ci-deploy.sh deploy
+            elif [ -n "$MAJOR_MOD_VERSION" ] && (( $MAJOR_MOD_VERSION < 3 )) ; then
+
+              source "${PWD}/meta.sh"
+              echo "CI determined this is a legacy deploy."
+
+              echo "Generating Deployment pvt key."
+              export LEGACY_PVT_KEY="${LEGACY_DIR}/domino.pem"
+              envsubst < "${LEGACY_DIR}/ci.tfvars.tftpl" | tee "${LEGACY_DIR}/terraform.tfvars"
+              ssh-keygen -q -P '' -t rsa -b 4096 -m PEM -f "$LEGACY_PVT_KEY"
+
+              echo "Running legacy monolithic deploy"
+              terraform -chdir="$LEGACY_DIR" init
+              terraform -chdir="$LEGACY_DIR" validate
+              terraform -chdir="$LEGACY_DIR" apply --auto-approve --input=false
+
+              echo "Running state migration on CI."
+              bash -xp ../../bin/state-migration/migrate-states.sh
+            else
+              echo "Running ci-deploy.sh deploy"
+              bash ci-deploy.sh deploy
+            fi
+  tf_deploy:
+    description: "Terraform deploy"
+    steps:
+      - aws-cli/setup:
+          role-arn: "${AWS_IAM_ROLE}"
+          session-duration: "43200"
+      - tf_init_apply
   tf_destroy:
     description: "Terraform destroy"
     steps:
       - run:
           name: Terraform destroy
-          working_directory: tests
-          command: |
-            echo "Current dir: $(pwd)"
-            terraform destroy --auto-approve || terraform destroy --auto-approve --refresh=false
-          when: always
-  tf_ws_delete:
-    description: "Terraform workspace delete"
-    steps:
-      - run:
-          name: Terraform workspace delete
-          working_directory: tests
-          command: |
-            echo "Current dir: $(pwd)"
-            terraform workspace select default
-            terraform workspace delete ${WORKSPACE}
+          working_directory: tests/deploy
+          command: bash ci-deploy.sh destroy
           when: always
   tf_plan_test:
     steps:
-      - aws-cli/setup:
-         role-arn: "${AWS_IAM_ROLE}"
-         session-duration: "900"
+      - set_aws_creds
       - run:
           name: Terraform plan test
-          working_directory: examples
-          command: |
-            bash tf-plan-test.sh
+          working_directory: tests/plan
+          command: bash tf-plan-test.sh
 jobs:
   tf-plan-test:
     docker:
@@ -131,93 +158,68 @@ jobs:
     docker:
       - image: cimg/aws:2023.04.1
     parameters:
-      workspace:
-        type: string
       terraform_version:
         type: string
       helm_version:
         type: string
-    environment:
-      WORKSPACE: << parameters.workspace >>
     steps:
       - checkout
       - install_tf:
           terraform_version: << parameters.terraform_version >>
       - install_helm:
           helm_version: << parameters.helm_version >>
-      - envsubst/install
       - set_tf_vars
-      - gen_pvt_key
-      - tf_init_apply
+      - tf_deploy
       - tf_destroy
-      - tf_ws_delete
   test-upgrade:
     docker:
       - image: cimg/aws:2023.04.1
     parameters:
-      workspace:
-        type: string
       terraform_version:
         type: string
       helm_version:
         type: string
-    environment:
-      WORKSPACE: << parameters.workspace >>
+      hcledit_version:
+        type: string
     steps:
       - checkout
       - install_tf:
           terraform_version: << parameters.terraform_version >>
       - install_helm:
           helm_version: << parameters.helm_version >>
-      - envsubst/install
+      - install_hcledit
       - set_tf_vars
-      - gen_pvt_key
       - run:
           name: "Set module source to latest published release"
-          working_directory: tests
-          command: |
-            latest_release_tag=$(jq -r '.tag_name' <(curl -sSfL -H "X-GitHub-Api-Version: 2022-11-28" -H "Accept: application/vnd.github+json" https://api.github.com/repos/dominodatalab/terraform-aws-eks/releases/latest))
-            export MOD_SOURCE="github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}.git?ref=${latest_release_tag}"
-            echo "Latest published release tag is: ${latest_release_tag}"
-            echo "Setting module source to: ${MOD_SOURCE}"
-            cat \<<< $(jq --arg mod_source "${MOD_SOURCE}" '.module[0].domino_eks.source = $mod_source' main.tf.json) >main.tf.json
+          working_directory: tests/deploy
+          command: bash ci-deploy.sh set_mod_src_latest_rel
+      - tf_deploy
+      - set_current_mod_source
       - tf_init_apply
-      - run:
-          name: "Upgrade module by applying this commit"
-          working_directory: tests
-          command: |
-            echo "Testing terraform module upgrade"
-            export MOD_SOURCE="./.."
-            echo "Setting module source to local ref: ${MOD_SOURCE}"
-            cat \<<< $(jq --arg mod_source "${MOD_SOURCE}" '.module[0].domino_eks.source = $mod_source' main.tf.json) >main.tf.json
-            terraform init --reconfigure --upgrade
-            terraform validate
-            terraform apply --auto-approve
       - tf_destroy
-      - tf_ws_delete
 
 workflows:
   test-deploy-workflow:
     when:
-      equal: ["test-deploy-workflow", << pipeline.parameters.GHA_Action >> ]
+      equal: ["test-deploy-workflow", << pipeline.parameters.GHA_Action >>]
     jobs:
       - test-deploy:
           context: aws-oidc
           terraform_version: << pipeline.parameters.terraform_version >>
           helm_version: << pipeline.parameters.helm_version >>
-          workspace: << pipeline.parameters.workspace >>
   test-upgrade-workflow:
     when:
-      equal: ["test-upgrade-workflow", << pipeline.parameters.GHA_Action >> ]
+      equal: ["test-upgrade-workflow", << pipeline.parameters.GHA_Action >>]
     jobs:
       - test-upgrade:
           context: aws-oidc
           terraform_version: << pipeline.parameters.terraform_version >>
           helm_version: << pipeline.parameters.helm_version >>
-          workspace: << pipeline.parameters.workspace >>
+          hcledit_version: << pipeline.parameters.hcledit_version >>
   examples-plan-test-workflow:
     when:
-      equal: ["examples-plan-test-workflow", << pipeline.parameters.GHA_Action >> ]
+      equal:
+        ["examples-plan-test-workflow", << pipeline.parameters.GHA_Action >>]
     jobs:
       - tf-plan-test:
           context: aws-oidc

diff --git a/.gitignore b/.gitignore
@@ -5,6 +5,13 @@
 *.tfstate
 *.tfstate.*
 
+*infra.outputs*
+*cluster.outputs*
+*nodes.outputs*
+**migrated.txt
+**/deploy-test/*
+**terraform.plan
+
 **.terraform.lock.hcl*
 **.terraform.lock.hcl
 # Crash log files
@@ -31,6 +38,7 @@ override.tf.json
 # example: *tfplan*
 aws-auth.yaml
 domino.pem
+plan-test.pem*
 domino.pem.pub
 k8s-functions.sh
 k8s-pre-setup.sh
@@ -41,3 +49,4 @@ terraform_tfsec.txt
 eniconfig.yaml
 kubeconfig-proxy
 terraform.tfvars
+k8s-proxy-tunnel.sh
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -19,11 +19,11 @@ repos:
       - id: check-dependabot
       - id: check-github-actions
   - repo: https://github.com/antonbabenko/pre-commit-terraform
-    rev: v1.79.1
+    rev: v1.81.0
     hooks:
       - id: terraform_validate
         # See #4 on https://github.com/antonbabenko/pre-commit-terraform#terraform_validate
-        exclude: submodules/eks/[^/]+$
+        exclude: modules/eks/[^/]+$
         args:
           - "--hook-config=--retry-once-with-cleanup=true"
       - id: terraform_providers_lock
@@ -61,7 +61,7 @@ repos:
         args:
           - "--args=--compact"
           - "--args=--quiet"
-          - "--args=--skip-check CKV_CIRCLECIPIPELINES_2,CKV_CIRCLECIPIPELINES_6,CKV2_AWS_11,CKV2_AWS_12,CKV2_AWS_6,CKV_AWS_109,CKV_AWS_111,CKV_AWS_135,CKV_AWS_144,CKV_AWS_145,CKV_AWS_158,CKV_AWS_18,CKV_AWS_184,CKV_AWS_19,CKV_AWS_21,CKV_AWS_66,CKV_AWS_88,CKV2_GHA_1,CKV_AWS_163,CKV_AWS_39,CKV_AWS_38,CKV2_AWS_61,CKV2_AWS_62,CKV_AWS_136,CKV_AWS_329,CKV_AWS_338,CKV_AWS_339,CKV_AWS_341,CKV_AWS_356,CKV2_AWS_19"
+          - "--args=--skip-check CKV_CIRCLECIPIPELINES_2,CKV_CIRCLECIPIPELINES_6,CKV2_AWS_11,CKV2_AWS_12,CKV2_AWS_6,CKV_AWS_109,CKV_AWS_111,CKV_AWS_135,CKV_AWS_144,CKV_AWS_145,CKV_AWS_158,CKV_AWS_18,CKV_AWS_184,CKV_AWS_19,CKV_AWS_21,CKV_AWS_66,CKV_AWS_88,CKV2_GHA_1,CKV_AWS_163,CKV_AWS_39,CKV_AWS_38,CKV2_AWS_61,CKV2_AWS_62,CKV_AWS_136,CKV_AWS_329,CKV_AWS_338,CKV_AWS_339,CKV_AWS_341,CKV_AWS_356,CKV2_AWS_19,CKV2_AWS_5,CKV_AWS_150,CKV_AWS_123"
       - id: terraform_tfsec
         args:
           - "--args=-e aws-s3-specify-public-access-block,aws-cloudwatch-log-group-customer-key,aws-s3-enable-bucket-logging,aws-s3-enable-versioning,aws-s3-no-public-buckets,aws-ec2-require-vpc-flow-logs-for-all-vpcs,aws-s3-encryption-customer-key,aws-ec2-no-public-egress-sgr,aws-iam-no-policy-wildcards,aws-s3-block-public-acls,aws-s3-block-public-policy,aws-s3-enable-bucket-encryption,aws-s3-ignore-public-acls,aws-ec2-no-public-ingress-sgr,aws-ecr-repository-customer-key,aws-ecr-enable-image-scans,aws-eks-no-public-cluster-access,aws-eks-no-public-cluster-access-to-cidr"
@@ -74,6 +74,6 @@ repos:
     hooks:
       - id: check_aws_partition
         name: Check for hard coded AWS partition
-        entry: ./bin/check-aws-partition.sh
+        entry: ./bin/pre-commit/check-aws-partition.sh
         language: script
-        exclude: "^(examples|bin)"
+        exclude: "^(bin|examples)"