Apply lineage changes to verify functionality in local environment an…

…d GitHub Actions.
cisagov · May 7, 2024 · 24f541f · 24f541f
1 parent 1c8d3a6
commit 24f541f
Show file tree

Hide file tree

Showing 12 changed files with 336 additions and 120 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -3,10 +3,22 @@
 # These owners will be the default owners for everything in the
 # repo. Unless a later match takes precedence, these owners will be
 # requested for review when someone opens a pull request.
-
-
 * @aloftus23 @cduhn17 @Matthew-Grayson @nickviola @rapidray12 @schmelz21
 
 # These folks own any files in the .github directory at the root of
 # the repository and any of its subdirectories.
 /.github/ @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj
+
+# These folks own all linting configuration files.
+/.ansible-lint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj
+/.bandit.yml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj
+/.flake8 @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj
+/.isort.cfg @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj
+/.mdl_config.yaml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj
+/.pre-commit-config.yaml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj
+/.prettierignore @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj
+/.yamllint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj
+/requirements.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj
+/requirements-dev.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj
+/requirements-test.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj
+/setup-env @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -2,16 +2,31 @@
 name: build
 
 on:
-  push:
+  merge_group:
+    types:
+      - checks_requested
   pull_request:
+  push:
   repository_dispatch:
-    types: [apb]
+    types:
+      - apb
+
+# Set a default shell for any run steps. The `-Eueo pipefail` sets errtrace,
+# nounset, errexit, and pipefail. The `-x` will print all commands as they are
+# run. Please see the GitHub Actions documentation for more information:
+# https://docs.github.com/en/actions/using-jobs/setting-default-values-for-jobs
+defaults:
+  run:
+    shell: bash -Eueo pipefail -x {0}
 
 env:
   CURL_CACHE_DIR: ~/.cache/curl
   PIP_CACHE_DIR: ~/.cache/pip
   PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit
   RUN_TMATE: ${{ secrets.RUN_TMATE }}
+  TERRAFORM_DOCS_REPO_BRANCH_NAME: improvement/support_atx_closed_markdown_headers
+  TERRAFORM_DOCS_REPO_DEPTH: 1
+  TERRAFORM_DOCS_REPO_URL: https://github.com/mcdonnnj/terraform-docs.git
 
 jobs:
   diagnostics:
@@ -27,7 +42,7 @@ jobs:
           egress-policy: audit
       - id: github-status
         name: Check GitHub status
-        uses: crazy-max/ghaction-github-status@v3
+        uses: crazy-max/ghaction-github-status@v4
       - id: dump-context
         name: Dump context
         uses: crazy-max/ghaction-dump-context@v2
@@ -45,28 +60,34 @@ jobs:
         uses: cisagov/setup-env-github-action@develop
       - uses: actions/checkout@v4
       - id: setup-python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
+          python-version: ${{ steps.setup-env.outputs.python-version }}
       # We need the Go version and Go cache location for the actions/cache step,
       # so the Go installation must happen before that.
       - id: setup-go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           # There is no expectation for actual Go code so we disable caching as
           # it relies on the existence of a go.sum file.
           cache: false
-          go-version: '1.20'
-      - name: Lookup Go cache directory
-        id: go-cache
-        run: echo "dir=$(go env GOCACHE)" >> $GITHUB_OUTPUT
+          go-version: ${{ steps.setup-env.outputs.go-version }}
+      - id: go-cache
+        name: Lookup Go cache directory
+        run: |
+          echo "dir=$(go env GOCACHE)" >> $GITHUB_OUTPUT
       - uses: actions/cache@v3
         env:
           BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\
             py${{ steps.setup-python.outputs.python-version }}-\
             go${{ steps.setup-go.outputs.go-version }}-\
+            packer${{ steps.setup-env.outputs.packer-version }}-\
             tf${{ steps.setup-env.outputs.terraform-version }}-"
         with:
+          key: "${{ env.BASE_CACHE_KEY }}\
+            ${{ hashFiles('**/requirements-test.txt') }}-\
+            ${{ hashFiles('**/requirements.txt') }}-\
+            ${{ hashFiles('**/.pre-commit-config.yaml') }}"
           # Note that the .terraform directory IS NOT included in the
           # cache because if we were caching, then we would need to use
           # the `-upgrade=true` option. This option blindly pulls down the
@@ -76,33 +97,71 @@ jobs:
           path: |
             ${{ env.PIP_CACHE_DIR }}
             ${{ env.PRE_COMMIT_CACHE_DIR }}
+            ${{ env.CURL_CACHE_DIR }}
             ${{ steps.go-cache.outputs.dir }}
-          key: "${{ env.BASE_CACHE_KEY }}\
-            ${{ hashFiles('**/requirements-test.txt') }}-\
-            ${{ hashFiles('**/requirements.txt') }}-\
-            ${{ hashFiles('**/.pre-commit-config.yaml') }}"
-          restore-keys: ${{ env.BASE_CACHE_KEY }}
-      - uses: hashicorp/setup-terraform@v2
+          restore-keys: |
+            ${{ env.BASE_CACHE_KEY }}
+      - name: Setup curl cache
+        run: mkdir -p ${{ env.CURL_CACHE_DIR }}
+      - name: Install Packer
+        env:
+          PACKER_VERSION: ${{ steps.setup-env.outputs.packer-version }}
+        run: |
+          PACKER_ZIP="packer_${PACKER_VERSION}_linux_amd64.zip"
+          curl --output ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \
+            --time-cond ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \
+            --location \
+            "https://releases.hashicorp.com/packer/${PACKER_VERSION}/${PACKER_ZIP}"
+          sudo unzip -d /opt/packer \
+            ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}"
+          sudo mv /usr/local/bin/packer /usr/local/bin/packer-default
+          sudo ln -s /opt/packer/packer /usr/local/bin/packer
+      - uses: hashicorp/setup-terraform@v3
         with:
           terraform_version: ${{ steps.setup-env.outputs.terraform-version }}
-      - name: Install shfmt
+      - name: Install go-critic
+        env:
+          PACKAGE_URL: github.com/go-critic/go-critic/cmd/gocritic
+          PACKAGE_VERSION: ${{ steps.setup-env.outputs.go-critic-version }}
+        run: go install ${PACKAGE_URL}@${PACKAGE_VERSION}
+      - name: Install goimports
+        env:
+          PACKAGE_URL: golang.org/x/tools/cmd/goimports
+          PACKAGE_VERSION: ${{ steps.setup-env.outputs.goimports-version }}
+        run: go install ${PACKAGE_URL}@${PACKAGE_VERSION}
+      - name: Install gosec
         env:
-          PACKAGE_URL: mvdan.cc/sh/v3/cmd/shfmt
-          PACKAGE_VERSION: ${{ steps.setup-env.outputs.shfmt-version }}
+          PACKAGE_URL: github.com/securego/gosec/v2/cmd/gosec
+          PACKAGE_VERSION: ${{ steps.setup-env.outputs.gosec-version }}
         run: go install ${PACKAGE_URL}@${PACKAGE_VERSION}
-      - name: Install Terraform-docs
+      - name: Install staticcheck
         env:
-          PACKAGE_URL: github.com/terraform-docs/terraform-docs
-          PACKAGE_VERSION: ${{ steps.setup-env.outputs.terraform-docs-version }}
+          PACKAGE_URL: honnef.co/go/tools/cmd/staticcheck
+          PACKAGE_VERSION: ${{ steps.setup-env.outputs.staticcheck-version }}
         run: go install ${PACKAGE_URL}@${PACKAGE_VERSION}
+      # TODO: https://github.com/cisagov/skeleton-generic/issues/165
+      # We are temporarily using @mcdonnnj's forked branch of terraform-docs
+      # until his PR: https://github.com/terraform-docs/terraform-docs/pull/745
+      # is approved. This temporary fix will allow for ATX header support when
+      # terraform-docs is run during linting.
+      - name: Clone ATX headers branch from terraform-docs fork
+        run: |
+          git clone \
+           --branch $TERRAFORM_DOCS_REPO_BRANCH_NAME \
+           --depth $TERRAFORM_DOCS_REPO_DEPTH \
+           --single-branch \
+           $TERRAFORM_DOCS_REPO_URL /tmp/terraform-docs
+      - name: Build and install terraform-docs binary
+        run: |
+          go build \
+          -C /tmp/terraform-docs \
+          -o $(go env GOPATH)/bin/terraform-docs
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip setuptools wheel
           pip install --upgrade --requirement requirements-test.txt
       - name: Set up pre-commit hook environments
         run: pre-commit install-hooks
-      - name: Create .env file needed by docker-compose-check pre-commit hook
-        run: cp dev.env.example .env
       - name: Run pre-commit on all files
         run: pre-commit run --all-files
       - name: Setup tmate debug session

diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml
@@ -4,21 +4,45 @@ name: sync-labels
 on:
   push:
     paths:
-      - .github/labels.yml
-      - .github/workflows/sync-labels.yml
+      - '.github/labels.yml'
+      - '.github/workflows/sync-labels.yml'
 
 permissions:
   contents: read
 
 jobs:
+  diagnostics:
+    name: Run diagnostics
+    runs-on: ubuntu-latest
+    steps:
+      # Note that a duplicate of this step must be added at the top of
+      # each job.
+      - id: harden-runner
+        name: Harden the runner
+        uses: step-security/harden-runner@v2
+        with:
+          egress-policy: audit
+      - id: github-status
+        name: Check GitHub status
+        uses: crazy-max/ghaction-github-status@v3
+      - id: dump-context
+        name: Dump context
+        uses: crazy-max/ghaction-dump-context@v2
   labeler:
+    needs:
+      - diagnostics
     permissions:
       # actions/checkout needs this to fetch code
       contents: read
       # crazy-max/ghaction-github-labeler needs this to manage repository labels
       issues: write
     runs-on: ubuntu-latest
     steps:
+      - id: harden-runner
+        name: Harden the runner
+        uses: step-security/harden-runner@v2
+        with:
+          egress-policy: audit
       - uses: actions/checkout@v4
       - name: Sync repository labels
         if: success()

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -5,12 +5,14 @@ default_language_version:
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
       - id: check-case-conflict
       - id: check-executables-have-shebangs
       - id: check-json
       - id: check-merge-conflict
+      - id: check-toml
+      - id: check-xml
       - id: debug-statements
       - id: detect-aws-credentials
         args:
@@ -31,90 +33,101 @@ repos:
 
   # Text file hooks
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.36.0
+    rev: v0.39.0
     hooks:
       - id: markdownlint
         args:
           - --config=.mdl_config.yaml
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v3.0.3
+    # This is the last version of v3 available from the mirror. We should hold
+    # here until v4, which is currently in alpha, is more stable.
+    rev: v3.1.0
     hooks:
       - id: prettier
+        # This is the latest version of v3 available from NPM. The pre-commit
+        # mirror does not pull tags for old major versions once a new major
+        # version tag is published.
+        additional_dependencies:
+          - [email protected]
   - repo: https://github.com/adrienverge/yamllint
-    rev: v1.32.0
+    rev: v1.35.1
     hooks:
       - id: yamllint
         args:
           - --strict
 
   # GitHub Actions hooks
   - repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: 0.26.3
+    rev: 0.28.0
     hooks:
       - id: check-github-actions
       - id: check-github-workflows
 
   # pre-commit hooks
   - repo: https://github.com/pre-commit/pre-commit
-    rev: v3.4.0
+    rev: v3.6.2
     hooks:
       - id: validate_manifest
 
   # Shell script hooks
-  - repo: https://github.com/cisagov/pre-commit-shfmt
-    rev: v0.0.2
+  - repo: https://github.com/scop/pre-commit-shfmt
+    rev: v3.7.0-4
     hooks:
       - id: shfmt
         args:
+          # List files that will be formatted
+          - --list
+          # Write result to file instead of stdout
+          - --write
           # Indent by two spaces
-          - -i
-          - '2'
+          - --indent
+          - "2"
           # Binary operators may start a line
-          - -bn
+          - --binary-next-line
           # Switch cases are indented
-          - -ci
+          - --case-indent
           # Redirect operators are followed by a space
-          - -sr
-  - repo: https://github.com/detailyang/pre-commit-shell
-    rev: 1.0.5
+          - --space-redirects
+  - repo: https://github.com/shellcheck-py/shellcheck-py
+    rev: v0.9.0.6
     hooks:
-      - id: shell-lint
+      - id: shellcheck
 
   # Python hooks
   - repo: https://github.com/PyCQA/bandit
-    rev: 1.7.5
+    rev: 1.7.7
     hooks:
       - id: bandit
         args:
           - --config=.bandit.yml
   - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 23.9.1
+    rev: 24.2.0
     hooks:
       - id: black
   - repo: https://github.com/PyCQA/flake8
-    rev: 6.1.0
+    rev: 7.0.0
     hooks:
       - id: flake8
         additional_dependencies:
           - flake8-docstrings
   - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
+    rev: 5.13.2
     hooks:
       - id: isort
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.5.1
+    rev: v1.8.0
     hooks:
       - id: mypy
         additional_dependencies:
           - types-requests
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.10.1
+    rev: v3.15.1
     hooks:
       - id: pyupgrade
 
   # Terraform hooks
   - repo: https://github.com/antonbabenko/pre-commit-terraform
-    rev: v1.83.2
+    rev: v1.88.0
     hooks:
       - id: terraform_fmt
       - id: terraform_validate