From 71783096518476bf7c109bedf9281e87b7dd7a5b Mon Sep 17 00:00:00 2001 From: Gary Brandon Date: Fri, 3 Nov 2023 09:55:54 +1100 Subject: [PATCH] Fixing up CDK scripts/build (#158) * Fixing up CDK scripts/build * Also update scripts * Any * Unneeded * Without caching --- .github/renovate.json | 11 ++ .github/workflows/build_container.yaml | 2 - .github/workflows/test_cdk.yaml | 22 ++- .github/workflows/test_master.yaml | 2 - scripts/cdk/bootstrap | 4 +- scripts/cdk/deploy | 2 +- scripts/cdk/ls | 2 +- scripts/cdk/synth | 2 +- scripts/include/build | 6 +- scripts/include/cdk | 10 +- scripts/include/env-app | 14 +- scripts/include/env-container-build | 241 ++++++++++++++----------- scripts/include/env-container-run | 1 - 13 files changed, 181 insertions(+), 138 deletions(-) diff --git a/.github/renovate.json b/.github/renovate.json index 687a756f..da781331 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -4,4 +4,15 @@ "config:base", "group:all" ] + "customManagers": [ + { + "customType": "regex", + "fileMatch": [ + "^scripts/include/env-app$" + ], + "matchStrings": [ + "# renovate: datasource=(?[a-z-.]+?) depName=(?[^\\s]+?)(?: (lookupName|packageName)=(?[^\\s]+?))?(?: versioning=(?[^\\s]+?))?(?: registryUrl=(?[^\\s]+?))?\\s(?:ENV|ARG|export) .+?_VERSION[ =]\"?(?.+?)\"?\\s" + ] + } + ] } diff --git a/.github/workflows/build_container.yaml b/.github/workflows/build_container.yaml index 01604ea5..f49c70d9 100644 --- a/.github/workflows/build_container.yaml +++ b/.github/workflows/build_container.yaml @@ -53,5 +53,3 @@ jobs: "POETRY_VERSION=${{ env.POETRY_VERSION }}" push: true tags: ${{ env.CONTAINER_IMAGE_REVISION }} - cache-from: type=gha - cache-to: type=gha,mode=max diff --git a/.github/workflows/test_cdk.yaml b/.github/workflows/test_cdk.yaml index e0186c13..ad156807 100644 --- a/.github/workflows/test_cdk.yaml +++ b/.github/workflows/test_cdk.yaml @@ -7,26 +7,28 @@ on: branches: - "main" paths: - - '.github/workflows/test_cdk.yaml' - '.github/workflows/lint_python.yaml' - '.github/workflows/lint_yaml.yaml' - - 'docker/Dockerfile.cdk' + - '.github/workflows/test_cdk.yaml' - 'deployment/**' - - 'test/unit/cdk/**' - - 'pyproject.toml' + - 'docker/Dockerfile.cdk' - 'poetry.lock' + - 'pyproject.toml' + - 'scripts/cdk/*' + - 'scripts/include/cdk' + - 'test/unit/cdk/**' pull_request: - branches: - - "main" paths: - - '.github/workflows/test_cdk.yaml' - '.github/workflows/lint_python.yaml' - '.github/workflows/lint_yaml.yaml' - - 'docker/Dockerfile.cdk' + - '.github/workflows/test_cdk.yaml' - 'deployment/**' - - 'test/unit/cdk/**' - - 'pyproject.toml' + - 'docker/Dockerfile.cdk' - 'poetry.lock' + - 'pyproject.toml' + - 'scripts/cdk/*' + - 'scripts/include/cdk' + - 'test/unit/cdk/**' jobs: build: diff --git a/.github/workflows/test_master.yaml b/.github/workflows/test_master.yaml index 2cbafcbf..0aa81bab 100644 --- a/.github/workflows/test_master.yaml +++ b/.github/workflows/test_master.yaml @@ -17,8 +17,6 @@ on: - 'pyproject.toml' - 'poetry.lock' pull_request: - branches: - - "main" paths: - '.github/workflows/test_master.yaml' - '.github/workflows/lint_python.yaml' diff --git a/scripts/cdk/bootstrap b/scripts/cdk/bootstrap index 24967b8f..58552128 100755 --- a/scripts/cdk/bootstrap +++ b/scripts/cdk/bootstrap @@ -4,5 +4,5 @@ set -euo pipefail cd "$(git rev-parse --show-toplevel)" || exit 1 ACCOUNT=$(aws sts get-caller-identity | jq -r '.Account') -scripts/include/cdk cdk bootstrap "aws://${ACCOUNT}/us-west-2" -scripts/include/cdk cdk bootstrap "aws://${ACCOUNT}/us-east-1" +scripts/include/cdk bootstrap "aws://${ACCOUNT}/us-west-2" +scripts/include/cdk bootstrap "aws://${ACCOUNT}/us-east-1" diff --git a/scripts/cdk/deploy b/scripts/cdk/deploy index 42880373..77262434 100755 --- a/scripts/cdk/deploy +++ b/scripts/cdk/deploy @@ -3,4 +3,4 @@ set -euo pipefail cd "$(git rev-parse --show-toplevel)" || exit 1 -scripts/include/cdk cdk deploy --app ./app.py +scripts/include/cdk deploy --app ./app.py diff --git a/scripts/cdk/ls b/scripts/cdk/ls index 63c36bab..53774b15 100755 --- a/scripts/cdk/ls +++ b/scripts/cdk/ls @@ -3,4 +3,4 @@ set -euo pipefail cd "$(git rev-parse --show-toplevel)" || exit 1 -scripts/include/cdk cdk ls +scripts/include/cdk ls diff --git a/scripts/cdk/synth b/scripts/cdk/synth index 10289491..0f188317 100755 --- a/scripts/cdk/synth +++ b/scripts/cdk/synth @@ -3,4 +3,4 @@ set -euo pipefail cd "$(git rev-parse --show-toplevel)" || exit 1 -scripts/include/cdk cdk synth +scripts/include/cdk synth diff --git a/scripts/include/build b/scripts/include/build index ae2a6e35..1a42f261 100755 --- a/scripts/include/build +++ b/scripts/include/build @@ -6,11 +6,11 @@ cd "$(git rev-parse --show-toplevel)" || exit CONTAINER=${1:-} source scripts/include/env-container-build "$CONTAINER" -if [ "$CONTAINER_SKIP_BUILD" != "TRUE" ]; then +if [ "$CONTAINER_SKIP_BUILD" == "TRUE" ]; then + echo "+++ Skipped build for ${CONTAINER_IMAGE_REVISION} +++" +else echo "+++ Building ${CONTAINER_IMAGE_REVISION} +++" $CONTAINER_COMMAND build \ "${CONTAINER_BUILD_OPTIONS[@]}" \ --tag "${CONTAINER_IMAGE_REVISION}" -else - echo "+++ Skipped build for ${CONTAINER_IMAGE_REVISION} +++" fi diff --git a/scripts/include/cdk b/scripts/include/cdk index bae1cb55..349ec5c7 100755 --- a/scripts/include/cdk +++ b/scripts/include/cdk @@ -3,8 +3,12 @@ set -euo pipefail cd "$(git rev-parse --show-toplevel)" || exit 1 -source scripts/include/build-env -scripts/build-cdk > /dev/null +source scripts/include/build cdk >/dev/null 2>&1 +source scripts/include/env-container-run + +CONTAINER_AWS_OPTIONS+=( + --env JSII_SILENCE_WARNING_UNTESTED_NODE_VERSION=True +) # shellcheck disable=SC2206 CONTAINER_RUN_OPTIONS+=( @@ -16,5 +20,5 @@ CONTAINER_RUN_OPTIONS+=( $CONTAINER_COMMAND run \ "${CONTAINER_RUN_OPTIONS[@]}" \ "${CONTAINER_AWS_OPTIONS[@]}" \ - "${APP_NAME}/cdk:latest" \ + "${CONTAINER_IMAGE_REVISION}" \ "$@" diff --git a/scripts/include/env-app b/scripts/include/env-app index 8e344a03..64a48342 100644 --- a/scripts/include/env-app +++ b/scripts/include/env-app @@ -3,11 +3,6 @@ set -euo pipefail cd "$(git rev-parse --show-toplevel)" || exit -if ! test -f pyproject.toml; then - echo "Missing pyproject.toml" - exit 1; -fi - # # General app variables # @@ -21,9 +16,12 @@ export MASTER_HEALTHCHECK="8080/tcp" # # Depenency versions # -CDK_VERSION=$(awk '/^aws-cdk-lib = / {gsub(/"/, "", $3); gsub(/\^/, "", $3); print $3}' pyproject.toml) -POETRY_VERSION=$(awk '/^poetry = / {gsub(/"/, "", $3); gsub(/\^/, "", $3); print $3}' pyproject.toml) -PIP_VERSION=$(awk '/^pip = / {gsub(/"/, "", $3); gsub(/\^/, "", $3); print $3}' pyproject.toml) +# renovate: datasource=pypi depName=aws-cdk-lib +export CDK_VERSION="2.103.1" +# renovate: datasource=pypi depName=pip +export PIP_VERSION="23.3.1" +# renovate: datasource=pypi depName=poetry +export POETRY_VERSION="1.6.1" export CDK_VERSION export POETRY_VERSION diff --git a/scripts/include/env-container-build b/scripts/include/env-container-build index ccd7dc22..5cc74703 100755 --- a/scripts/include/env-container-build +++ b/scripts/include/env-container-build @@ -2,9 +2,38 @@ set -euo pipefail cd "$(git rev-parse --show-toplevel)" || exit + +CONTAINER=${1:-} +if [ -z "$CONTAINER" ]; then + echo "+++ $0 supplied with no argument +++" + exit 1 +else + echo "+++ Configuring build environment for $1 +++" +fi + source scripts/include/env-app source scripts/include/env-container-any +function container-pull() { + local result + local return_result=${2:-"FALSE"} + + echo -n "+++ Attempting to pull $1... " + if $CONTAINER_COMMAND pull "$1" >/dev/null 2>&1; then + echo "pull successful +++" + result=0 + else + echo "pull failed +++" + result=1 + fi + + if [ "${return_result}" == "TRUE" ]; then + return ${result} + fi + + return 0 +} + CONTAINER_SKIP_BUILD="FALSE" CONTAINER_BUILD_OPTIONS=( @@ -37,120 +66,124 @@ fi # # Container specific build variables # -CONTAINER=${1:-} CDK_TAG="None" -if [ -n "$CONTAINER" ]; then - case $CONTAINER in - master) - CONTAINER_TARGET="master" - CONTAINER_IMAGE=$MASTER_GHCR_TAG - CONTAINER_FILE="docker/Dockerfile" - ;; - - cdk-base) - CONTAINER_TARGET="cdk-base" - CONTAINER_IMAGE=$CDK_BASE_GHCR_TAG - CONTAINER_REVISION=$CDK_VERSION - CONTAINER_FILE="URL" - CONTAINER_URL="github.com/gazwald/cdk-docker.git" - CONTAINER_BUILD_OPTIONS+=( - --build-arg "CDK_VERSION=$CDK_VERSION" - ) - ;; - cdk) - CONTAINER_TARGET="cdk" - CONTAINER_IMAGE=$CDK_GHCR_TAG - CONTAINER_FILE="docker/Dockerfile.cdk" - CDK_TAG=$CDK_BASE_GHCR_TAG - CONTAINER_BUILD_OPTIONS+=( - --build-arg "CDK_TAG=$CDK_TAG" - --build-arg "CDK_VERSION=$CDK_VERSION" - ) - ;; - - cdk-test) - CONTAINER_TARGET="cdk-test" - CONTAINER_IMAGE=$CDK_TEST_GHCR_TAG - CONTAINER_FILE="docker/Dockerfile.cdk" - CDK_TAG=$CDK_BASE_GHCR_TAG - CONTAINER_BUILD_OPTIONS+=( - --build-arg "CDK_TAG=$CDK_TAG" - --build-arg "CDK_VERSION=$CDK_VERSION" - ) - ;; - - docs) - CONTAINER_TARGET="docs" - CONTAINER_IMAGE=$DOCS_GHCR_TAG - CONTAINER_FILE="docker/Dockerfile.docs" - ;; - - test) - CONTAINER_TARGET="test" - CONTAINER_IMAGE=$TEST_GHCR_TAG - CONTAINER_FILE="docker/Dockerfile" - ;; - - *) - echo "$CONTAINER is unknown" - exit 1 - ;; - - esac - - # - # Final tags - # - CONTAINER_IMAGE_LATEST="${CONTAINER_IMAGE}:latest" - CONTAINER_IMAGE_REVISION="${CONTAINER_IMAGE}:${CONTAINER_TARGET}-${CONTAINER_REVISION}" - - # - # Caching - # - if docker inspect "$CONTAINER_IMAGE_LATEST" >/dev/null 2>&1; then +case $CONTAINER in + master) + CONTAINER_TARGET="master" + CONTAINER_IMAGE=$MASTER_GHCR_TAG + CONTAINER_FILE="docker/Dockerfile" + ;; + + cdk-base) + CONTAINER_TARGET="cdk-base" + CONTAINER_IMAGE=$CDK_BASE_GHCR_TAG + CONTAINER_REVISION=$CDK_VERSION + CONTAINER_FILE="URL" + CONTAINER_URL="github.com/gazwald/cdk-docker.git" CONTAINER_BUILD_OPTIONS+=( - --cache-from "${CONTAINER_IMAGE_LATEST}" + --build-arg "CDK_VERSION=$CDK_VERSION" ) - fi - if docker inspect "$CONTAINER_IMAGE_REVISION" >/dev/null 2>&1; then + ;; + cdk) + CONTAINER_TARGET="cdk" + CONTAINER_IMAGE=$CDK_GHCR_TAG + CONTAINER_FILE="docker/Dockerfile.cdk" + CDK_TAG=$CDK_BASE_GHCR_TAG CONTAINER_BUILD_OPTIONS+=( - --cache-from "${CONTAINER_IMAGE_REVISION}" + --build-arg "CDK_TAG=$CDK_TAG" + --build-arg "CDK_VERSION=$CDK_VERSION" ) - fi - - # - # Try to pull the container to speed up CI - # - echo "+++ Attempting to pull $CONTAINER_IMAGE_LATEST +++" - if $CONTAINER_COMMAND pull "$CONTAINER_IMAGE_LATEST" > /dev/null; then - echo "+++ Pull successful +++" - else - echo "+++ Pull failed +++" - fi + ;; - # - # Support for using a URL instead of a PATH - # Try to pull the container and if that fails build it - # - echo "+++ Attempting to pull $CONTAINER_IMAGE_REVISION +++" - if $CONTAINER_COMMAND pull "$CONTAINER_IMAGE_REVISION" > /dev/null; then - echo "+++ Pull successful, skipping build... +++" - CONTAINER_SKIP_BUILD="TRUE" - else - echo "+++ Pull failed, continuing building locally +++" - fi - - if [[ "$CONTAINER_FILE" == "URL" ]]; then - CONTAINER_BUILD_OPTIONS+=( - "$CONTAINER_URL" - ) - else + cdk-test) + CONTAINER_TARGET="cdk-test" + CONTAINER_IMAGE=$CDK_TEST_GHCR_TAG + CONTAINER_FILE="docker/Dockerfile.cdk" + CDK_TAG=$CDK_BASE_GHCR_TAG CONTAINER_BUILD_OPTIONS+=( - --file "${CONTAINER_FILE}" - . + --build-arg "CDK_TAG=$CDK_TAG" + --build-arg "CDK_VERSION=$CDK_VERSION" ) + ;; + + docs) + CONTAINER_TARGET="docs" + CONTAINER_IMAGE=$DOCS_GHCR_TAG + CONTAINER_FILE="docker/Dockerfile.docs" + ;; + + test) + CONTAINER_TARGET="test" + CONTAINER_IMAGE=$TEST_GHCR_TAG + CONTAINER_FILE="docker/Dockerfile" + ;; + + *) + echo "+++ Target $CONTAINER is unknown +++" + exit 1 + ;; + +esac + +# +# Final tags +# +CONTAINER_IMAGE_LATEST="${CONTAINER_IMAGE}:latest" +if [[ "$CONTAINER_TARGET" == "cdk-base" ]]; then + CONTAINER_IMAGE_REVISION="${CONTAINER_IMAGE}:${CDK_VERSION}" +else + CONTAINER_IMAGE_REVISION="${CONTAINER_IMAGE}:${CONTAINER_TARGET}-${CONTAINER_REVISION}" +fi + + +# +# If we're building 'cdk' check to see if we can pull the base image +# If not then build it locally +# +if [[ "$CONTAINER_TARGET" == "cdk" ]]; then + if ! container-pull "$CDK_BASE_GHCR_TAG:$CDK_VERSION" "TRUE"; then + echo "+++ Unable to pull base CDK image... building locally +++" + scripts/include/build cdk-base fi fi +# +# Try to pull the container to speed up container commands +# +if container-pull "$CONTAINER_IMAGE_REVISION" "TRUE"; then + CONTAINER_SKIP_BUILD="TRUE" +else + container-pull "$CONTAINER_IMAGE_LATEST" +fi + +# +# Support for using a URL instead of a PATH +# Try to pull the container and if that fails build it +# +if [[ "$CONTAINER_FILE" == "URL" ]]; then + CONTAINER_BUILD_OPTIONS+=( + "$CONTAINER_URL" + ) +else + CONTAINER_BUILD_OPTIONS+=( + --file "${CONTAINER_FILE}" + . + ) +fi + +# +# Caching +# +if docker inspect "$CONTAINER_IMAGE_LATEST" >/dev/null 2>&1; then + CONTAINER_BUILD_OPTIONS+=( + --cache-from "${CONTAINER_IMAGE_LATEST}" + ) +fi +if docker inspect "$CONTAINER_IMAGE_REVISION" >/dev/null 2>&1; then + CONTAINER_BUILD_OPTIONS+=( + --cache-from "${CONTAINER_IMAGE_REVISION}" + ) +fi + export CONTAINER_SKIP_BUILD export CONTAINER_BUILD_OPTIONS diff --git a/scripts/include/env-container-run b/scripts/include/env-container-run index 0ab09757..9df7aa71 100644 --- a/scripts/include/env-container-run +++ b/scripts/include/env-container-run @@ -14,7 +14,6 @@ CONTAINER_AWS_OPTIONS=( --env AWS_ACCOUNT --env AWS_SECURITY_TOKEN --env AWS_DEFAULT_REGION - --env JSII_SILENCE_WARNING_UNTESTED_NODE_VERSION=True ) CONTAINER_RUN_OPTIONS=( --interactive