From d02e4636e0b4a1ef395f0f197fd35737852cdaab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Chastanet?= Date: Sun, 12 Nov 2023 17:50:25 +0100 Subject: [PATCH] - updated pre-commit hooks to 1.1.5 - added dependabot to check version used in github actionsa - align github workflow with bash-tools-framework --- .framework-config | 36 +++-- .github/dependabot.yml | 11 ++ .github/workflows/lint-test.yml | 140 +++++++++++------- .pre-commit-config.yaml | 69 +++++---- .v8rrc.yaml | 21 +++ bin/dbImportProfile | 2 +- src/_binaries/DbImport/dbImportProfile.sh | 2 +- .../testsData/auto_default.local_fromDb_20.sh | 4 +- .../testsData/auto_default.local_fromDb_70.sh | 6 +- 9 files changed, 192 insertions(+), 99 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .v8rrc.yaml diff --git a/.framework-config b/.framework-config index e3320e82..a1cca9e2 100755 --- a/.framework-config +++ b/.framework-config @@ -1,24 +1,34 @@ #!/usr/bin/env bash # shellcheck disable=SC2034 -REAL_SCRIPT_FILE="$(readlink -e "$(realpath "${BASH_SOURCE[0]}")")" -FRAMEWORK_ROOT_DIR="$(cd "$(readlink -e "${REAL_SCRIPT_FILE%/*}")" && pwd -P)" -FRAMEWORK_SRC_DIR="${FRAMEWORK_ROOT_DIR}/src" +REAL_SCRIPT_FILE="${REAL_SCRIPT_FILE:-$(readlink -e "$(realpath "${BASH_SOURCE[0]}")")}" +FRAMEWORK_ROOT_DIR="${FRAMEWORK_ROOT_DIR:-$(cd "$(readlink -e "${REAL_SCRIPT_FILE%/*}")" && pwd -P)}" +FRAMEWORK_SRC_DIR="${FRAMEWORK_SRC_DIR:-${FRAMEWORK_ROOT_DIR}/src}" +FRAMEWORK_BIN_DIR="${FRAMEWORK_BIN_DIR:-${FRAMEWORK_ROOT_DIR}/bin}" +FRAMEWORK_VENDOR_DIR="${FRAMEWORK_VENDOR_DIR:-${FRAMEWORK_ROOT_DIR}/vendor}" +FRAMEWORK_VENDOR_BIN_DIR="${FRAMEWORK_VENDOR_BIN_DIR:-${FRAMEWORK_ROOT_DIR}/vendor/bin}" # describe the functions that will be skipped from being imported -FRAMEWORK_FUNCTIONS_IGNORE_REGEXP='^(Namespace::functions|Functions::myFunction|Namespace::requireSomething|IMPORT::dir::file|Acquire::ForceIPv4)$' +FRAMEWORK_FUNCTIONS_IGNORE_REGEXP="${FRAMEWORK_FUNCTIONS_IGNORE_REGEXP:-^(Namespace::functions|Functions::myFunction|Namespace::requireSomething|IMPORT::dir::file|Acquire::ForceIPv4)$}" # describe the files that do not contain function to be imported -NON_FRAMEWORK_FILES_REGEXP="(^bin/|.framework-config|^install$|.bats$|/testsData/|^manualTests/|/_.sh$|/ZZZ.sh$|/__all.sh$|^src/_binaries|^src/_includes|^src/batsHeaders.sh$|^conf)" +NON_FRAMEWORK_FILES_REGEXP="${NON_FRAMEWORK_FILES_REGEXP:-(^bin/|.framework-config|^install$|.bats$|/testsData/|^manualTests/|/_.sh$|/ZZZ.sh$|/__all.sh$|^src/_binaries|^src/_includes|^src/batsHeaders.sh$|^conf)}" # describe the files that are allowed to not have an associated bats file -BATS_FILE_NOT_NEEDED_REGEXP="(^bin/|.framework-config|^install$|.bats$|/testsData/|^manualTests/|/_.sh$|/ZZZ.sh$|/__all.sh$|^src/batsHeaders.sh$|^src/_includes)" +BATS_FILE_NOT_NEEDED_REGEXP="${BATS_FILE_NOT_NEEDED_REGEXP:-(^bin/|.framework-config|^install$|.bats$|/testsData/|^manualTests/|/_.sh$|/ZZZ.sh$|/__all.sh$|^src/batsHeaders.sh$|^src/_includes)}" # describe the files that are allowed to not have a function matching the filename -FRAMEWORK_FILES_FUNCTION_MATCHING_IGNORE_REGEXP="^conf/|^bin/|^\.framework-config$|^build.sh$|\.tpl$|testsData/binaryFile$" +FRAMEWORK_FILES_FUNCTION_MATCHING_IGNORE_REGEXP="${FRAMEWORK_FILES_FUNCTION_MATCHING_IGNORE_REGEXP:-^conf/|^bin/|^\.framework-config$|^build.sh$|\.tpl$|testsData/binaryFile$}" # Source directories -FRAMEWORK_SRC_DIRS=( - "${FRAMEWORK_ROOT_DIR}/src" - "${FRAMEWORK_ROOT_DIR}/vendor/bash-tools-framework/src" -) +if [[ ! -v FRAMEWORK_SRC_DIRS ]]; then + FRAMEWORK_SRC_DIRS=( + "${FRAMEWORK_ROOT_DIR}/src" + "${FRAMEWORK_ROOT_DIR}/vendor/bash-tools-framework/src" + ) +fi -export REPOSITORY_URL="https://github.com/fchastanet/bash-tools" +# export here all the variables that will be used in your templates +export REPOSITORY_URL="${REPOSITORY_URL:-https://github.com/fchastanet/bash-tools}" -export BASH_FRAMEWORK_DISPLAY_LEVEL="3" +BASH_FRAMEWORK_THEME="${BASH_FRAMEWORK_THEME:-default}" +BASH_FRAMEWORK_LOG_LEVEL="${BASH_FRAMEWORK_LOG_LEVEL:-0}" +BASH_FRAMEWORK_DISPLAY_LEVEL="${BASH_FRAMEWORK_DISPLAY_LEVEL:-3}" +BASH_FRAMEWORK_LOG_FILE="${BASH_FRAMEWORK_LOG_FILE:-${FRAMEWORK_ROOT_DIR}/logs/$(basename "$0").log}" +BASH_FRAMEWORK_LOG_FILE_MAX_ROTATION="${BASH_FRAMEWORK_LOG_FILE_MAX_ROTATION:-5}" diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..ea9353ca --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +# Set update schedule for GitHub Actions + +version: 2 +updates: + - package-ecosystem: 'github-actions' + directory: '/' + schedule: + # Check for updates to GitHub Actions every week + interval: 'weekly' + day: 'friday' + open-pull-requests-limit: 1 diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 2502eb00..056bff3d 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -3,34 +3,113 @@ name: Lint and test on: # yamllint disable-line rule:truthy push: + # execute when pushing only branches, not tags + branches: + - '**' workflow_dispatch: jobs: - linters: + build: runs-on: ubuntu-22.04 strategy: fail-fast: true + matrix: + vendor: + - ubuntu + - alpine + bashTarVersion: + - '4.4' + - '5.0' + - '5.1' + include: + - vendor: ubuntu + bashImage: ubuntu:20.04 + batsOptions: -j 30 + bashTarVersion: 4.4 + runPrecommitTests: false + - vendor: ubuntu + bashImage: ubuntu:20.04 + bashTarVersion: 5.0 + batsOptions: -j 30 + runPrecommitTests: false + - vendor: ubuntu + bashImage: ubuntu:20.04 + bashTarVersion: 5.1 + batsOptions: -j 30 + runPrecommitTests: true + - vendor: alpine + bashTarVersion: 4.4 + bashImage: amd64/bash:4.4-alpine3.18 + batsOptions: -j 30 + runPrecommitTests: false + - vendor: alpine + bashTarVersion: 5.0 + bashImage: amd64/bash:5.0-alpine3.18 + batsOptions: -j 30 + runPrecommitTests: false + - vendor: alpine + bashTarVersion: 5.1 + bashImage: amd64/bash:5.1-alpine3.18 + batsOptions: -j 30 + runPrecommitTests: false steps: - - uses: actions/checkout@v3 + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - uses: ouzi-dev/commit-status-updater@v2 + with: + name: build bash-tools-${{matrix.vendor}}-${{matrix.bashTarVersion}} + status: pending + + - name: Pull/Build/Push docker image + env: + BRANCH: ${GITHUB_REF##*/} + run: | + vendor/bash-tools-framework/bin/buildPushDockerImage \ + --vendor "${{ matrix.vendor }}" \ + --bash-version "${{ matrix.bashTarVersion }}" \ + --bash-base-image "${{ matrix.bashImage }}" \ + --branch-name "${GITHUB_REF##*/}" \ + --push -vvv + + - name: Check image + run: | + docker run --rm \ + "build:bash-tools-${{ matrix.vendor }}-${{ matrix.bashTarVersion }}" bash \ + --version + # only if pre-commit - name: Set up Python + if: matrix.runPrecommitTests uses: actions/setup-python@v4 with: python-version: 3.9 - name: Install pre-commit + if: matrix.runPrecommitTests run: pip install pre-commit - uses: ouzi-dev/commit-status-updater@v2 + if: matrix.runPrecommitTests with: name: lint status: pending - name: Run pre-commit + if: matrix.runPrecommitTests run: pre-commit run -a --hook-stage manual - name: Archive results - if: ${{ always() }} + if: matrix.runPrecommitTests && always() continue-on-error: true uses: actions/upload-artifact@v3 with: @@ -39,8 +118,8 @@ jobs: megalinter-reports/** - name: Create Pull Request - if: ${{ failure() }} - uses: peter-evans/create-pull-request@v4 + if: matrix.runPrecommitTests && failure() + uses: peter-evans/create-pull-request@v5 with: branch: update/pre-commit-fixes title: lint fixes @@ -50,57 +129,19 @@ jobs: labels: updates - uses: ouzi-dev/commit-status-updater@v2 - if: always() + if: matrix.runPrecommitTests && always() with: name: lint status: ${{ job.status }} - tests: - needs: linters - runs-on: ubuntu-22.04 - strategy: - fail-fast: true - matrix: - vendor: - - ubuntu - - alpine - bashTarVersion: - - '4.4' - - '5.0' - - '5.1' - include: - - vendor: ubuntu - bashImage: ubuntu:20.04 - options: -j 30 - - vendor: alpine - bashImage: bash - options: -j 30 - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - uses: ouzi-dev/commit-status-updater@v2 - with: - name: build bash-tools-${{matrix.vendor}}-${{matrix.bashTarVersion}} - status: pending - + # Run unit tests - name: run unit tests run: | set -exo pipefail - ./bin/installRequirements + bin/installRequirements chmod -R 777 logs - # shellcheck disable=SC2266 USER_ID=1000 \ GROUP_ID=1000 \ @@ -109,8 +150,8 @@ jobs: --vendor "${{matrix.vendor}}" \ --bash-version "${{matrix.bashTarVersion}}" \ --bash-base-image "${{matrix.bashImage}}" \ - --branch-name "${BRANCH}" \ - ${{matrix.options}} --report-formatter junit -o logs -r src --ci + --branch-name "${GITHUB_REF##*/}" \ + ${{matrix.batsOptions}} --report-formatter junit -o logs -r src --ci - name: Upload Test Results if: always() @@ -128,7 +169,7 @@ jobs: publishTestResults: name: 'Publish Tests Results' if: ${{ always() }} - needs: [linters, tests] + needs: [build] runs-on: ubuntu-latest permissions: checks: write @@ -153,7 +194,6 @@ jobs: # You can get the conclusion via env (env.WORKFLOW_CONCLUSION) - uses: technote-space/workflow-conclusion-action@v3 - # run other action with the workflow conclusion - uses: ouzi-dev/commit-status-updater@v2 with: name: build diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fa13bef3..6328db0a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,5 @@ --- +default_stages: [pre-commit, manual] repos: - repo: local hooks: @@ -9,38 +10,29 @@ repos: always_run: true require_serial: true fail_fast: true - stages: [commit] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: - id: mixed-line-ending - stages: [commit] - id: end-of-file-fixer - stages: [commit] - id: check-executables-have-shebangs - stages: [commit] - id: check-shebang-scripts-are-executable - stages: [commit] - id: check-xml - stages: [commit] - id: check-yaml - stages: [commit] - id: forbid-new-submodules - stages: [commit] - id: mixed-line-ending args: [--fix=lf] - stages: [commit] - id: check-json exclude: | (?x)^( conf\/.vscode\/settings.json| + .vscode\/settings.json| .vscode\/launch.json )$ - stages: [commit] - - repo: https://github.com/jumanjihouse/pre-commit-hooks - rev: 3.0.0 + - repo: https://github.com/fchastanet/jumanjihouse-pre-commit-hooks + rev: 3.0.2 hooks: - id: shfmt args: [-i, '2', -ci] @@ -51,25 +43,27 @@ repos: ^conf/dbScripts/| ^install ) - stages: [commit] + stages: [pre-commit] # Check both committed and uncommitted files for git conflict # markers and whitespace errors according to core.whitespace # and conflict-marker-size configuration in a git repo. - id: git-check exclude: /testsData/ - stages: [commit] - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.3 + rev: v3.1.0 hooks: - id: prettier - stages: [commit] - repo: https://github.com/fchastanet/bash-tools-framework - rev: 1.1.3 + rev: 1.1.5 hooks: - id: fixShebangExecutionBit + - id: fixShebangExecutionBitGithubActions + - id: awkLint + - id: shellcheckLint + - id: shellcheckLintGithubAction - id: frameworkLinter args: [ @@ -79,31 +73,48 @@ repos: plain, --theme, default-force, - --bash-framework-config, - .framework-config, ] - - id: shellcheckLint + - id: frameworkLinterGithubAction + args: + [ + --expected-warnings-count, + '6', + --format, + checkstyle, + --theme, + default-force, + ] + - id: plantuml - id: buildShFiles - args: [--bash-framework-config, .framework-config, --ignore-missing] - id: buildShFilesGithubAction - args: [--bash-framework-config, .framework-config] - repo: local hooks: - - id: buildDocFiles + - &BUILD_DOC_FILES + id: buildDocFiles name: build doc files - entry: bash -c './bin/doc' - language: system - always_run: true + language: script + entry: bin/doc + args: [--verbose, --skip-docker-build] + pass_filenames: false require_serial: true + always_run: true fail_fast: true - stages: [push] + stages: [pre-push] + - <<: *BUILD_DOC_FILES + id: buildDocFilesGithubAction + name: build doc files for Github Actions + args: [--verbose] + fail_fast: false + stages: [manual] + # manual stage used to select github action with megalinter full config - repo: https://github.com/fchastanet/bash-tools-framework - rev: 1.1.3 + rev: 1.1.5 hooks: - id: runUnitTests - - id: plantuml + # not manual as github will run UT with several versions of bash and arch + stages: [pre-commit] - id: megalinterCheckVersion - id: megalinter - id: megalinterGithubAction diff --git a/.v8rrc.yaml b/.v8rrc.yaml new file mode 100644 index 00000000..d68b9cfa --- /dev/null +++ b/.v8rrc.yaml @@ -0,0 +1,21 @@ +# - Level of verbose logging. 0 is standard, higher numbers are more verbose +# - overridden by passing --verbose / -v +# - default = 0 +verbose: 0 + +# - Remove cached HTTP responses older than cacheTtl seconds old. +# Specifying 0 clears and disables cache completely +# - overridden by passing --cache-ttl +# - default = 600 +cacheTtl: 86400 + +customCatalog: + schemas: + - name: prettier + description: prettier + fileMatch: ['.prettierrc.yaml'] + location: https://json.schemastore.org/prettierrc.json + - name: megalinter + description: megalinter + fileMatch: ['.mega-linter*.yml'] + location: https://raw.githubusercontent.com/megalinter/megalinter/main/megalinter/descriptors/schemas/megalinter-configuration.jsonschema.json diff --git a/bin/dbImportProfile b/bin/dbImportProfile index 7b05dd83..66f5282d 100755 --- a/bin/dbImportProfile +++ b/bin/dbImportProfile @@ -2028,7 +2028,7 @@ run() { else excludedTablesCount=$((excludedTablesCount + 1)) fi - echo " grep -v '^${tableName}$' | # table size ${tableSize}MB" + echo " grep -v '^${tableName}$' | # table size ${tableSize}MB" done < <(echo "${tableList}") echo "cat" tablesCount="$(echo "${tableList}" | wc -l)" diff --git a/src/_binaries/DbImport/dbImportProfile.sh b/src/_binaries/DbImport/dbImportProfile.sh index 8f116a42..bc4a7c53 100755 --- a/src/_binaries/DbImport/dbImportProfile.sh +++ b/src/_binaries/DbImport/dbImportProfile.sh @@ -68,7 +68,7 @@ run() { else excludedTablesCount=$((excludedTablesCount + 1)) fi - echo " grep -v '^${tableName}$' | # table size ${tableSize}MB" + echo " grep -v '^${tableName}$' | # table size ${tableSize}MB" done < <(echo "${tableList}") echo "cat" tablesCount="$(echo "${tableList}" | wc -l)" diff --git a/src/_binaries/DbImport/testsData/auto_default.local_fromDb_20.sh b/src/_binaries/DbImport/testsData/auto_default.local_fromDb_20.sh index 726241d0..2ec8a5fd 100755 --- a/src/_binaries/DbImport/testsData/auto_default.local_fromDb_20.sh +++ b/src/_binaries/DbImport/testsData/auto_default.local_fromDb_20.sh @@ -2,7 +2,7 @@ # cat represents the whole list of tables cat | - grep -v '^table1$' | # table size 29MB - grep -v '^table2$' | # table size 10MB + grep -v '^table1$' | # table size 29MB + grep -v '^table2$' | # table size 10MB # grep -v '^table3$' | # table size 4MB cat diff --git a/src/_binaries/DbImport/testsData/auto_default.local_fromDb_70.sh b/src/_binaries/DbImport/testsData/auto_default.local_fromDb_70.sh index 1a491a9e..8dcd2195 100755 --- a/src/_binaries/DbImport/testsData/auto_default.local_fromDb_70.sh +++ b/src/_binaries/DbImport/testsData/auto_default.local_fromDb_70.sh @@ -2,7 +2,7 @@ # cat represents the whole list of tables cat | - grep -v '^table1$' | # table size 29MB -# grep -v '^table2$' | # table size 10MB -# grep -v '^table3$' | # table size 4MB + grep -v '^table1$' | # table size 29MB +# grep -v '^table2$' | # table size 10MB +# grep -v '^table3$' | # table size 4MB cat