From 1a91f9534b26aa694dd7da621c79a0763c68a606 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Chastanet?= Date: Sun, 12 Nov 2023 14:14:07 +0100 Subject: [PATCH] use bash-tools-framework pre-commit hooks --- .framework-config | 4 + .github/workflows/lint-test.yml | 119 ++++++++++-------- .gitignore | 9 +- .pre-commit-config.yaml | 19 ++- Commands.tmpl.md | 71 +---------- bin/installRequirements | 4 +- src/_binaries/DbImport/dbImportProfile.bats | 5 +- src/_binaries/Utils/waitForMysql.bats | 2 +- .../build/installRequirements.options.tpl | 7 -- 9 files changed, 88 insertions(+), 152 deletions(-) diff --git a/.framework-config b/.framework-config index fca7f907..e3320e82 100755 --- a/.framework-config +++ b/.framework-config @@ -1,6 +1,10 @@ #!/usr/bin/env bash # shellcheck disable=SC2034 +REAL_SCRIPT_FILE="$(readlink -e "$(realpath "${BASH_SOURCE[0]}")")" +FRAMEWORK_ROOT_DIR="$(cd "$(readlink -e "${REAL_SCRIPT_FILE%/*}")" && pwd -P)" +FRAMEWORK_SRC_DIR="${FRAMEWORK_ROOT_DIR}/src" + # describe the functions that will be skipped from being imported FRAMEWORK_FUNCTIONS_IGNORE_REGEXP='^(Namespace::functions|Functions::myFunction|Namespace::requireSomething|IMPORT::dir::file|Acquire::ForceIPv4)$' # describe the files that do not contain function to be imported diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index f946e9bc..2502eb00 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -1,7 +1,7 @@ --- # Lint the code base and launch unit test at each push or pull request name: Lint and test -on: +on: # yamllint disable-line rule:truthy push: workflow_dispatch: @@ -10,55 +10,44 @@ jobs: runs-on: ubuntu-22.04 strategy: fail-fast: true - matrix: - vendor: - - ubuntu steps: - - name: Checkout Code - uses: actions/checkout@v3 + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 + + - name: Install pre-commit + run: pip install pre-commit - uses: ouzi-dev/commit-status-updater@v2 with: name: lint status: pending - - name: Install vendors - run: | - set -exo pipefail - ./bin/installRequirements - - - name: Linting - Shellcheck - run: | - set -exo pipefail - ./bin/shellcheckLint -f checkstyle | tee logs/shellcheck-checkstyle.xml - - - name: Linting - Framework - run: | - set -exo pipefail - ./bin/frameworkLint -f checkstyle | tee logs/framework-checkstyle.xml - - - name: Linting - Awk - run: | - set -exo pipefail - ./bin/awkLint | tee logs/awk-checkstyle.xml - - - name: build bin files + check md5 - run: | - set -exo pipefail - ./bin/buildBinFiles 2>&1 | tee logs/buidBinFiles.log - - - name: Checkstyle aggregation - uses: jwgmeligmeyling/checkstyle-github-action@master - with: - path: 'logs/*-checkstyle.xml' + - name: Run pre-commit + run: pre-commit run -a --hook-stage manual - name: Archive results + if: ${{ always() }} + continue-on-error: true uses: actions/upload-artifact@v3 with: name: linter-reports path: | - logs/*.xml - logs/*.log + megalinter-reports/** + + - name: Create Pull Request + if: ${{ failure() }} + uses: peter-evans/create-pull-request@v4 + with: + branch: update/pre-commit-fixes + title: lint fixes + commit-message: Auto-update lint fixes + body: | + some auto fixes have been generated during pre-commit run + labels: updates - uses: ouzi-dev/commit-status-updater@v2 if: always() @@ -82,10 +71,10 @@ jobs: include: - vendor: ubuntu bashImage: ubuntu:20.04 - options: -j 1 + options: -j 30 - vendor: alpine bashImage: bash - options: -j 1 + options: -j 30 steps: - name: Checkout uses: actions/checkout@v3 @@ -110,21 +99,25 @@ jobs: ./bin/installRequirements + chmod -R 777 logs + # shellcheck disable=SC2266 - VENDOR="${{ matrix.vendor }}" \ - BASH_TAR_VERSION="${{ matrix.bashTarVersion }}" \ - BASH_IMAGE="${{ matrix.bashImage }}" \ - SKIP_BUILD=0 \ - SKIP_USER=1 \ - ./bin/test ${{matrix.options}} --report-formatter junit -o logs -r src - - - name: Publish Unit Test Results - uses: EnricoMi/publish-unit-test-result-action@v1 + USER_ID=1000 \ + GROUP_ID=1000 \ + ./bin/test \ + -vvv \ + --vendor "${{matrix.vendor}}" \ + --bash-version "${{matrix.bashTarVersion}}" \ + --bash-base-image "${{matrix.bashImage}}" \ + --branch-name "${BRANCH}" \ + ${{matrix.options}} --report-formatter junit -o logs -r src --ci + + - name: Upload Test Results if: always() + uses: actions/upload-artifact@v3 with: - files: logs/report.xml - check_name: - 'Unit Test Results ${{matrix.vendor}} ${{matrix.bashTarVersion}}' + name: Test Results ${{matrix.vendor}} ${{matrix.bashTarVersion}} + path: logs/report.xml - uses: ouzi-dev/commit-status-updater@v2 if: always() @@ -132,12 +125,30 @@ jobs: name: build bash-tools-${{matrix.vendor}}-${{matrix.bashTarVersion}} status: ${{ job.status }} - buildResults: + publishTestResults: + name: 'Publish Tests Results' if: ${{ always() }} - runs-on: ubuntu-latest - name: Final Build Results needs: [linters, tests] + runs-on: ubuntu-latest + permissions: + checks: write + + # needed by ouzi-dev/commit-status-updater@v2 + statuses: write + + # only needed unless run with comment_mode: off + pull-requests: write steps: + - name: Download Artifacts + uses: actions/download-artifact@v3 + with: + path: artifacts + + - name: Checkstyle aggregation + uses: lcollins/checkstyle-github-action@v2.0.0 + with: + path: 'artifacts/**/*.xml' + # run this action to get the workflow conclusion # You can get the conclusion via env (env.WORKFLOW_CONCLUSION) - uses: technote-space/workflow-conclusion-action@v3 diff --git a/.gitignore b/.gitignore index 9bb824d1..efb08e98 100644 --- a/.gitignore +++ b/.gitignore @@ -12,14 +12,7 @@ !/**/_sidebar.md !/**/_navbar.md -/bin/awkLint -/bin/buildBinFiles -/bin/generateShellDoc -/bin/megalinter -/bin/runBuildContainer -/bin/shellcheckLint +bin/runBuildContainer /bin/test -/bin/findShebangFiles /bin/buildPushDockerImages /bin/buildPushDockerImage -/bin/frameworkLint diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 922f4bcd..41cba6b7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -67,7 +67,7 @@ repos: stages: [commit] - repo: https://github.com/fchastanet/bash-tools-framework - rev: 1.0.4 + rev: 1.1.0 hooks: - id: fixShebangExecutionBit - id: frameworkLinter @@ -79,19 +79,14 @@ repos: plain, --theme, default-force, + --bash-framework-config, + .framework-config, ] - id: shellcheckLint - - - repo: local - hooks: - id: buildShFiles - name: build sh files - entry: bash -c './bin/buildBinFiles --ignore-missing' - language: system - always_run: true - require_serial: true - fail_fast: true - stages: [commit] + args: [--bash-framework-config, .framework-config, --ignore-missing] + - id: buildShFilesGithubAction + args: [--bash-framework-config, .framework-config] - repo: local hooks: @@ -105,7 +100,7 @@ repos: stages: [push] - repo: https://github.com/fchastanet/bash-tools-framework - rev: 1.0.4 + rev: 1.1.0 hooks: - id: runUnitTests - id: plantuml diff --git a/Commands.tmpl.md b/Commands.tmpl.md index 899cd2c1..d59164a5 100644 --- a/Commands.tmpl.md +++ b/Commands.tmpl.md @@ -6,15 +6,8 @@ - [1.3. bin/waitForMysql](#13-binwaitformysql) - [1.4. bin/doc](#14-bindoc) - [1.5. bin/findShebangFiles](#15-binfindshebangfiles) - - [1.6. bin/dockerLint](#16-bindockerlint) - - [1.7. bin/shellcheckLint](#17-binshellchecklint) - - [1.8. bin/awkLint](#18-binawklint) - - [1.9. bin/frameworkLint](#19-binframeworklint) - - [1.10. bin/megalinter](#110-binmegalinter) - - [1.11. .github/workflows/buildBinFiles](#111-githubworkflowsbuildbinfiles) - - [1.12. bin/test](#112-bintest) - - [1.13. bin/runBuildContainer](#113-binrunbuildcontainer) - - [1.14. bin/buildPushDockerImage](#114-binbuildpushdockerimage) + - [1.6. bin/test](#16-bintest) + - [1.7. bin/runBuildContainer](#17-binrunbuildcontainer) - [2. Converter and Generator tools](#2-converter-and-generator-tools) - [2.1. bin/generateShellDoc](#21-bingenerateshelldoc) - [2.2. bin/mysql2puml](#22-binmysql2puml) @@ -77,55 +70,7 @@ imported from bash-tools-framework @@@findShebangFiles_help@@@ ``` -### 1.6. bin/dockerLint - -imported from bash-tools-framework - -```text -@@@dockerLint_help@@@ -``` - -### 1.7. bin/shellcheckLint - -imported from bash-tools-framework - -```text -@@@shellcheckLint_help@@@ -``` - -### 1.8. bin/awkLint - -imported from bash-tools-framework - -```text -@@@awkLint_help@@@ -``` - -### 1.9. bin/frameworkLint - -imported from bash-tools-framework - -```text -@@@frameworkLint_help@@@ -``` - -### 1.10. bin/megalinter - -imported from bash-tools-framework - -```text -@@@megalinter_help@@@ -``` - -### 1.11. .github/workflows/buildBinFiles - -imported from bash-tools-framework - -```text -@@@buildBinFiles_help@@@ -``` - -### 1.12. bin/test +### 1.6. bin/test imported from bash-tools-framework @@ -133,7 +78,7 @@ imported from bash-tools-framework @@@test_help@@@ ``` -### 1.13. bin/runBuildContainer +### 1.7. bin/runBuildContainer imported from bash-tools-framework @@ -141,14 +86,6 @@ imported from bash-tools-framework @@@runBuildContainer_help@@@ ``` -### 1.14. bin/buildPushDockerImage - -imported from bash-tools-framework - -```text -@@@buildPushDockerImage_help@@@ -``` - ## 2. Converter and Generator tools ### 2.1. bin/generateShellDoc diff --git a/bin/installRequirements b/bin/installRequirements index f6a3b8f5..c0590875 100755 --- a/bin/installRequirements +++ b/bin/installRequirements @@ -1552,7 +1552,7 @@ installRequirementsCommand() { INSTALLS REQUIREMENTS: - fchastanet/bash-tools-framework - and fchastanet/bash-tools-framework useful binaries: - bin/awkLint, bin/buildBinFiles, bin/frameworkLint, bin/findShebangFiles, bin/megalinter, bin/runBuildContainer, bin/shellcheckLint, bin/test, bin/buildPushDockerImage""" + bin/runBuildContainer, bin/test""" echo echo -n -e "${__HELP_TITLE_COLOR}VERSION: ${__RESET_COLOR}" echo '1.0' @@ -1572,7 +1572,7 @@ installRequirementsCommand() { return 1 fi } -declare -a externalBinaries=([0]="bin/awkLint" [1]="bin/buildBinFiles" [2]="bin/frameworkLint" [3]="bin/findShebangFiles" [4]="bin/megalinter" [5]="bin/runBuildContainer" [6]="bin/shellcheckLint" [7]="bin/test" [8]="bin/buildPushDockerImage") +declare -a externalBinaries=([0]="bin/runBuildContainer" [1]="bin/test") installRequirementsCommand parse "${BASH_FRAMEWORK_ARGV[@]}" diff --git a/src/_binaries/DbImport/dbImportProfile.bats b/src/_binaries/DbImport/dbImportProfile.bats index 320a65a8..9a40d19f 100755 --- a/src/_binaries/DbImport/dbImportProfile.bats +++ b/src/_binaries/DbImport/dbImportProfile.bats @@ -98,7 +98,10 @@ function Database::dbImportProfile::remote_db_fully_functional_default_ratio { # run "${binDir}/dbImportProfile" --verbose -f default.local fromDb 2>&1 [[ -f "${HOME}/tableSizeQuery.sql" ]] - assert_output --partial "Profile generated - 1/3 tables bigger than 70% of max table size (29MB) automatically excluded" + assert_lines_count 3 + assert_line --index 0 --partial "INFO - Using from dsn" + assert_line --index 1 --partial "INFO - Profile generated - 1/3 tables bigger than 70% of max table size (29MB) automatically excluded" + assert_line --index 2 --partial "INFO - File saved" diff >&3 "${HOME}/tableSizeQuery.sql" "${BATS_TEST_DIRNAME}/testsData/expectedDbImportProfileTableListQuery.sql" [[ -f "${HOME}/.bash-tools/dbImportProfiles/auto_default.local_fromDb.sh" ]] diff -u "${HOME}/.bash-tools/dbImportProfiles/auto_default.local_fromDb.sh" \ diff --git a/src/_binaries/Utils/waitForMysql.bats b/src/_binaries/Utils/waitForMysql.bats index 58dfff32..6bd011a9 100755 --- a/src/_binaries/Utils/waitForMysql.bats +++ b/src/_binaries/Utils/waitForMysql.bats @@ -93,6 +93,6 @@ function Utils::waitForMysql::mysqlNotAvailableAfter1SecondTimeout { #@test assert_failure 2 assert_line --index 0 --partial "INFO - Waiting for mysql" assert_line --index 1 --partial "." - assert_line --index 2 --partial "ERROR - waitForMysql - timeout for localhost:3306 occurred after 1 seconds" + assert_line --index 2 --partial "ERROR - waitForMysql - timeout for localhost:3306 occurred after" assert_lines_count 3 } diff --git a/src/_binaries/build/installRequirements.options.tpl b/src/_binaries/build/installRequirements.options.tpl index 8697294e..02df895f 100644 --- a/src/_binaries/build/installRequirements.options.tpl +++ b/src/_binaries/build/installRequirements.options.tpl @@ -1,14 +1,7 @@ % declare -a externalBinaries=( - bin/awkLint - bin/buildBinFiles - bin/frameworkLint - bin/findShebangFiles - bin/megalinter bin/runBuildContainer - bin/shellcheckLint bin/test - bin/buildPushDockerImage ) declare versionNumber="1.0" declare commandFunctionName="installRequirementsCommand"