From 13b69a2f981c3fb44e15d9a26a061d2f3164b31c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Chastanet?= Date: Sun, 7 Jan 2024 23:08:10 +0100 Subject: [PATCH] fixed UT flakiness --- .github/workflows/lint-test.yml | 12 ++-- .pre-commit-config.yaml | 2 +- bin/cli | 2 +- bin/dbImport | 44 ++++++++++++- bin/dbImportProfile | 2 +- bin/dbImportStream | 62 +++++++++++++++---- bin/dbQueryAllDatabases | 42 +++++++++++-- bin/dbScriptAllDatabases | 6 +- bin/doc | 2 +- bin/gitIsAncestorOf | 2 +- bin/gitIsBranch | 2 +- bin/gitRenameBranch | 2 +- bin/installRequirements | 2 +- bin/mysql2puml | 4 +- bin/postmanCli | 40 +++++++++++- bin/upgradeGithubRelease | 4 +- bin/waitForIt | 4 +- bin/waitForMysql | 4 +- conf/dbImportProfiles/default.sh | 6 +- install | 2 +- src/Postman/Model/validate.bats | 15 +++-- src/Postman/api.bats | 2 + src/Postman/api.sh | 8 ++- src/_binaries/DbImport/dbImport.bats | 8 ++- src/_binaries/DbImport/dbImport.sh | 6 +- src/_binaries/DbImport/dbImportStream.sh | 24 ++++--- .../testsData/dbImportProfiles/default.sh | 6 +- 27 files changed, 245 insertions(+), 70 deletions(-) diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index b9496822..6006ee44 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -34,33 +34,33 @@ jobs: include: - vendor: ubuntu bashImage: ubuntu:20.04 - batsOptions: -j 5 + batsOptions: -j 30 bashTarVersion: 4.4 runPrecommitTests: false - vendor: ubuntu bashImage: ubuntu:20.04 bashTarVersion: 5.0 - batsOptions: -j 5 + batsOptions: -j 30 runPrecommitTests: false - vendor: ubuntu bashImage: ubuntu:20.04 bashTarVersion: 5.1 - batsOptions: -j 5 + batsOptions: -j 30 runPrecommitTests: true - vendor: alpine bashTarVersion: 4.4 bashImage: amd64/bash:4.4-alpine3.18 - batsOptions: -j 5 + batsOptions: -j 30 runPrecommitTests: false - vendor: alpine bashTarVersion: 5.0 bashImage: amd64/bash:5.0-alpine3.18 - batsOptions: -j 5 + batsOptions: -j 30 runPrecommitTests: false - vendor: alpine bashTarVersion: 5.1 bashImage: amd64/bash:5.1-alpine3.18 - batsOptions: -j 5 + batsOptions: -j 30 runPrecommitTests: false steps: - name: Checkout diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ad7dd902..92fd961b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -76,7 +76,7 @@ repos: exclude: /testsData/ - repo: https://github.com/fchastanet/bash-tools-framework - rev: 2.2.0 + rev: 2.2.1 hooks: - id: fixShebangExecutionBit - id: fixShebangExecutionBitGithubActions diff --git a/bin/cli b/bin/cli index 71eac343..f98a7d15 100755 --- a/bin/cli +++ b/bin/cli @@ -1672,7 +1672,7 @@ cliCommand() { # shellcheck disable=SC2054 helpArray=(choose\ color\ theme\ -\ default-force\ means\ colors\ will\ be\ produced\ even\ if\ command\ is\ piped) echo -e " $(Array::wrap2 " " 76 4 "${helpArray[@]}")" - echo " Default value: default" + echo ' Default value: default' echo ' Possible values: default|default-force|noColor' echo -e " ${__HELP_OPTION_COLOR}--help${__HELP_NORMAL}, ${__HELP_OPTION_COLOR}-h${__HELP_NORMAL} {single}" local -a helpArray diff --git a/bin/dbImport b/bin/dbImport index ef851b43..725095a3 100755 --- a/bin/dbImport +++ b/bin/dbImport @@ -277,6 +277,36 @@ Assert::commandExists() { return 0 } +# @description ignore exit code 141 from simple command pipes +# @example use with: +# local resultingStatus=0 +# local -a originalPipeStatus=() +# cmd1 | cmd2 || Bash::handlePipelineFailure resultingStatus originalPipeStatus || true +# [[ "${resultingStatus}" = "0" ]] +# @arg $1 resultingStatusCode:&int (passed by reference) (optional) resulting status code +# @arg $2 originalStatus:int[] (passed by reference) (optional) copy of original PIPESTATUS array +# @env PIPESTATUS assuming that this function is called like in the example provided +# @see https://unix.stackexchange.com/a/709880/582856 +Bash::handlePipelineFailure() { + local -a pipeStatusBackup=("${PIPESTATUS[@]}") + local -n handlePipelineFailure_resultingStatusCode=$1 + local -n handlePipelineFailure_originalStatus=$2 + # shellcheck disable=SC2034 + handlePipelineFailure_originalStatus=("${pipeStatusBackup[@]}") + handlePipelineFailure_resultingStatusCode=0 + local statusCode + for statusCode in "${pipeStatusBackup[@]}"; do + if ((statusCode == 141)); then + return 0 + elif ((statusCode > 0)); then + # shellcheck disable=SC2034 + handlePipelineFailure_resultingStatusCode="${statusCode}" + break + fi + done + return "${handlePipelineFailure_resultingStatusCode}" +} + # @description get absolute conf file from specified conf folder deduced using these rules # * from absolute file (ignores and ) # * relative to where script is executed (ignores and ) @@ -832,8 +862,12 @@ Version::checkMinimal() { Assert::commandExists "${commandName}" "${help}" || return 2 + # shellcheck disable=SC2034 + local status=0 + # shellcheck disable=SC2034 + local -a pipeStatus=() local version - version="$("${commandName}" "${argVersion}" 2>&1 | ${parseVersionCallback})" + version="$("${commandName}" "${argVersion}" 2>&1 | ${parseVersionCallback} || Bash::handlePipelineFailure status pipeStatus)" Log::displayDebug "check ${commandName} version ${version} against minimal ${minimalVersion}" @@ -2194,7 +2228,7 @@ dbImportCommand() { # shellcheck disable=SC2054 helpArray=(choose\ color\ theme\ -\ default-force\ means\ colors\ will\ be\ produced\ even\ if\ command\ is\ piped) echo -e " $(Array::wrap2 " " 76 4 "${helpArray[@]}")" - echo " Default value: default" + echo ' Default value: default' echo ' Possible values: default|default-force|noColor' echo -e " ${__HELP_OPTION_COLOR}--help${__HELP_NORMAL}, ${__HELP_OPTION_COLOR}-h${__HELP_NORMAL} {single}" local -a helpArray @@ -2480,9 +2514,13 @@ run() { Log::displayInfo "avoid to create db structure" else Log::displayInfo "create db structure from ${remoteDbStructureDumpTempFile}" + # shellcheck disable=SC2034 + local status=0 + # shellcheck disable=SC2034 + local -a pipeStatus=() time ( pv "${remoteDbStructureDumpTempFile}" | zcat | - Database::query dbTargetDatabase "" "${targetDbName}" + Database::query dbTargetDatabase "" "${targetDbName}" || Bash::handlePipelineFailure status pipeStatus ) fi fi diff --git a/bin/dbImportProfile b/bin/dbImportProfile index fc1bb5d9..673e4b25 100755 --- a/bin/dbImportProfile +++ b/bin/dbImportProfile @@ -1807,7 +1807,7 @@ dbImportProfileCommand() { # shellcheck disable=SC2054 helpArray=(choose\ color\ theme\ -\ default-force\ means\ colors\ will\ be\ produced\ even\ if\ command\ is\ piped) echo -e " $(Array::wrap2 " " 76 4 "${helpArray[@]}")" - echo " Default value: default" + echo ' Default value: default' echo ' Possible values: default|default-force|noColor' echo -e " ${__HELP_OPTION_COLOR}--help${__HELP_NORMAL}, ${__HELP_OPTION_COLOR}-h${__HELP_NORMAL} {single}" local -a helpArray diff --git a/bin/dbImportStream b/bin/dbImportStream index 3104bced..50326de1 100755 --- a/bin/dbImportStream +++ b/bin/dbImportStream @@ -277,6 +277,36 @@ Assert::commandExists() { return 0 } +# @description ignore exit code 141 from simple command pipes +# @example use with: +# local resultingStatus=0 +# local -a originalPipeStatus=() +# cmd1 | cmd2 || Bash::handlePipelineFailure resultingStatus originalPipeStatus || true +# [[ "${resultingStatus}" = "0" ]] +# @arg $1 resultingStatusCode:&int (passed by reference) (optional) resulting status code +# @arg $2 originalStatus:int[] (passed by reference) (optional) copy of original PIPESTATUS array +# @env PIPESTATUS assuming that this function is called like in the example provided +# @see https://unix.stackexchange.com/a/709880/582856 +Bash::handlePipelineFailure() { + local -a pipeStatusBackup=("${PIPESTATUS[@]}") + local -n handlePipelineFailure_resultingStatusCode=$1 + local -n handlePipelineFailure_originalStatus=$2 + # shellcheck disable=SC2034 + handlePipelineFailure_originalStatus=("${pipeStatusBackup[@]}") + handlePipelineFailure_resultingStatusCode=0 + local statusCode + for statusCode in "${pipeStatusBackup[@]}"; do + if ((statusCode == 141)); then + return 0 + elif ((statusCode > 0)); then + # shellcheck disable=SC2034 + handlePipelineFailure_resultingStatusCode="${statusCode}" + break + fi + done + return "${handlePipelineFailure_resultingStatusCode}" +} + # @description get absolute conf file from specified conf folder deduced using these rules # * from absolute file (ignores and ) # * relative to where script is executed (ignores and ) @@ -742,8 +772,12 @@ Version::checkMinimal() { Assert::commandExists "${commandName}" "${help}" || return 2 + # shellcheck disable=SC2034 + local status=0 + # shellcheck disable=SC2034 + local -a pipeStatus=() local version - version="$("${commandName}" "${argVersion}" 2>&1 | ${parseVersionCallback})" + version="$("${commandName}" "${argVersion}" 2>&1 | ${parseVersionCallback} || Bash::handlePipelineFailure status pipeStatus)" Log::displayDebug "check ${commandName} version ${version} against minimal ${minimalVersion}" @@ -2006,7 +2040,7 @@ dbImportStreamCommand() { # shellcheck disable=SC2054 helpArray=(choose\ color\ theme\ -\ default-force\ means\ colors\ will\ be\ produced\ even\ if\ command\ is\ piped) echo -e " $(Array::wrap2 " " 76 4 "${helpArray[@]}")" - echo " Default value: default" + echo ' Default value: default' echo ' Possible values: default|default-force|noColor' echo -e " ${__HELP_OPTION_COLOR}--help${__HELP_NORMAL}, ${__HELP_OPTION_COLOR}-h${__HELP_NORMAL} {single}" local -a helpArray @@ -2149,7 +2183,7 @@ run() { Version::checkMinimal "gawk" "--version" "5.0.1" # create db instances - declare -Agx dbTargetInstance + local -Agx dbTargetInstance Database::newInstance dbTargetInstance "${optionTargetDsn}" Database::setQueryOptions dbTargetInstance "${dbTargetInstance[QUERY_OPTIONS]} --connect-timeout=5" @@ -2158,7 +2192,11 @@ run() { initializeDefaultTargetMysqlOptions dbTargetInstance "${argTargetDbName}" # TODO character set should be retrieved from dump files if possible - declare remoteCharacterSet="${optionCharacterSet:-${defaultRemoteCharacterSet}}" + local remoteCharacterSet="${optionCharacterSet:-${defaultRemoteCharacterSet}}" + + local status=0 + # shellcheck disable=SC2034 + local -a pipeStatus=() # shellcheck disable=2086 ( @@ -2167,19 +2205,19 @@ run() { elif [[ "${argDumpFile}" =~ \.gz$ ]]; then zcat "${argDumpFile}" fi - # zcat will continue to write to stdout whereas awk has finished if table has been found - # we detect this case because zcat will return code 141 because pipe closed - status=$? - if [[ "${status}" -eq "141" ]]; then true; else exit "${status}"; fi ) | awk \ -v PROFILE_COMMAND="${profileCommandFile}" \ -v CHARACTER_SET="${remoteCharacterSet}" \ --source "${awkScript}" \ - - | mysql \ - "--defaults-extra-file=${dbTargetInstance['AUTH_FILE']}" \ - ${dbTargetInstance['DB_IMPORT_OPTIONS']} \ - "${argTargetDbName}" || exit $? + - | + mysql \ + "--defaults-extra-file=${dbTargetInstance['AUTH_FILE']}" \ + ${dbTargetInstance['DB_IMPORT_OPTIONS']} \ + "${argTargetDbName}" || + # zcat will continue to write to stdout whereas awk has finished if table has been found + # we detect this case because zcat will return code 141 because pipe closed + Bash::handlePipelineFailure status pipeStatus } if [[ "${BASH_FRAMEWORK_QUIET_MODE:-0}" = "1" ]]; then diff --git a/bin/dbQueryAllDatabases b/bin/dbQueryAllDatabases index 8d87d2d5..414dabe4 100755 --- a/bin/dbQueryAllDatabases +++ b/bin/dbQueryAllDatabases @@ -733,8 +733,12 @@ Version::checkMinimal() { Assert::commandExists "${commandName}" "${help}" || return 2 + # shellcheck disable=SC2034 + local status=0 + # shellcheck disable=SC2034 + local -a pipeStatus=() local version - version="$("${commandName}" "${argVersion}" 2>&1 | ${parseVersionCallback})" + version="$("${commandName}" "${argVersion}" 2>&1 | ${parseVersionCallback} || Bash::handlePipelineFailure status pipeStatus)" Log::displayDebug "check ${commandName} version ${version} against minimal ${minimalVersion}" @@ -890,6 +894,36 @@ Assert::tty() { [[ -t 1 || -t 2 ]] } +# @description ignore exit code 141 from simple command pipes +# @example use with: +# local resultingStatus=0 +# local -a originalPipeStatus=() +# cmd1 | cmd2 || Bash::handlePipelineFailure resultingStatus originalPipeStatus || true +# [[ "${resultingStatus}" = "0" ]] +# @arg $1 resultingStatusCode:&int (passed by reference) (optional) resulting status code +# @arg $2 originalStatus:int[] (passed by reference) (optional) copy of original PIPESTATUS array +# @env PIPESTATUS assuming that this function is called like in the example provided +# @see https://unix.stackexchange.com/a/709880/582856 +Bash::handlePipelineFailure() { + local -a pipeStatusBackup=("${PIPESTATUS[@]}") + local -n handlePipelineFailure_resultingStatusCode=$1 + local -n handlePipelineFailure_originalStatus=$2 + # shellcheck disable=SC2034 + handlePipelineFailure_originalStatus=("${pipeStatusBackup[@]}") + handlePipelineFailure_resultingStatusCode=0 + local statusCode + for statusCode in "${pipeStatusBackup[@]}"; do + if ((statusCode == 141)); then + return 0 + elif ((statusCode > 0)); then + # shellcheck disable=SC2034 + handlePipelineFailure_resultingStatusCode="${statusCode}" + break + fi + done + return "${handlePipelineFailure_resultingStatusCode}" +} + # @description list files of dir with given extension and display it as a list one by line # # @arg $1 dir:String the directory to list @@ -1924,7 +1958,7 @@ dbQueryAllDatabasesCommand() { # shellcheck disable=SC2054 helpArray=(specify\ the\ number\ of\ db\ to\ query\ in\ parallel) echo -e " $(Array::wrap2 " " 76 4 "${helpArray[@]}")" - echo " Default value: 1" + echo ' Default value: 1' echo -e " ${__HELP_OPTION_COLOR}--bar${__HELP_NORMAL}, ${__HELP_OPTION_COLOR}-b${__HELP_NORMAL} {single}" local -a helpArray # shellcheck disable=SC2054 @@ -1935,7 +1969,7 @@ dbQueryAllDatabasesCommand() { # shellcheck disable=SC2054 helpArray=(character\ to\ use\ to\ separate\ mysql\ column) echo -e " $(Array::wrap2 " " 76 4 "${helpArray[@]}")" - echo " Default value: |" + echo ' Default value: |' echo echo -e "${__HELP_TITLE_COLOR}QUERY OPTIONS:${__RESET_COLOR}" echo -e " ${__HELP_OPTION_COLOR}--from-dsn${__HELP_NORMAL}, ${__HELP_OPTION_COLOR}-f ${__HELP_NORMAL} {single}" @@ -1985,7 +2019,7 @@ dbQueryAllDatabasesCommand() { # shellcheck disable=SC2054 helpArray=(choose\ color\ theme\ -\ default-force\ means\ colors\ will\ be\ produced\ even\ if\ command\ is\ piped) echo -e " $(Array::wrap2 " " 76 4 "${helpArray[@]}")" - echo " Default value: default" + echo ' Default value: default' echo ' Possible values: default|default-force|noColor' echo -e " ${__HELP_OPTION_COLOR}--help${__HELP_NORMAL}, ${__HELP_OPTION_COLOR}-h${__HELP_NORMAL} {single}" local -a helpArray diff --git a/bin/dbScriptAllDatabases b/bin/dbScriptAllDatabases index 67c8ae57..bee04e43 100755 --- a/bin/dbScriptAllDatabases +++ b/bin/dbScriptAllDatabases @@ -1891,7 +1891,7 @@ dbScriptAllDatabasesCommand() { # shellcheck disable=SC2054 helpArray=(specify\ the\ number\ of\ db\ to\ query\ in\ parallel) echo -e " $(Array::wrap2 " " 76 4 "${helpArray[@]}")" - echo " Default value: 1" + echo ' Default value: 1' echo -e " ${__HELP_OPTION_COLOR}--bar${__HELP_NORMAL}, ${__HELP_OPTION_COLOR}-b${__HELP_NORMAL} {single}" local -a helpArray # shellcheck disable=SC2054 @@ -1919,7 +1919,7 @@ dbScriptAllDatabasesCommand() { # shellcheck disable=SC2054 helpArray=(if\ output\ dir\ provided\,\ will\ log\ each\ db\ result\ to\ log\ file) echo -e " $(Array::wrap2 " " 76 4 "${helpArray[@]}")" - echo " Default value: none" + echo ' Default value: none' echo ' Possible values: none|log' echo echo -e "${__HELP_TITLE_COLOR}GLOBAL OPTIONS:${__RESET_COLOR}" @@ -1963,7 +1963,7 @@ dbScriptAllDatabasesCommand() { # shellcheck disable=SC2054 helpArray=(choose\ color\ theme\ -\ default-force\ means\ colors\ will\ be\ produced\ even\ if\ command\ is\ piped) echo -e " $(Array::wrap2 " " 76 4 "${helpArray[@]}")" - echo " Default value: default" + echo ' Default value: default' echo ' Possible values: default|default-force|noColor' echo -e " ${__HELP_OPTION_COLOR}--help${__HELP_NORMAL}, ${__HELP_OPTION_COLOR}-h${__HELP_NORMAL} {single}" local -a helpArray diff --git a/bin/doc b/bin/doc index 5630a250..a1c58809 100755 --- a/bin/doc +++ b/bin/doc @@ -1650,7 +1650,7 @@ docCommand() { # shellcheck disable=SC2054 helpArray=(choose\ color\ theme\ -\ default-force\ means\ colors\ will\ be\ produced\ even\ if\ command\ is\ piped) echo -e " $(Array::wrap2 " " 76 4 "${helpArray[@]}")" - echo " Default value: default" + echo ' Default value: default' echo ' Possible values: default|default-force|noColor' echo -e " ${__HELP_OPTION_COLOR}--help${__HELP_NORMAL}, ${__HELP_OPTION_COLOR}-h${__HELP_NORMAL} {single}" local -a helpArray diff --git a/bin/gitIsAncestorOf b/bin/gitIsAncestorOf index f772b8b5..e1004a39 100755 --- a/bin/gitIsAncestorOf +++ b/bin/gitIsAncestorOf @@ -1400,7 +1400,7 @@ gitIsAncestorOfCommand() { # shellcheck disable=SC2054 helpArray=(choose\ color\ theme\ -\ default-force\ means\ colors\ will\ be\ produced\ even\ if\ command\ is\ piped) echo -e " $(Array::wrap2 " " 76 4 "${helpArray[@]}")" - echo " Default value: default" + echo ' Default value: default' echo ' Possible values: default|default-force|noColor' echo -e " ${__HELP_OPTION_COLOR}--help${__HELP_NORMAL}, ${__HELP_OPTION_COLOR}-h${__HELP_NORMAL} {single}" local -a helpArray diff --git a/bin/gitIsBranch b/bin/gitIsBranch index 5ecf97d5..45a75834 100755 --- a/bin/gitIsBranch +++ b/bin/gitIsBranch @@ -1378,7 +1378,7 @@ gitIsBranchCommand() { # shellcheck disable=SC2054 helpArray=(choose\ color\ theme\ -\ default-force\ means\ colors\ will\ be\ produced\ even\ if\ command\ is\ piped) echo -e " $(Array::wrap2 " " 76 4 "${helpArray[@]}")" - echo " Default value: default" + echo ' Default value: default' echo ' Possible values: default|default-force|noColor' echo -e " ${__HELP_OPTION_COLOR}--help${__HELP_NORMAL}, ${__HELP_OPTION_COLOR}-h${__HELP_NORMAL} {single}" local -a helpArray diff --git a/bin/gitRenameBranch b/bin/gitRenameBranch index cb2015aa..2cfaff6e 100755 --- a/bin/gitRenameBranch +++ b/bin/gitRenameBranch @@ -1484,7 +1484,7 @@ gitRenameBranchCommand() { # shellcheck disable=SC2054 helpArray=(choose\ color\ theme\ -\ default-force\ means\ colors\ will\ be\ produced\ even\ if\ command\ is\ piped) echo -e " $(Array::wrap2 " " 76 4 "${helpArray[@]}")" - echo " Default value: default" + echo ' Default value: default' echo ' Possible values: default|default-force|noColor' echo -e " ${__HELP_OPTION_COLOR}--help${__HELP_NORMAL}, ${__HELP_OPTION_COLOR}-h${__HELP_NORMAL} {single}" local -a helpArray diff --git a/bin/installRequirements b/bin/installRequirements index 4430f140..96391fa8 100755 --- a/bin/installRequirements +++ b/bin/installRequirements @@ -1445,7 +1445,7 @@ installRequirementsCommand() { # shellcheck disable=SC2054 helpArray=(choose\ color\ theme\ -\ default-force\ means\ colors\ will\ be\ produced\ even\ if\ command\ is\ piped) echo -e " $(Array::wrap2 " " 76 4 "${helpArray[@]}")" - echo " Default value: default" + echo ' Default value: default' echo ' Possible values: default|default-force|noColor' echo -e " ${__HELP_OPTION_COLOR}--help${__HELP_NORMAL}, ${__HELP_OPTION_COLOR}-h${__HELP_NORMAL} {single}" local -a helpArray diff --git a/bin/mysql2puml b/bin/mysql2puml index cff4e150..d1badf33 100755 --- a/bin/mysql2puml +++ b/bin/mysql2puml @@ -1551,7 +1551,7 @@ mysql2pumlCommand() { # shellcheck disable=SC2054 helpArray=(header\ configuration\ of\ the\ plant\ uml\ file) echo -e " $(Array::wrap2 " " 76 4 "${helpArray[@]}")" - echo " Default value: default" + echo ' Default value: default' echo echo -e "${__HELP_TITLE_COLOR}GLOBAL OPTIONS:${__RESET_COLOR}" echo -e " ${__HELP_OPTION_COLOR}--bash-framework-config ${__HELP_NORMAL} {single}" @@ -1594,7 +1594,7 @@ mysql2pumlCommand() { # shellcheck disable=SC2054 helpArray=(choose\ color\ theme\ -\ default-force\ means\ colors\ will\ be\ produced\ even\ if\ command\ is\ piped) echo -e " $(Array::wrap2 " " 76 4 "${helpArray[@]}")" - echo " Default value: default" + echo ' Default value: default' echo ' Possible values: default|default-force|noColor' echo -e " ${__HELP_OPTION_COLOR}--help${__HELP_NORMAL}, ${__HELP_OPTION_COLOR}-h${__HELP_NORMAL} {single}" local -a helpArray diff --git a/bin/postmanCli b/bin/postmanCli index b360eab6..eaaddee2 100755 --- a/bin/postmanCli +++ b/bin/postmanCli @@ -1152,6 +1152,8 @@ Postman::api() { responseFile="$(Framework::createTempFile)" local status=0 + # shellcheck disable=SC2034 + local -a pipeStatus=() getCollectionDataFromFile "${collectionFile}" | curl \ --request POST https://api.getpostman.com/collections \ @@ -1160,7 +1162,7 @@ Postman::api() { --header 'Accept: application/json' \ --header "X-Api-Key: ${POSTMAN_API_KEY}" \ --data @- \ - --fail --silent --show-error || status=$? + --fail --silent --show-error || Bash::handlePipelineFailure status pipeStatus Postman::displayResponse "createCollectionFromFile" "${responseFile}" @@ -1174,6 +1176,8 @@ Postman::api() { responseFile="$(Framework::createTempFile)" local status=0 + # shellcheck disable=SC2034 + local -a pipeStatus=() getCollectionDataFromFile "${collectionFile}" | curl \ --request PUT "https://api.getpostman.com/collections/${collectionId}" \ @@ -1182,7 +1186,7 @@ Postman::api() { --header 'Accept: application/json' \ --header "X-Api-Key: ${POSTMAN_API_KEY}" \ --data @- \ - --fail --silent --show-error || status=$? + --fail --silent --show-error || Bash::handlePipelineFailure status pipeStatus Postman::displayResponse "updateCollectionFromFile" "${responseFile}" @@ -1217,6 +1221,36 @@ Postman::api() { esac } +# @description ignore exit code 141 from simple command pipes +# @example use with: +# local resultingStatus=0 +# local -a originalPipeStatus=() +# cmd1 | cmd2 || Bash::handlePipelineFailure resultingStatus originalPipeStatus || true +# [[ "${resultingStatus}" = "0" ]] +# @arg $1 resultingStatusCode:&int (passed by reference) (optional) resulting status code +# @arg $2 originalStatus:int[] (passed by reference) (optional) copy of original PIPESTATUS array +# @env PIPESTATUS assuming that this function is called like in the example provided +# @see https://unix.stackexchange.com/a/709880/582856 +Bash::handlePipelineFailure() { + local -a pipeStatusBackup=("${PIPESTATUS[@]}") + local -n handlePipelineFailure_resultingStatusCode=$1 + local -n handlePipelineFailure_originalStatus=$2 + # shellcheck disable=SC2034 + handlePipelineFailure_originalStatus=("${pipeStatusBackup[@]}") + handlePipelineFailure_resultingStatusCode=0 + local statusCode + for statusCode in "${pipeStatusBackup[@]}"; do + if ((statusCode == 141)); then + return 0 + elif ((statusCode > 0)); then + # shellcheck disable=SC2034 + handlePipelineFailure_resultingStatusCode="${statusCode}" + break + fi + done + return "${handlePipelineFailure_resultingStatusCode}" +} + # @description log message to file # @arg $1 message:String the message to display Log::logSkipped() { @@ -1978,7 +2012,7 @@ postmanCliCommand() { # shellcheck disable=SC2054 helpArray=(choose\ color\ theme\ -\ default-force\ means\ colors\ will\ be\ produced\ even\ if\ command\ is\ piped) echo -e " $(Array::wrap2 " " 76 4 "${helpArray[@]}")" - echo " Default value: default" + echo ' Default value: default' echo ' Possible values: default|default-force|noColor' echo -e " ${__HELP_OPTION_COLOR}--help${__HELP_NORMAL}, ${__HELP_OPTION_COLOR}-h${__HELP_NORMAL} {single}" local -a helpArray diff --git a/bin/upgradeGithubRelease b/bin/upgradeGithubRelease index 0712dfef..a2b002ac 100755 --- a/bin/upgradeGithubRelease +++ b/bin/upgradeGithubRelease @@ -1759,7 +1759,7 @@ upgradeGithubReleaseCommand() { # shellcheck disable=SC2054 helpArray=($'The argument that will be provided to the currently installed binary\n\n to check the version of the software. \n\n See options constraints below.') echo -e " $(Array::wrap2 " " 76 4 "${helpArray[@]}")" - echo " Default value: --version" + echo ' Default value: --version' echo -e " ${__HELP_OPTION_COLOR}--current-version${__HELP_NORMAL}, ${__HELP_OPTION_COLOR}-c ${__HELP_NORMAL} {single}" local -a helpArray # shellcheck disable=SC2054 @@ -1817,7 +1817,7 @@ upgradeGithubReleaseCommand() { # shellcheck disable=SC2054 helpArray=(choose\ color\ theme\ -\ default-force\ means\ colors\ will\ be\ produced\ even\ if\ command\ is\ piped) echo -e " $(Array::wrap2 " " 76 4 "${helpArray[@]}")" - echo " Default value: default" + echo ' Default value: default' echo ' Possible values: default|default-force|noColor' echo -e " ${__HELP_OPTION_COLOR}--help${__HELP_NORMAL}, ${__HELP_OPTION_COLOR}-h${__HELP_NORMAL} {single}" local -a helpArray diff --git a/bin/waitForIt b/bin/waitForIt index 3729ae6c..cc76a458 100755 --- a/bin/waitForIt +++ b/bin/waitForIt @@ -1477,7 +1477,7 @@ waitForItCommand() { # shellcheck disable=SC2054 helpArray=(Timeout\ in\ seconds\,\ zero\ for\ no\ timeout.) echo -e " $(Array::wrap2 " " 76 4 "${helpArray[@]}")" - echo " Default value: 15" + echo ' Default value: 15' echo -e " ${__HELP_OPTION_COLOR}--host${__HELP_NORMAL}, ${__HELP_OPTION_COLOR}-i ${__HELP_NORMAL} {single} (mandatory)" local -a helpArray # shellcheck disable=SC2054 @@ -1540,7 +1540,7 @@ waitForItCommand() { # shellcheck disable=SC2054 helpArray=(choose\ color\ theme\ -\ default-force\ means\ colors\ will\ be\ produced\ even\ if\ command\ is\ piped) echo -e " $(Array::wrap2 " " 76 4 "${helpArray[@]}")" - echo " Default value: default" + echo ' Default value: default' echo ' Possible values: default|default-force|noColor' echo -e " ${__HELP_OPTION_COLOR}--help${__HELP_NORMAL}, ${__HELP_OPTION_COLOR}-h${__HELP_NORMAL} {single}" local -a helpArray diff --git a/bin/waitForMysql b/bin/waitForMysql index ea64b857..2bcbab82 100755 --- a/bin/waitForMysql +++ b/bin/waitForMysql @@ -1453,7 +1453,7 @@ waitForMysqlCommand() { # shellcheck disable=SC2054 helpArray=(Timeout\ in\ seconds\,\ zero\ for\ no\ timeout.) echo -e " $(Array::wrap2 " " 76 4 "${helpArray[@]}")" - echo " Default value: 15" + echo ' Default value: 15' echo echo -e "${__HELP_TITLE_COLOR}GLOBAL OPTIONS:${__RESET_COLOR}" echo -e " ${__HELP_OPTION_COLOR}--bash-framework-config ${__HELP_NORMAL} {single}" @@ -1496,7 +1496,7 @@ waitForMysqlCommand() { # shellcheck disable=SC2054 helpArray=(choose\ color\ theme\ -\ default-force\ means\ colors\ will\ be\ produced\ even\ if\ command\ is\ piped) echo -e " $(Array::wrap2 " " 76 4 "${helpArray[@]}")" - echo " Default value: default" + echo ' Default value: default' echo ' Possible values: default|default-force|noColor' echo -e " ${__HELP_OPTION_COLOR}--help${__HELP_NORMAL}, ${__HELP_OPTION_COLOR}-h${__HELP_NORMAL} {single}" local -a helpArray diff --git a/conf/dbImportProfiles/default.sh b/conf/dbImportProfiles/default.sh index b88c5445..70c55437 100755 --- a/conf/dbImportProfiles/default.sh +++ b/conf/dbImportProfiles/default.sh @@ -8,4 +8,8 @@ cat | grep -v '.*stats' | grep -v '.*history.*' | # always finish by a cat to be sure the command does not return exit code != 0 - cat + cat || { + declare exitCode=$? + if ((exitCode == 141)); then exit 0; fi + exit "${exitCode}" +} diff --git a/install b/install index 9d5947fb..c5de47af 100755 --- a/install +++ b/install @@ -1351,7 +1351,7 @@ installCommand() { # shellcheck disable=SC2054 helpArray=(choose\ color\ theme\ -\ default-force\ means\ colors\ will\ be\ produced\ even\ if\ command\ is\ piped) echo -e " $(Array::wrap2 " " 76 4 "${helpArray[@]}")" - echo " Default value: default" + echo ' Default value: default' echo ' Possible values: default|default-force|noColor' echo -e " ${__HELP_OPTION_COLOR}--help${__HELP_NORMAL}, ${__HELP_OPTION_COLOR}-h${__HELP_NORMAL} {single}" local -a helpArray diff --git a/src/Postman/Model/validate.bats b/src/Postman/Model/validate.bats index 77255f80..4747f4a3 100755 --- a/src/Postman/Model/validate.bats +++ b/src/Postman/Model/validate.bats @@ -11,6 +11,10 @@ setup() { export BASH_FRAMEWORK_THEME="noColor" } +teardown() { + chmod -R u+w "${BATS_TEST_TMPDIR}/pullMode" 2>/dev/null || true +} + function Postman::Model::validate::missingMode { #@test run Postman::Model::validate \ "fileNotFound.json" "invalidMode" @@ -89,17 +93,20 @@ function Postman::Model::validate::pushModeCollectionsWithErrors { #@test } function Postman::Model::validate::pullModeCollectionsWithErrors { #@test + cp -R "${BATS_TEST_DIRNAME}/testsData/pullMode" "${BATS_TEST_TMPDIR}" + chmod -w "${BATS_TEST_TMPDIR}/pullMode/GithubAPI/notWritableFile.json" + chmod -w "${BATS_TEST_TMPDIR}/pullMode" Postman::Model::getRelativeConfigDirectory() { - echo "${BATS_TEST_DIRNAME}/testsData/pullMode" + echo "${BATS_TEST_TMPDIR}/pullMode" } - local file="${BATS_TEST_DIRNAME}/testsData/pullMode/getCollectionRefs-collectionsWithErrors.json" + local file="${BATS_TEST_TMPDIR}/pullMode/getCollectionRefs-collectionsWithErrors.json" run Postman::Model::validate \ "${file}" "pull" assert_failure 1 assert_line --index 0 "ERROR - File '${file}' - collection 0 - missing file property" - assert_line --index 1 "ERROR - File '${file}' - collection fileNotWritable - collection file ${BATS_TEST_DIRNAME}/testsData/pullMode/GithubAPI/notWritableFile.json is not writable" - assert_line --index 2 "ERROR - File '${file}' - collection dirNotWritable - config directory ${BATS_TEST_DIRNAME}/testsData/pullMode is not writable" + assert_line --index 1 "ERROR - File '${file}' - collection fileNotWritable - collection file ${BATS_TEST_TMPDIR}/pullMode/GithubAPI/notWritableFile.json is not writable" + assert_line --index 2 "ERROR - File '${file}' - collection dirNotWritable - config directory ${BATS_TEST_TMPDIR}/pullMode is not writable" assert_lines_count 3 } diff --git a/src/Postman/api.bats b/src/Postman/api.bats index add09791..43e3b429 100755 --- a/src/Postman/api.bats +++ b/src/Postman/api.bats @@ -4,6 +4,8 @@ source "$(cd "${BATS_TEST_DIRNAME}/.." && pwd)/batsHeaders.sh" # shellcheck source=src/Postman/api.sh source "${rootDir}/src/Postman/api.sh" +# shellcheck source=vendor/bash-tools-framework/src/Bash/handlePipelineFailure.sh +source "${FRAMEWORK_ROOT_DIR}/src/Bash/handlePipelineFailure.sh" setup() { export BASH_FRAMEWORK_THEME="noColor" diff --git a/src/Postman/api.sh b/src/Postman/api.sh index 817f46bc..f3980587 100755 --- a/src/Postman/api.sh +++ b/src/Postman/api.sh @@ -30,6 +30,8 @@ Postman::api() { responseFile="$(Framework::createTempFile)" local status=0 + # shellcheck disable=SC2034 + local -a pipeStatus=() getCollectionDataFromFile "${collectionFile}" | curl \ --request POST https://api.getpostman.com/collections \ @@ -38,7 +40,7 @@ Postman::api() { --header 'Accept: application/json' \ --header "X-Api-Key: ${POSTMAN_API_KEY}" \ --data @- \ - --fail --silent --show-error || status=$? + --fail --silent --show-error || Bash::handlePipelineFailure status pipeStatus Postman::displayResponse "createCollectionFromFile" "${responseFile}" @@ -52,6 +54,8 @@ Postman::api() { responseFile="$(Framework::createTempFile)" local status=0 + # shellcheck disable=SC2034 + local -a pipeStatus=() getCollectionDataFromFile "${collectionFile}" | curl \ --request PUT "https://api.getpostman.com/collections/${collectionId}" \ @@ -60,7 +64,7 @@ Postman::api() { --header 'Accept: application/json' \ --header "X-Api-Key: ${POSTMAN_API_KEY}" \ --data @- \ - --fail --silent --show-error || status=$? + --fail --silent --show-error || Bash::handlePipelineFailure status pipeStatus Postman::displayResponse "updateCollectionFromFile" "${responseFile}" diff --git a/src/_binaries/DbImport/dbImport.bats b/src/_binaries/DbImport/dbImport.bats index c8b72c00..aad66a92 100755 --- a/src/_binaries/DbImport/dbImport.bats +++ b/src/_binaries/DbImport/dbImport.bats @@ -173,13 +173,13 @@ function Database::dbImport::remote_db_fully_functional_from_mysql { #@test export BASH_FRAMEWORK_ENV_FILEPATH="${BATS_TEST_DIRNAME}/testsData/.env" run "${binDir}/dbImport" --verbose -f default.local fromDb toDb 2>&1 - unstub_all assert_output --partial "Import database duration : " assert_output --partial "begin insert emptyTable" assert_output --partial "begin insert dataTable" assert_output --partial "begin insert otherTable" [[ -f "${HOME}/.bash-tools/dbImportDumps/fromDb_default.sql.gz" ]] [[ -f "${HOME}/.bash-tools/dbImportDumps/fromDb_default_structure.sql.gz" ]] + unstub_all [[ "$(zcat "${HOME}/.bash-tools/dbImportDumps/fromDb_default.sql.gz" | grep '####data####')" = "####data####" ]] [[ "$(zcat "${HOME}/.bash-tools/dbImportDumps/fromDb_default_structure.sql.gz" | grep '####structure####')" = "####structure####" ]] } @@ -283,10 +283,12 @@ function Database::dbImport::import_local_dump_not_aws_with_tables_filter { #@te $'* --connect-timeout=5 --batch --raw --default-character-set=utf8 toDb : i=0 ; while read line; do ((i=i+1)); echo "line $i"; done < /dev/stdin' run "${binDir}/dbImport" --verbose -f default.local fromDb toDb --tables dataTable,otherTable 2>&1 - assert_output --partial "Import database duration : " + assert_output --partial "db created" + assert_output --partial "import structure dump" assert_output --partial "ignore table emptyTable" - assert_output --partial "begin insert dataTable" + assert_output --partial "Import database duration : " assert_output --partial "begin insert otherTable" + assert_output --partial "begin insert dataTable" [[ -f "${HOME}/.bash-tools/dbImportDumps/fromDb_default.sql.gz" ]] [[ -f "${HOME}/.bash-tools/dbImportDumps/fromDb_default_structure.sql.gz" ]] # check files have been touched diff --git a/src/_binaries/DbImport/dbImport.sh b/src/_binaries/DbImport/dbImport.sh index 13a0b5ab..a8dbe33a 100755 --- a/src/_binaries/DbImport/dbImport.sh +++ b/src/_binaries/DbImport/dbImport.sh @@ -190,9 +190,13 @@ run() { Log::displayInfo "avoid to create db structure" else Log::displayInfo "create db structure from ${remoteDbStructureDumpTempFile}" + # shellcheck disable=SC2034 + local status=0 + # shellcheck disable=SC2034 + local -a pipeStatus=() time ( pv "${remoteDbStructureDumpTempFile}" | zcat | - Database::query dbTargetDatabase "" "${targetDbName}" + Database::query dbTargetDatabase "" "${targetDbName}" || Bash::handlePipelineFailure status pipeStatus ) fi fi diff --git a/src/_binaries/DbImport/dbImportStream.sh b/src/_binaries/DbImport/dbImportStream.sh index e716c4eb..896892ea 100755 --- a/src/_binaries/DbImport/dbImportStream.sh +++ b/src/_binaries/DbImport/dbImportStream.sh @@ -37,7 +37,7 @@ run() { Version::checkMinimal "gawk" "--version" "5.0.1" # create db instances - declare -Agx dbTargetInstance + local -Agx dbTargetInstance Database::newInstance dbTargetInstance "${optionTargetDsn}" Database::setQueryOptions dbTargetInstance "${dbTargetInstance[QUERY_OPTIONS]} --connect-timeout=5" @@ -46,7 +46,11 @@ run() { initializeDefaultTargetMysqlOptions dbTargetInstance "${argTargetDbName}" # TODO character set should be retrieved from dump files if possible - declare remoteCharacterSet="${optionCharacterSet:-${defaultRemoteCharacterSet}}" + local remoteCharacterSet="${optionCharacterSet:-${defaultRemoteCharacterSet}}" + + local status=0 + # shellcheck disable=SC2034 + local -a pipeStatus=() # shellcheck disable=2086 ( @@ -55,19 +59,19 @@ run() { elif [[ "${argDumpFile}" =~ \.gz$ ]]; then zcat "${argDumpFile}" fi - # zcat will continue to write to stdout whereas awk has finished if table has been found - # we detect this case because zcat will return code 141 because pipe closed - status=$? - if [[ "${status}" -eq "141" ]]; then true; else exit "${status}"; fi ) | awk \ -v PROFILE_COMMAND="${profileCommandFile}" \ -v CHARACTER_SET="${remoteCharacterSet}" \ --source "${awkScript}" \ - - | mysql \ - "--defaults-extra-file=${dbTargetInstance['AUTH_FILE']}" \ - ${dbTargetInstance['DB_IMPORT_OPTIONS']} \ - "${argTargetDbName}" || exit $? + - | + mysql \ + "--defaults-extra-file=${dbTargetInstance['AUTH_FILE']}" \ + ${dbTargetInstance['DB_IMPORT_OPTIONS']} \ + "${argTargetDbName}" || + # zcat will continue to write to stdout whereas awk has finished if table has been found + # we detect this case because zcat will return code 141 because pipe closed + Bash::handlePipelineFailure status pipeStatus } if [[ "${BASH_FRAMEWORK_QUIET_MODE:-0}" = "1" ]]; then diff --git a/src/_binaries/DbImport/testsData/dbImportProfiles/default.sh b/src/_binaries/DbImport/testsData/dbImportProfiles/default.sh index 68138349..8cf9b92b 100755 --- a/src/_binaries/DbImport/testsData/dbImportProfiles/default.sh +++ b/src/_binaries/DbImport/testsData/dbImportProfiles/default.sh @@ -8,4 +8,8 @@ cat | grep -v '.*stats' | grep -v '.*history.*' | # always finish by a cat to be sure the command does not return exit code != 0 - cat + cat || { + # avoid failure on premature process close (check Bash::handlePipelineFailure) + declare exitCode=$?; if (( exitCode == 141 )); then exit 0; fi + exit "${exitCode}" + }