From ae8ae589f418216faa31b0a409f4e36ef986ac28 Mon Sep 17 00:00:00 2001 From: hkobew Date: Mon, 2 Dec 2024 14:12:53 -0500 Subject: [PATCH 01/10] put input in env before passing --- .github/workflows/copyPasteDetection.yml | 150 ++++++++++++----------- 1 file changed, 76 insertions(+), 74 deletions(-) diff --git a/.github/workflows/copyPasteDetection.yml b/.github/workflows/copyPasteDetection.yml index ad78e409fca..c78ccb5d630 100644 --- a/.github/workflows/copyPasteDetection.yml +++ b/.github/workflows/copyPasteDetection.yml @@ -1,86 +1,88 @@ -# # github actions: https://docs.github.com/en/actions/use-cases-and-examples/building-and-testing/building-and-testing-nodejs -# # setup-node: https://github.com/actions/setup-node +# github actions: https://docs.github.com/en/actions/use-cases-and-examples/building-and-testing/building-and-testing-nodejs +# setup-node: https://github.com/actions/setup-node -# name: Copy-Paste Detection +name: Copy-Paste Detection -# on: -# pull_request: -# branches: [master, feature/*, staging] +on: + pull_request: + branches: [master, feature/*, staging] -# jobs: -# jscpd: -# runs-on: ubuntu-latest -# strategy: -# matrix: -# node-version: [18.x] -# env: -# NODE_OPTIONS: '--max-old-space-size=8192' +jobs: + jscpd: + runs-on: ubuntu-latest + strategy: + matrix: + node-version: [18.x] + env: + NODE_OPTIONS: '--max-old-space-size=8192' -# steps: -# - uses: actions/checkout@v4 -# with: -# fetch-depth: 0 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 -# - name: Use Node.js ${{ matrix.node-version }} -# uses: actions/setup-node@v4 -# with: -# node-version: ${{ matrix.node-version }} + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} -# - name: Fetch fork upstream -# run: | -# git remote add forkUpstream https://github.com/${{ github.event.pull_request.head.repo.full_name }} # URL of the fork -# git fetch forkUpstream # Fetch fork + - name: Fetch fork upstream + env: + REPO_NAME: ${{ github.event.pull_request.head.repo.full_name }} + run: | + git remote add forkUpstream https://github.com/$REPO_NAME # URL of the fork + git fetch forkUpstream # Fetch fork -# - name: Determine base and target branches for comparison. -# run: | -# echo "CURRENT_BRANCH=${{ github.head_ref }}" >> $GITHUB_ENV -# echo "TARGET_BRANCH=${{ github.event.pull_request.base.ref }}" >> $GITHUB_ENV -# - run: git diff --name-only origin/$TARGET_BRANCH forkUpstream/$CURRENT_BRANCH > diff_output.txt -# - run: | -# npm install -g jscpd + - name: Compute git diff + env: + CURRENT_BRANCH: ${{ github.head_ref }} + TARGET_BRANCH: ${{ github.event.pull_request.base.ref }} + run: git diff --name-only origin/$TARGET_BRANCH forkUpstream/$CURRENT_BRANCH > diff_output.txt -# - run: jscpd --config "$GITHUB_WORKSPACE/.github/workflows/jscpd.json" + - run: npm install -g jscpd -# - if: always() -# uses: actions/upload-artifact@v4 -# with: -# name: unfiltered-jscpd-report -# path: ./jscpd-report.json + - run: jscpd --config "$GITHUB_WORKSPACE/.github/workflows/jscpd.json" -# - name: Filter jscpd report for changed files -# run: | -# if [ ! -f ./jscpd-report.json ]; then -# echo "jscpd-report.json not found" -# exit 1 -# fi -# echo "Filtering jscpd report for changed files..." -# CHANGED_FILES=$(jq -R -s -c 'split("\n")[:-1]' diff_output.txt) -# echo "Changed files: $CHANGED_FILES" -# jq --argjson changed_files "$CHANGED_FILES" ' -# .duplicates | map(select( -# (.firstFile?.name as $fname | $changed_files | any(. == $fname)) or -# (.secondFile?.name as $sname | $changed_files | any(. == $sname)) -# )) -# ' ./jscpd-report.json > filtered-jscpd-report.json -# cat filtered-jscpd-report.json + - if: always() + uses: actions/upload-artifact@v4 + with: + name: unfiltered-jscpd-report + path: ./jscpd-report.json -# - name: Check for duplicates -# run: | -# if [ $(wc -l < ./filtered-jscpd-report.json) -gt 1 ]; then -# echo "filtered_report_exists=true" >> $GITHUB_ENV -# else -# echo "filtered_report_exists=false" >> $GITHUB_ENV -# fi -# - name: upload filtered report (if applicable) -# if: env.filtered_report_exists == 'true' -# uses: actions/upload-artifact@v4 -# with: -# name: filtered-jscpd-report -# path: ./filtered-jscpd-report.json + - name: Filter jscpd report for changed files + run: | + if [ ! -f ./jscpd-report.json ]; then + echo "jscpd-report.json not found" + exit 1 + fi + echo "Filtering jscpd report for changed files..." + CHANGED_FILES=$(jq -R -s -c 'split("\n")[:-1]' diff_output.txt) + echo "Changed files: $CHANGED_FILES" + jq --argjson changed_files "$CHANGED_FILES" ' + .duplicates | map(select( + (.firstFile?.name as $fname | $changed_files | any(. == $fname)) or + (.secondFile?.name as $sname | $changed_files | any(. == $sname)) + )) + ' ./jscpd-report.json > filtered-jscpd-report.json + cat filtered-jscpd-report.json -# - name: Fail and log found duplicates. -# if: env.filtered_report_exists == 'true' -# run: | -# cat ./filtered-jscpd-report.json -# echo "Duplications found, failing the check." -# exit 1 + - name: Check for duplicates + run: | + if [ $(wc -l < ./filtered-jscpd-report.json) -gt 1 ]; then + echo "filtered_report_exists=true" >> $GITHUB_ENV + else + echo "filtered_report_exists=false" >> $GITHUB_ENV + fi + - name: upload filtered report (if applicable) + if: env.filtered_report_exists == 'true' + uses: actions/upload-artifact@v4 + with: + name: filtered-jscpd-report + path: ./filtered-jscpd-report.json + + - name: Fail and log found duplicates. + if: env.filtered_report_exists == 'true' + run: | + cat ./filtered-jscpd-report.json + echo "Duplications found, failing the check." + exit 1 From 812d67e3817e072e4fa37e4cfa5249ac1f284947 Mon Sep 17 00:00:00 2001 From: hkobew Date: Tue, 3 Dec 2024 10:37:52 -0500 Subject: [PATCH 02/10] implement extra linting step --- .github/workflows/lintbranch.js | 67 +++++++++++++++++++++++++++++++++ .github/workflows/node.js.yml | 3 ++ 2 files changed, 70 insertions(+) create mode 100644 .github/workflows/lintbranch.js diff --git a/.github/workflows/lintbranch.js b/.github/workflows/lintbranch.js new file mode 100644 index 00000000000..deeaadaa949 --- /dev/null +++ b/.github/workflows/lintbranch.js @@ -0,0 +1,67 @@ +// Check that branch name conforms to GitHub naming convention: +// https://docs.github.com/en/get-started/using-git/dealing-with-special-characters-in-branch-and-tag-names#naming-branches-and-tags + +// To run self-tests, +// node lintbranch.js test +// TODO: deduplicate code from lintbranch.js and lintcommit.js. + +function isValid(branchName) { + const branchNameRegex = /^[a-zA-Z][a-zA-Z0-9._/-]*$/ + + return branchNameRegex.test(branchName) +} + +function run(branchName) { + if (isValid(branchName)) { + console.log(`Branch name "${branchName}" is valid.`) + process.exit(0) + } else { + const helpUrl = + 'https://docs.github.com/en/get-started/using-git/dealing-with-special-characters-in-branch-and-tag-names#naming-branches-and-tags' + console.log(`Branch name "${branchName}" is invalid see ${helpUrl} for more information.`) + process.exit(1) + } +} + +function _test() { + const tests = { + 'feature/branch-name': true, + feature_123: true, + 'my-branch': true, + '123invalid-start': false, + '!invalid@start': false, + '': false, + 'another/valid-name134': true, + 'Hweinstock:feature/123";id;{echo,Y2F0IC9ldGMvcGFzc3dk}|{base64,-d}|{bash,-i};#': false, + } + + let passed = 0 + let failed = 0 + + for (const [branchName, expected] of Object.entries(tests)) { + const result = isValid(branchName) + if (result === expected) { + console.log(`✅ Test passed for "${branchName}"`) + passed++ + } else { + console.log(`❌ Test failed for "${branchName}" (expected "${expected}", got "${result}")`) + failed++ + } + } + + console.log(`\n${passed} tests passed, ${failed} tests failed`) +} + +function main() { + const mode = process.argv[2] + + if (mode === 'test') { + _test() + } else if (mode === 'run') { + run(process.argv[3]) + } else { + throw new Error(`Unknown mode: ${mode}`) + } +} + +main() diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index b9e1529958f..381279eec02 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -32,6 +32,9 @@ jobs: - uses: actions/setup-node@v4 with: node-version: '20' + - name: Check Branch title + run: | + node "$GITHUB_WORKSPACE/.github/workflows/lintbranch.js" - name: Check PR title run: | node "$GITHUB_WORKSPACE/.github/workflows/lintcommit.js" From 081b968a11a7f6feac10ae243ca47df44ffb655c Mon Sep 17 00:00:00 2001 From: hkobew Date: Tue, 3 Dec 2024 10:39:48 -0500 Subject: [PATCH 03/10] use env var for passing branch name --- .github/workflows/node.js.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index 381279eec02..d336a77a5e9 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -33,8 +33,10 @@ jobs: with: node-version: '20' - name: Check Branch title + env: + BRANCH_NAME: ${{ github.head_ref }} run: | - node "$GITHUB_WORKSPACE/.github/workflows/lintbranch.js" + node "$GITHUB_WORKSPACE/.github/workflows/lintbranch.js run $BRANCH_NAME" - name: Check PR title run: | node "$GITHUB_WORKSPACE/.github/workflows/lintcommit.js" From aca96048d0cfdd0e67ab55331c0fcbe1f474dfb6 Mon Sep 17 00:00:00 2001 From: hkobew Date: Tue, 3 Dec 2024 10:56:54 -0500 Subject: [PATCH 04/10] refactor github actions --- .github/workflows/copyPasteDetection.yml | 5 +---- .github/workflows/node.js.yml | 3 +++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/copyPasteDetection.yml b/.github/workflows/copyPasteDetection.yml index c78ccb5d630..f0ec3eaf8cb 100644 --- a/.github/workflows/copyPasteDetection.yml +++ b/.github/workflows/copyPasteDetection.yml @@ -3,10 +3,7 @@ name: Copy-Paste Detection -on: - pull_request: - branches: [master, feature/*, staging] - +on: [workflow_call] jobs: jscpd: runs-on: ubuntu-latest diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index d336a77a5e9..1c63a2146dd 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -59,6 +59,9 @@ jobs: - run: npm ci - run: npm run testCompile - run: npm run lint + jscpd: + needs: lint-commits + uses: ./github/workflows/copyPasteDetection.yml macos: needs: lint-commits From db353e25ebdd50e2932c0166a65740ced860fe1d Mon Sep 17 00:00:00 2001 From: hkobew Date: Tue, 3 Dec 2024 11:11:45 -0500 Subject: [PATCH 05/10] revert refactor --- .github/workflows/copyPasteDetection.yml | 6 ++++-- .github/workflows/node.js.yml | 3 --- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/copyPasteDetection.yml b/.github/workflows/copyPasteDetection.yml index f0ec3eaf8cb..ab10ccc7b5f 100644 --- a/.github/workflows/copyPasteDetection.yml +++ b/.github/workflows/copyPasteDetection.yml @@ -1,9 +1,11 @@ # github actions: https://docs.github.com/en/actions/use-cases-and-examples/building-and-testing/building-and-testing-nodejs # setup-node: https://github.com/actions/setup-node -name: Copy-Paste Detection +name: jscpd -on: [workflow_call] +on: + pull_request: + branches: [master, feature/*, staging] jobs: jscpd: runs-on: ubuntu-latest diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index 1c63a2146dd..d336a77a5e9 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -59,9 +59,6 @@ jobs: - run: npm ci - run: npm run testCompile - run: npm run lint - jscpd: - needs: lint-commits - uses: ./github/workflows/copyPasteDetection.yml macos: needs: lint-commits From 62fd73e387774270a7afbc2b17344f329e2e9201 Mon Sep 17 00:00:00 2001 From: hkobew Date: Tue, 3 Dec 2024 11:14:21 -0500 Subject: [PATCH 06/10] fix missing quote --- .github/workflows/node.js.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index d336a77a5e9..a91828595b9 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -36,7 +36,7 @@ jobs: env: BRANCH_NAME: ${{ github.head_ref }} run: | - node "$GITHUB_WORKSPACE/.github/workflows/lintbranch.js run $BRANCH_NAME" + node "$GITHUB_WORKSPACE/.github/workflows/lintbranch.js" run $BRANCH_NAME" - name: Check PR title run: | node "$GITHUB_WORKSPACE/.github/workflows/lintcommit.js" From ad39a20fe3fe9862a1d901b4b6d1371b5f4fddfb Mon Sep 17 00:00:00 2001 From: hkobew Date: Tue, 3 Dec 2024 11:15:23 -0500 Subject: [PATCH 07/10] fix again --- .github/workflows/node.js.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index a91828595b9..9b6d14e9b39 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -36,7 +36,7 @@ jobs: env: BRANCH_NAME: ${{ github.head_ref }} run: | - node "$GITHUB_WORKSPACE/.github/workflows/lintbranch.js" run $BRANCH_NAME" + node "$GITHUB_WORKSPACE/.github/workflows/lintbranch.js" run "$BRANCH_NAME" - name: Check PR title run: | node "$GITHUB_WORKSPACE/.github/workflows/lintcommit.js" From a07362c33217b3c5adc95caf18b275f01621769b Mon Sep 17 00:00:00 2001 From: hkobew Date: Tue, 3 Dec 2024 11:23:24 -0500 Subject: [PATCH 08/10] move into same file to make dependency more transparent --- .github/workflows/copyPasteDetection.yml | 87 ------------------------ .github/workflows/node.js.yml | 81 ++++++++++++++++++++++ 2 files changed, 81 insertions(+), 87 deletions(-) delete mode 100644 .github/workflows/copyPasteDetection.yml diff --git a/.github/workflows/copyPasteDetection.yml b/.github/workflows/copyPasteDetection.yml deleted file mode 100644 index ab10ccc7b5f..00000000000 --- a/.github/workflows/copyPasteDetection.yml +++ /dev/null @@ -1,87 +0,0 @@ -# github actions: https://docs.github.com/en/actions/use-cases-and-examples/building-and-testing/building-and-testing-nodejs -# setup-node: https://github.com/actions/setup-node - -name: jscpd - -on: - pull_request: - branches: [master, feature/*, staging] -jobs: - jscpd: - runs-on: ubuntu-latest - strategy: - matrix: - node-version: [18.x] - env: - NODE_OPTIONS: '--max-old-space-size=8192' - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node-version }} - - - name: Fetch fork upstream - env: - REPO_NAME: ${{ github.event.pull_request.head.repo.full_name }} - run: | - git remote add forkUpstream https://github.com/$REPO_NAME # URL of the fork - git fetch forkUpstream # Fetch fork - - - name: Compute git diff - env: - CURRENT_BRANCH: ${{ github.head_ref }} - TARGET_BRANCH: ${{ github.event.pull_request.base.ref }} - run: git diff --name-only origin/$TARGET_BRANCH forkUpstream/$CURRENT_BRANCH > diff_output.txt - - - run: npm install -g jscpd - - - run: jscpd --config "$GITHUB_WORKSPACE/.github/workflows/jscpd.json" - - - if: always() - uses: actions/upload-artifact@v4 - with: - name: unfiltered-jscpd-report - path: ./jscpd-report.json - - - name: Filter jscpd report for changed files - run: | - if [ ! -f ./jscpd-report.json ]; then - echo "jscpd-report.json not found" - exit 1 - fi - echo "Filtering jscpd report for changed files..." - CHANGED_FILES=$(jq -R -s -c 'split("\n")[:-1]' diff_output.txt) - echo "Changed files: $CHANGED_FILES" - jq --argjson changed_files "$CHANGED_FILES" ' - .duplicates | map(select( - (.firstFile?.name as $fname | $changed_files | any(. == $fname)) or - (.secondFile?.name as $sname | $changed_files | any(. == $sname)) - )) - ' ./jscpd-report.json > filtered-jscpd-report.json - cat filtered-jscpd-report.json - - - name: Check for duplicates - run: | - if [ $(wc -l < ./filtered-jscpd-report.json) -gt 1 ]; then - echo "filtered_report_exists=true" >> $GITHUB_ENV - else - echo "filtered_report_exists=false" >> $GITHUB_ENV - fi - - name: upload filtered report (if applicable) - if: env.filtered_report_exists == 'true' - uses: actions/upload-artifact@v4 - with: - name: filtered-jscpd-report - path: ./filtered-jscpd-report.json - - - name: Fail and log found duplicates. - if: env.filtered_report_exists == 'true' - run: | - cat ./filtered-jscpd-report.json - echo "Duplications found, failing the check." - exit 1 diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index 9b6d14e9b39..a84f0f9d3db 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -60,6 +60,87 @@ jobs: - run: npm run testCompile - run: npm run lint + jscpd: + needs: [lint-commits] + if: ${{ github.event_name == 'pull_request'}} + runs-on: ubuntu-latest + strategy: + matrix: + node-version: [18.x] + env: + NODE_OPTIONS: '--max-old-space-size=8192' + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + + - name: Fetch fork upstream + env: + REPO_NAME: ${{ github.event.pull_request.head.repo.full_name }} + run: | + git remote add forkUpstream https://github.com/$REPO_NAME # URL of the fork + git fetch forkUpstream # Fetch fork + + - name: Compute git diff + env: + CURRENT_BRANCH: ${{ github.head_ref }} + TARGET_BRANCH: ${{ github.event.pull_request.base.ref }} + run: git diff --name-only origin/$TARGET_BRANCH forkUpstream/$CURRENT_BRANCH > diff_output.txt + + - run: npm install -g jscpd + + - run: jscpd --config "$GITHUB_WORKSPACE/.github/workflows/jscpd.json" + + - if: always() + uses: actions/upload-artifact@v4 + with: + name: unfiltered-jscpd-report + path: ./jscpd-report.json + + - name: Filter jscpd report for changed files + run: | + if [ ! -f ./jscpd-report.json ]; then + echo "jscpd-report.json not found" + exit 1 + fi + echo "Filtering jscpd report for changed files..." + CHANGED_FILES=$(jq -R -s -c 'split("\n")[:-1]' diff_output.txt) + echo "Changed files: $CHANGED_FILES" + jq --argjson changed_files "$CHANGED_FILES" ' + .duplicates | map(select( + (.firstFile?.name as $fname | $changed_files | any(. == $fname)) or + (.secondFile?.name as $sname | $changed_files | any(. == $sname)) + )) + ' ./jscpd-report.json > filtered-jscpd-report.json + cat filtered-jscpd-report.json + + - name: Check for duplicates + run: | + if [ $(wc -l < ./filtered-jscpd-report.json) -gt 1 ]; then + echo "filtered_report_exists=true" >> $GITHUB_ENV + else + echo "filtered_report_exists=false" >> $GITHUB_ENV + fi + - name: upload filtered report (if applicable) + if: env.filtered_report_exists == 'true' + uses: actions/upload-artifact@v4 + with: + name: filtered-jscpd-report + path: ./filtered-jscpd-report.json + + - name: Fail and log found duplicates. + if: env.filtered_report_exists == 'true' + run: | + cat ./filtered-jscpd-report.json + echo "Duplications found, failing the check." + exit 1 + macos: needs: lint-commits name: test macOS From 94949575996e4f9bf30c4a285a296e468fe449cf Mon Sep 17 00:00:00 2001 From: hkobew Date: Tue, 3 Dec 2024 11:28:53 -0500 Subject: [PATCH 09/10] remove unnecessary brackets --- .github/workflows/node.js.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index a84f0f9d3db..8ba36eeb3a2 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -61,7 +61,7 @@ jobs: - run: npm run lint jscpd: - needs: [lint-commits] + needs: lint-commits if: ${{ github.event_name == 'pull_request'}} runs-on: ubuntu-latest strategy: From 4e2613f5c8b5d7302cb62b6f6623277616f0e1ea Mon Sep 17 00:00:00 2001 From: hkobew Date: Tue, 3 Dec 2024 14:10:49 -0500 Subject: [PATCH 10/10] fix test case, avoid username --- .github/workflows/lintbranch.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lintbranch.js b/.github/workflows/lintbranch.js index deeaadaa949..05fc677dac5 100644 --- a/.github/workflows/lintbranch.js +++ b/.github/workflows/lintbranch.js @@ -32,7 +32,7 @@ function _test() { '!invalid@start': false, '': false, 'another/valid-name134': true, - 'Hweinstock:feature/123";id;{echo,Y2F0IC9ldGMvcGFzc3dk}|{base64,-d}|{bash,-i};#': false, + 'feature/123";id;{echo,Y2F0IC9ldGMvcGFzc3dk}|{base64,-d}|{bash,-i};#': false, } let passed = 0