From 365c4ea5db025eb405b5f1a6ed3ebfc76e3c4192 Mon Sep 17 00:00:00 2001 From: barshaul Date: Thu, 15 Feb 2024 12:57:10 +0000 Subject: [PATCH 1/2] Fixed updating the library version in the CD workflow --- .github/workflows/npm-cd.yml | 18 ++++++++++--- .github/workflows/pypi-cd.yml | 20 +++++++++++--- .../workflows/update-glide-version/action.yml | 26 +++++++++++++++++++ benchmarks/rust/.cargo/config.toml | 2 +- csharp/lib/.cargo/config.toml | 2 +- glide-core/.cargo/config.toml | 2 +- java/.cargo/config.toml | 2 +- node/rust-client/.cargo/config.toml | 2 +- node/tests/SharedTests.ts | 2 +- python/.cargo/config.toml | 2 +- python/python/tests/test_async_client.py | 4 +-- utils/get_licenses_from_ort.py | 8 +++--- 12 files changed, 70 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/update-glide-version/action.yml diff --git a/.github/workflows/npm-cd.yml b/.github/workflows/npm-cd.yml index 22f55a0e1b..69d78aa85e 100644 --- a/.github/workflows/npm-cd.yml +++ b/.github/workflows/npm-cd.yml @@ -3,6 +3,10 @@ name: Continuous Deployment on: + pull_request: + paths: + - .github/workflows/npm-cd.yml + - .github/workflows/build-node-wrapper/action.yml push: tags: - "v*.*" @@ -56,7 +60,8 @@ jobs: - name: Set the release version shell: bash run: | - echo "RELEASE_VERSION=${GITHUB_REF:11}" >> $GITHUB_ENV + export version=`if ${{ github.event_name == 'pull_request' }}; then echo '255.255.255'; else echo ${GITHUB_REF:11}; fi` + echo "RELEASE_VERSION=${version}" >> $GITHUB_ENV - name: Setup node if: matrix.build.NPM_PUBLISH == true @@ -68,7 +73,13 @@ jobs: scope: "${{ vars.NPM_SCOPE }}" always-auth: true token: ${{ secrets.NPM_AUTH_TOKEN }} - + + - name: Update package version in config.toml + uses: ./.github/workflows/update-glide-version + with: + folder_path: "${{ github.workspace }}/node/rust-client/.cargo" + named_os: ${{ matrix.build.NAMED_OS }} + - name: Build Node wrapper if: matrix.build.NPM_PUBLISH == true uses: ./.github/workflows/build-node-wrapper @@ -82,7 +93,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Publish to NPM - if: matrix.build.NPM_PUBLISH == true + if: matrix.build.NPM_PUBLISH == true && github.event_name != 'pull_request' shell: bash working-directory: ./node run: | @@ -190,6 +201,7 @@ jobs: done - name: Publish the base package + if: github.event_name != 'pull_request' shell: bash working-directory: ./node/npm/glide run: | diff --git a/.github/workflows/pypi-cd.yml b/.github/workflows/pypi-cd.yml index 56e407b89d..cc273a3ede 100644 --- a/.github/workflows/pypi-cd.yml +++ b/.github/workflows/pypi-cd.yml @@ -3,6 +3,10 @@ name: Continuous Deployment on: + pull_request: + paths: + - .github/workflows/pypi-cd.yml + - .github/workflows/build-python-wrapper/action.yml push: tags: - "v*.*" @@ -56,7 +60,8 @@ jobs: - name: Set the release version shell: bash run: | - echo "RELEASE_VERSION=${GITHUB_REF:11}" >> $GITHUB_ENV + export version=`if ${{ github.event_name == 'pull_request' }}; then echo '255.255.255'; else echo ${GITHUB_REF:11}; fi` + echo "RELEASE_VERSION=${version}" >> $GITHUB_ENV - name: Set the package version for Python if: matrix.build.PYPI_PUBLISH == true @@ -85,6 +90,12 @@ jobs: sudo apt upgrade -y sudo apt install python3 python3-venv python3-pip -y + - name: Update package version in config.toml + uses: ./.github/workflows/update-glide-version + with: + folder_path: "${{ github.workspace }}/python/.cargo" + named_os: ${{ matrix.build.NAMED_OS }} + - name: Build Python wrapper if: matrix.build.PYPI_PUBLISH == true uses: ./.github/workflows/build-python-wrapper @@ -104,7 +115,7 @@ jobs: cat .gitignore - name: Build Python wheels (linux) - if: matrix.build.PYPI_PUBLISH == true && startsWith(matrix.build.NAMED_OS, 'linux') + if: matrix.build.PYPI_PUBLISH == true && startsWith(matrix.build.NAMED_OS, 'linux') && github.event_name != 'pull_request' uses: PyO3/maturin-action@v1 with: working-directory: ./python @@ -125,7 +136,7 @@ jobs: fi - name: Build Python wheels (macos) - if: matrix.build.PYPI_PUBLISH == true && startsWith(matrix.build.NAMED_OS, 'darwin') + if: matrix.build.PYPI_PUBLISH == true && startsWith(matrix.build.NAMED_OS, 'darwin') && github.event_name != 'pull_request' uses: PyO3/maturin-action@v1 with: working-directory: ./python @@ -133,7 +144,7 @@ jobs: args: --release --strip --out wheels -i python3.8 python3.9 python3.10 python3.11 python3.12 - name: Upload Python wheels - if: matrix.build.PYPI_PUBLISH == true + if: matrix.build.PYPI_PUBLISH == true && github.event_name != 'pull_request' uses: actions/upload-artifact@v3 with: name: wheels @@ -141,6 +152,7 @@ jobs: if-no-files-found: error publish-to-pypi: + if: github.event_name != 'pull_request' name: Publish the base PyPi package runs-on: ubuntu-latest needs: publish-binaries diff --git a/.github/workflows/update-glide-version/action.yml b/.github/workflows/update-glide-version/action.yml new file mode 100644 index 0000000000..77d60f4bb4 --- /dev/null +++ b/.github/workflows/update-glide-version/action.yml @@ -0,0 +1,26 @@ +name: Update GLIDE version in the config.toml file + +inputs: + folder_path: + description: "The folder path of the config.toml file" + required: true + type: string + named_os: + description: "The name of the current operating system" + required: false + default: "linux" + type: string + options: + - linux + - darwin +runs: + using: "composite" + steps: + - name: Update package version in config.toml + working-directory: ${{ inputs.folder_path }} + shell: bash + run: | + SED_FOR_MACOS=`if ${{ inputs.named_os == 'darwin' }}; then echo "''"; fi` + sed -i $SED_FOR_MACOS "s|unknown|${{ env.RELEASE_VERSION }}|g" ./config.toml + # log the config.toml file + cat ./config.toml diff --git a/benchmarks/rust/.cargo/config.toml b/benchmarks/rust/.cargo/config.toml index d7336d3306..cc73b700a2 100644 --- a/benchmarks/rust/.cargo/config.toml +++ b/benchmarks/rust/.cargo/config.toml @@ -1,3 +1,3 @@ [env] GLIDE_NAME = "Glide" # This should be overwritten by each wrapper library. -GLIDE_VERSION = "0.1.0" +GLIDE_VERSION = "unknown" diff --git a/csharp/lib/.cargo/config.toml b/csharp/lib/.cargo/config.toml index 45873d24a5..51f56c6ec2 100644 --- a/csharp/lib/.cargo/config.toml +++ b/csharp/lib/.cargo/config.toml @@ -1,3 +1,3 @@ [env] GLIDE_NAME = { value = "GlideC#", force = true } -GLIDE_VERSION = "0.1.0" +GLIDE_VERSION = "unknown" diff --git a/glide-core/.cargo/config.toml b/glide-core/.cargo/config.toml index d7336d3306..cc73b700a2 100644 --- a/glide-core/.cargo/config.toml +++ b/glide-core/.cargo/config.toml @@ -1,3 +1,3 @@ [env] GLIDE_NAME = "Glide" # This should be overwritten by each wrapper library. -GLIDE_VERSION = "0.1.0" +GLIDE_VERSION = "unknown" diff --git a/java/.cargo/config.toml b/java/.cargo/config.toml index 25c9e01f33..44b6861735 100644 --- a/java/.cargo/config.toml +++ b/java/.cargo/config.toml @@ -1,3 +1,3 @@ [env] GLIDE_NAME = { value = "GlideJava", force = true } -GLIDE_VERSION = "0.1.0" +GLIDE_VERSION = "unknown" diff --git a/node/rust-client/.cargo/config.toml b/node/rust-client/.cargo/config.toml index 56be8c8010..551cab4c9a 100644 --- a/node/rust-client/.cargo/config.toml +++ b/node/rust-client/.cargo/config.toml @@ -1,3 +1,3 @@ [env] GLIDE_NAME = { value = "GlideJS", force = true } -GLIDE_VERSION = "0.1.0" +GLIDE_VERSION = "unknown" diff --git a/node/tests/SharedTests.ts b/node/tests/SharedTests.ts index 3aa60e9101..35310a2dc3 100644 --- a/node/tests/SharedTests.ts +++ b/node/tests/SharedTests.ts @@ -85,7 +85,7 @@ export function runBaseTests(config: { const result = await client.customCommand(["CLIENT", "INFO"]); expect(result).toContain("lib-name=GlideJS"); - expect(result).toContain("lib-ver=0.1.0"); + expect(result).toContain("lib-ver=unknown"); }, protocol); }, config.timeout diff --git a/python/.cargo/config.toml b/python/.cargo/config.toml index 3188c03f8a..605cb60f60 100644 --- a/python/.cargo/config.toml +++ b/python/.cargo/config.toml @@ -1,3 +1,3 @@ [env] GLIDE_NAME = { value = "GlidePy", force = true } -GLIDE_VERSION = "0.1.0" +GLIDE_VERSION = "unknown" diff --git a/python/python/tests/test_async_client.py b/python/python/tests/test_async_client.py index 290613e453..2c357e0e0a 100644 --- a/python/python/tests/test_async_client.py +++ b/python/python/tests/test_async_client.py @@ -116,7 +116,7 @@ async def test_register_client_name_and_version(self, redis_client: TRedisClient info = await redis_client.custom_command(["CLIENT", "INFO"]) assert type(info) is str assert "lib-name=GlidePy" in info - assert "lib-ver=0.1.0" in info + assert "lib-ver=unknown" in info @pytest.mark.parametrize("cluster_mode", [True, False]) @pytest.mark.parametrize("protocol", [ProtocolVersion.RESP2, ProtocolVersion.RESP3]) @@ -981,7 +981,7 @@ async def test_expireat_pexpireat_ttl_with_positive_timeout( # set command clears the timeout. assert await redis_client.set(key, "bar") == OK current_time_ms = int(time.time() * 1000) - if not check_if_server_version_lt(redis_client, "7.0.0"): + if not await check_if_server_version_lt(redis_client, "7.0.0"): assert ( await redis_client.pexpireat( key, current_time_ms + 50000, ExpireOptions.HasExistingExpiry diff --git a/utils/get_licenses_from_ort.py b/utils/get_licenses_from_ort.py index 4ecf3c6ef6..877b4d7787 100644 --- a/utils/get_licenses_from_ort.py +++ b/utils/get_licenses_from_ort.py @@ -67,7 +67,7 @@ def __str__(self): ] all_licenses_set: Set = set() -unkown_licenses: List[PackageLicense] = [] +unknown_licenses: List[PackageLicense] = [] for ort_result in ort_results_per_lang: with open(ort_result.analyzer_result_file, "r") as ort_results, open( @@ -93,7 +93,7 @@ def __str__(self): final_licenses = [license] for license in final_licenses: if license not in APPROVED_LICENSES: - unkown_licenses.append( + unknown_licenses.append( PackageLicense(package["id"], ort_result.name, license) ) all_licenses_set.add(license) @@ -108,6 +108,6 @@ def __str__(self): for license in all_licenses_set: print(f"{license}") -print("\n\n#### Unkown / Not Pre-Approved Licenses #####\n") -for package in unkown_licenses: +print("\n\n#### unknown / Not Pre-Approved Licenses #####\n") +for package in unknown_licenses: print(str(package)) From f742bb3e0546445009dc497d5d70a5d1d1717bff Mon Sep 17 00:00:00 2001 From: barshaul Date: Sun, 18 Feb 2024 11:14:43 +0000 Subject: [PATCH 2/2] Removed unnecessary flags from the CD. --- .github/workflows/npm-cd.yml | 16 +++++----------- .github/workflows/pypi-cd.yml | 24 +++++++++--------------- 2 files changed, 14 insertions(+), 26 deletions(-) diff --git a/.github/workflows/npm-cd.yml b/.github/workflows/npm-cd.yml index 69d78aa85e..68dd3b3b82 100644 --- a/.github/workflows/npm-cd.yml +++ b/.github/workflows/npm-cd.yml @@ -12,6 +12,7 @@ on: - "v*.*" jobs: publish-binaries: + if: github.repository_owner == 'aws' name: Publish packages to NPM runs-on: ${{ matrix.build.RUNNER }} strategy: @@ -24,7 +25,6 @@ jobs: RUNNER: ubuntu-latest, ARCH: x64, TARGET: x86_64-unknown-linux-gnu, - NPM_PUBLISH: true, } - { OS: ubuntu-latest, @@ -32,7 +32,6 @@ jobs: RUNNER: [self-hosted, Linux, ARM64], ARCH: arm64, TARGET: aarch64-unknown-linux-gnu, - NPM_PUBLISH: true, CONTAINER: "2_28", } - { @@ -41,7 +40,6 @@ jobs: RUNNER: macos-latest, ARCH: x64, TARGET: x86_64-apple-darwin, - NPM_PUBLISH: true, } - { OS: macos-latest, @@ -49,7 +47,6 @@ jobs: RUNNER: macos-13-xlarge, arch: arm64, TARGET: aarch64-apple-darwin, - NPM_PUBLISH: true, } steps: - name: Checkout @@ -60,11 +57,10 @@ jobs: - name: Set the release version shell: bash run: | - export version=`if ${{ github.event_name == 'pull_request' }}; then echo '255.255.255'; else echo ${GITHUB_REF:11}; fi` - echo "RELEASE_VERSION=${version}" >> $GITHUB_ENV + export version=`if ${{ github.event_name == 'pull_request' }}; then echo '255.255.255'; else echo ${GITHUB_REF:11}; fi` + echo "RELEASE_VERSION=${version}" >> $GITHUB_ENV - name: Setup node - if: matrix.build.NPM_PUBLISH == true uses: actions/setup-node@v3 with: node-version: "16" @@ -81,7 +77,6 @@ jobs: named_os: ${{ matrix.build.NAMED_OS }} - name: Build Node wrapper - if: matrix.build.NPM_PUBLISH == true uses: ./.github/workflows/build-node-wrapper with: os: ${{ matrix.build.OS }} @@ -93,7 +88,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Publish to NPM - if: matrix.build.NPM_PUBLISH == true && github.event_name != 'pull_request' + if: github.event_name != 'pull_request' shell: bash working-directory: ./node run: | @@ -112,7 +107,6 @@ jobs: NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }} - name: Pack the Node package - if: matrix.build.NPM_PUBLISH == true shell: bash working-directory: ./node run: | @@ -126,7 +120,7 @@ jobs: NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }} - name: Upload the Node package - if: matrix.build.NPM_PUBLISH == true + if: github.event_name != 'pull_request' uses: actions/upload-artifact@v3 with: name: ${{ matrix.build.TARGET }} diff --git a/.github/workflows/pypi-cd.yml b/.github/workflows/pypi-cd.yml index cc273a3ede..003f2093b4 100644 --- a/.github/workflows/pypi-cd.yml +++ b/.github/workflows/pypi-cd.yml @@ -12,6 +12,7 @@ on: - "v*.*" jobs: publish-binaries: + if: github.repository_owner == 'aws' name: Publish packages to PyPi runs-on: ${{ matrix.build.RUNNER }} strategy: @@ -24,7 +25,6 @@ jobs: RUNNER: ubuntu-latest, ARCH: x64, TARGET: x86_64-unknown-linux-gnu, - PYPI_PUBLISH: true, } - { OS: ubuntu-latest, @@ -32,7 +32,6 @@ jobs: RUNNER: [self-hosted, Linux, ARM64], ARCH: arm64, TARGET: aarch64-unknown-linux-gnu, - PYPI_PUBLISH: true, CONTAINER: "2_28", } - { @@ -41,7 +40,6 @@ jobs: RUNNER: macos-latest, ARCH: x64, TARGET: x86_64-apple-darwin, - PYPI_PUBLISH: true, } - { OS: macos-latest, @@ -49,7 +47,6 @@ jobs: RUNNER: macos-13-xlarge, arch: arm64, TARGET: aarch64-apple-darwin, - PYPI_PUBLISH: true, } steps: - name: Checkout @@ -64,7 +61,6 @@ jobs: echo "RELEASE_VERSION=${version}" >> $GITHUB_ENV - name: Set the package version for Python - if: matrix.build.PYPI_PUBLISH == true working-directory: ./python run: | SED_FOR_MACOS=`if [[ "${{ matrix.build.OS }}" =~ .*"macos".* ]]; then echo "''"; fi` @@ -73,18 +69,18 @@ jobs: cat Cargo.toml - name: Set up Python - if: matrix.build.PYPI_PUBLISH == true && !contains(matrix.build.RUNNER, 'self-hosted') + if: ${{ !contains(matrix.build.RUNNER, 'self-hosted') }} uses: actions/setup-python@v4 with: python-version: "3.10" - name: Set up Python older versions for MacOS - if: matrix.build.PYPI_PUBLISH == true && startsWith(matrix.build.NAMED_OS, 'darwin') + if: startsWith(matrix.build.NAMED_OS, 'darwin') run: | brew install python@3.8 python@3.9 - name: Setup Python for self-hosted Ubuntu runners - if: matrix.build.PYPI_PUBLISH == true && contains(matrix.build.OS, 'ubuntu') && contains(matrix.build.RUNNER, 'self-hosted') + if: contains(matrix.build.OS, 'ubuntu') && contains(matrix.build.RUNNER, 'self-hosted') run: | sudo apt update -y sudo apt upgrade -y @@ -97,7 +93,6 @@ jobs: named_os: ${{ matrix.build.NAMED_OS }} - name: Build Python wrapper - if: matrix.build.PYPI_PUBLISH == true uses: ./.github/workflows/build-python-wrapper with: os: ${{ matrix.build.OS }} @@ -107,7 +102,6 @@ jobs: - name: Include protobuf files in the package working-directory: ./python - if: matrix.build.PYPI_PUBLISH == true run: | SED_FOR_MACOS=`if [[ "${{ matrix.build.OS }}" =~ .*"macos".* ]]; then echo "''"; fi` sed -i $SED_FOR_MACOS '/pb2.py/d' .gitignore @@ -115,12 +109,12 @@ jobs: cat .gitignore - name: Build Python wheels (linux) - if: matrix.build.PYPI_PUBLISH == true && startsWith(matrix.build.NAMED_OS, 'linux') && github.event_name != 'pull_request' + if: startsWith(matrix.build.NAMED_OS, 'linux') uses: PyO3/maturin-action@v1 with: working-directory: ./python target: ${{ matrix.build.TARGET }} - args: --release --strip --out wheels -i python3.8 python3.9 python3.10 python3.11 python3.12 + args: --release --strip --out wheels -i ${{ github.event_name != 'pull_request' && 'python3.8 python3.9 python3.10 python3.11 python3.12' || 'python3.10' }} manylinux: auto container: ${{ matrix.build.CONTAINER != '' && matrix.build.CONTAINER || '2014' }} before-script-linux: | @@ -136,15 +130,15 @@ jobs: fi - name: Build Python wheels (macos) - if: matrix.build.PYPI_PUBLISH == true && startsWith(matrix.build.NAMED_OS, 'darwin') && github.event_name != 'pull_request' + if: startsWith(matrix.build.NAMED_OS, 'darwin') uses: PyO3/maturin-action@v1 with: working-directory: ./python target: ${{ matrix.build.TARGET }} - args: --release --strip --out wheels -i python3.8 python3.9 python3.10 python3.11 python3.12 + args: --release --strip --out wheels -i ${{ github.event_name != 'pull_request' && 'python3.8 python3.9 python3.10 python3.11 python3.12' || 'python3.10' }} - name: Upload Python wheels - if: matrix.build.PYPI_PUBLISH == true && github.event_name != 'pull_request' + if: github.event_name != 'pull_request' uses: actions/upload-artifact@v3 with: name: wheels