From 774c22090dca8663c1a6c27990c61c712792bb88 Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 10 Aug 2021 14:07:50 +0300 Subject: [PATCH 1/9] Adds update_package_properties and build_packages pipelines --- .github/workflows/build-package.yml | 62 +++++++++++++++++++ .../workflows/update_package_properties.yml | 52 ++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 .github/workflows/build-package.yml create mode 100644 .github/workflows/update_package_properties.yml diff --git a/.github/workflows/build-package.yml b/.github/workflows/build-package.yml new file mode 100644 index 000000000..c17d2b314 --- /dev/null +++ b/.github/workflows/build-package.yml @@ -0,0 +1,62 @@ +name: Build Package + +env: + MAIN_BRANCH: "all-pgautofailover-enterprise" + PACKAGE_CLOUD_REPO_NAME: "sample" + PACKAGE_ENCRYPTION_KEY: ${{ secrets.PACKAGE_ENCRYPTION_KEY }} +on: + push: + branches: "**" + + workflow_dispatch: + +jobs: + build_package: + name: Build package + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + platform: + - el/7 + - el/8 + - ol/7 + - debian/stretch + - debian/buster + - ubuntu/xenial + - ubuntu/bionic + - ubuntu/focal + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Clone tools branch + run: git clone -b issue_156 --depth=1 https://github.com/citusdata/tools.git tools + + - name: Install package dependencies + run: sudo apt install libcurl4-openssl-dev libssl-dev + + - name: Install python requirements + run: python -m pip install -r tools/packaging_automation/requirements.txt + + - name: Build packages + run: | + python -m tools.packaging_automation.citus_package \ + --gh_token "${{ secrets.GH_TOKEN }}" \ + --platform "${{ matrix.platform }}" \ + --build_type 'release' \ + --secret_key "${{ secrets.PACKAGING_SECRET_KEY }}" \ + --passphrase "${{ secrets.PACKAGING_PASSPHRASE }}" \ + --output_dir "$(pwd)/packages/" \ + --input_files_dir "$(pwd)" + + - name: Publish packages + run: | + python -m tools.packaging_automation.upload_to_package_cloud \ + --platform ${{ matrix.platform }} \ + --package_cloud_api_token ${{ secrets.PACKAGE_CLOUD_API_TOKEN }} \ + --repository_name "${PACKAGE_CLOUD_REPO_NAME}" \ + --output_file_path "$(pwd)/packages" \ + --current_branch "${GITHUB_REF##*/}" \ + --main_branch ${MAIN_BRANCH} diff --git a/.github/workflows/update_package_properties.yml b/.github/workflows/update_package_properties.yml new file mode 100644 index 000000000..a03ee1c7c --- /dev/null +++ b/.github/workflows/update_package_properties.yml @@ -0,0 +1,52 @@ +name: Update Package Properties + +env: + PRJ_NAME: "pg_auto_failover_enterprise" + GH_TOKEN: ${{ secrets.GH_TOKEN }} +on: + workflow_dispatch: + inputs: + tag_name: + description: "The tag name on the project repo to be released" + fancy_version_no: + description: "If fancy versioning is needed, set value greater than 1 e.g 2" + default: "1" + microsoft_email: + description: "Email to be written on changelogs" + default: "gindibay@microsoft.com" + name: + description: "Name to be written on changelogs" + default: "Gurkan Indibay" +jobs: + update_package_properties: + name: Update package properties + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install dependencies + run: sudo apt install libcurl4-openssl-dev libssl-dev + + - name: Clone tools branch + run: git clone -b issue-149 --depth=1 https://github.com/citusdata/tools.git tools + + - name: Set git name and email + run: | + git config --global user.email "${{ github.event.inputs.microsoft_email }}" && \ + git config --global user.name "${{ github.event.inputs.name }}" + + - name: Install python requirements + run: python -m pip install -r tools/packaging_automation/requirements.txt + + - name: Update package properties + run: | + python -m tools.packaging_automation.update_package_properties \ + --gh_token="${GH_TOKEN}" \ + --prj_name "${PRJ_NAME}" \ + --tag_name ${{ github.event.inputs.tag_name }} \ + --email ${{ github.event.inputs.microsoft_email }} \ + --name "${{ github.event.inputs.name }}" \ + --pipeline \ + --exec_path "$(pwd)" From 9d28259ebda7caae5ad29c419d71bc78b8abcb72 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 26 Aug 2021 12:07:01 +0300 Subject: [PATCH 2/9] Fixes PR #713 comments --- .github/workflows/build-package.yml | 4 ++-- .github/workflows/update_package_properties.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-package.yml b/.github/workflows/build-package.yml index c17d2b314..8e78089da 100644 --- a/.github/workflows/build-package.yml +++ b/.github/workflows/build-package.yml @@ -32,7 +32,7 @@ jobs: uses: actions/checkout@v2 - name: Clone tools branch - run: git clone -b issue_156 --depth=1 https://github.com/citusdata/tools.git tools + run: git clone -b v0.8.1 --depth=1 https://github.com/citusdata/tools.git tools - name: Install package dependencies run: sudo apt install libcurl4-openssl-dev libssl-dev @@ -59,4 +59,4 @@ jobs: --repository_name "${PACKAGE_CLOUD_REPO_NAME}" \ --output_file_path "$(pwd)/packages" \ --current_branch "${GITHUB_REF##*/}" \ - --main_branch ${MAIN_BRANCH} + --main_branch "${MAIN_BRANCH}" diff --git a/.github/workflows/update_package_properties.yml b/.github/workflows/update_package_properties.yml index a03ee1c7c..acce3f972 100644 --- a/.github/workflows/update_package_properties.yml +++ b/.github/workflows/update_package_properties.yml @@ -30,7 +30,7 @@ jobs: run: sudo apt install libcurl4-openssl-dev libssl-dev - name: Clone tools branch - run: git clone -b issue-149 --depth=1 https://github.com/citusdata/tools.git tools + run: git clone -b v0.8.1 --depth=1 https://github.com/citusdata/tools.git tools - name: Set git name and email run: | @@ -47,6 +47,6 @@ jobs: --prj_name "${PRJ_NAME}" \ --tag_name ${{ github.event.inputs.tag_name }} \ --email ${{ github.event.inputs.microsoft_email }} \ - --name "${{ github.event.inputs.name }}" \ + --name ${{ github.event.inputs.name }} \ --pipeline \ --exec_path "$(pwd)" From fca8c0651695f1d95886747cdaef4a1364961970 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 26 Aug 2021 12:30:56 +0300 Subject: [PATCH 3/9] Removes ubuntu/xenial among supported os list --- .github/workflows/build-package.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-package.yml b/.github/workflows/build-package.yml index 8e78089da..f32559d43 100644 --- a/.github/workflows/build-package.yml +++ b/.github/workflows/build-package.yml @@ -23,7 +23,6 @@ jobs: - ol/7 - debian/stretch - debian/buster - - ubuntu/xenial - ubuntu/bionic - ubuntu/focal From 83f58db067f7597789fd4f6ee8bf44ff4e1e9383 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 26 Aug 2021 12:38:29 +0300 Subject: [PATCH 4/9] Transforms secret varaibles into environment variables --- .github/workflows/build-package.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-package.yml b/.github/workflows/build-package.yml index 8e78089da..c4e860e0a 100644 --- a/.github/workflows/build-package.yml +++ b/.github/workflows/build-package.yml @@ -4,6 +4,10 @@ env: MAIN_BRANCH: "all-pgautofailover-enterprise" PACKAGE_CLOUD_REPO_NAME: "sample" PACKAGE_ENCRYPTION_KEY: ${{ secrets.PACKAGE_ENCRYPTION_KEY }} + GH_TOKEN: ${{ secrets.GH_TOKEN }} + PACKAGE_CLOUD_API_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_TOKEN }} + PACKAGING_SECRET_KEY: ${{ secrets.PACKAGING_SECRET_KEY }} + PACKAGING_PASSPHRASE: ${{ secrets.PACKAGING_PASSPHRASE }} on: push: branches: "**" @@ -43,11 +47,11 @@ jobs: - name: Build packages run: | python -m tools.packaging_automation.citus_package \ - --gh_token "${{ secrets.GH_TOKEN }}" \ - --platform "${{ matrix.platform }}" \ + --gh_token "${GH_TOKEN}" \ + --platform ${{ matrix.platform }} \ --build_type 'release' \ - --secret_key "${{ secrets.PACKAGING_SECRET_KEY }}" \ - --passphrase "${{ secrets.PACKAGING_PASSPHRASE }}" \ + --secret_key "${PACKAGING_SECRET_KEY}" \ + --passphrase "${PACKAGING_PASSPHRASE}" \ --output_dir "$(pwd)/packages/" \ --input_files_dir "$(pwd)" @@ -55,7 +59,7 @@ jobs: run: | python -m tools.packaging_automation.upload_to_package_cloud \ --platform ${{ matrix.platform }} \ - --package_cloud_api_token ${{ secrets.PACKAGE_CLOUD_API_TOKEN }} \ + --package_cloud_api_token "${PACKAGE_CLOUD_API_TOKEN}" \ --repository_name "${PACKAGE_CLOUD_REPO_NAME}" \ --output_file_path "$(pwd)/packages" \ --current_branch "${GITHUB_REF##*/}" \ From 58c4e2ff3a61971776a742a42fcf84f3f4fc0619 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Thu, 26 Aug 2021 12:42:54 +0300 Subject: [PATCH 5/9] Apply suggestions from code review Co-authored-by: Hanefi Onaldi --- .github/workflows/build-package.yml | 9 +++++---- .github/workflows/update_package_properties.yml | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-package.yml b/.github/workflows/build-package.yml index f32559d43..704dfbbae 100644 --- a/.github/workflows/build-package.yml +++ b/.github/workflows/build-package.yml @@ -6,7 +6,8 @@ env: PACKAGE_ENCRYPTION_KEY: ${{ secrets.PACKAGE_ENCRYPTION_KEY }} on: push: - branches: "**" + branches: + - "**" workflow_dispatch: @@ -31,7 +32,7 @@ jobs: uses: actions/checkout@v2 - name: Clone tools branch - run: git clone -b v0.8.1 --depth=1 https://github.com/citusdata/tools.git tools + run: git clone -b v0.8.1 --depth=1 https://github.com/citusdata/tools.git tools - name: Install package dependencies run: sudo apt install libcurl4-openssl-dev libssl-dev @@ -41,7 +42,7 @@ jobs: - name: Build packages run: | - python -m tools.packaging_automation.citus_package \ + python -m tools.packaging_automation.citus_package \ --gh_token "${{ secrets.GH_TOKEN }}" \ --platform "${{ matrix.platform }}" \ --build_type 'release' \ @@ -52,7 +53,7 @@ jobs: - name: Publish packages run: | - python -m tools.packaging_automation.upload_to_package_cloud \ + python -m tools.packaging_automation.upload_to_package_cloud \ --platform ${{ matrix.platform }} \ --package_cloud_api_token ${{ secrets.PACKAGE_CLOUD_API_TOKEN }} \ --repository_name "${PACKAGE_CLOUD_REPO_NAME}" \ diff --git a/.github/workflows/update_package_properties.yml b/.github/workflows/update_package_properties.yml index acce3f972..62375e4ec 100644 --- a/.github/workflows/update_package_properties.yml +++ b/.github/workflows/update_package_properties.yml @@ -30,7 +30,7 @@ jobs: run: sudo apt install libcurl4-openssl-dev libssl-dev - name: Clone tools branch - run: git clone -b v0.8.1 --depth=1 https://github.com/citusdata/tools.git tools + run: git clone -b v0.8.1 --depth=1 https://github.com/citusdata/tools.git tools - name: Set git name and email run: | From c0ad0e4f2a2f074449743a268d6b1320d0e6b32d Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 27 Aug 2021 10:31:10 +0300 Subject: [PATCH 6/9] Fixes format errors --- .github/workflows/build-package.yml | 10 +++++----- .github/workflows/update_package_properties.yml | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-package.yml b/.github/workflows/build-package.yml index 22794a702..64ea755bb 100644 --- a/.github/workflows/build-package.yml +++ b/.github/workflows/build-package.yml @@ -42,14 +42,14 @@ jobs: run: sudo apt install libcurl4-openssl-dev libssl-dev - name: Install python requirements - run: python -m pip install -r tools/packaging_automation/requirements.txt + run: sudo apt-get update && python -m pip install -r tools/packaging_automation/requirements.txt - name: Build packages run: | python -m tools.packaging_automation.citus_package \ --gh_token "${GH_TOKEN}" \ - --platform ${{ matrix.platform }} \ - --build_type 'release' \ + --platform "${{ matrix.platform }}" \ + --build_type "release" \ --secret_key "${PACKAGING_SECRET_KEY}" \ --passphrase "${PACKAGING_PASSPHRASE}" \ --output_dir "$(pwd)/packages/" \ @@ -58,8 +58,8 @@ jobs: - name: Publish packages run: | python -m tools.packaging_automation.upload_to_package_cloud \ - --platform ${{ matrix.platform }} \ - --package_cloud_api_token ${{ secrets.PACKAGE_CLOUD_API_TOKEN }} \ + --platform "${{ matrix.platform }}" \ + --package_cloud_api_token "${{ secrets.PACKAGE_CLOUD_API_TOKEN }}" \ --repository_name "${PACKAGE_CLOUD_REPO_NAME}" \ --output_file_path "$(pwd)/packages" \ --current_branch "${GITHUB_REF##*/}" \ diff --git a/.github/workflows/update_package_properties.yml b/.github/workflows/update_package_properties.yml index 62375e4ec..b4f14675a 100644 --- a/.github/workflows/update_package_properties.yml +++ b/.github/workflows/update_package_properties.yml @@ -27,7 +27,7 @@ jobs: uses: actions/checkout@v2 - name: Install dependencies - run: sudo apt install libcurl4-openssl-dev libssl-dev + run: sudo apt-get update && apt install libcurl4-openssl-dev libssl-dev - name: Clone tools branch run: git clone -b v0.8.1 --depth=1 https://github.com/citusdata/tools.git tools @@ -45,8 +45,8 @@ jobs: python -m tools.packaging_automation.update_package_properties \ --gh_token="${GH_TOKEN}" \ --prj_name "${PRJ_NAME}" \ - --tag_name ${{ github.event.inputs.tag_name }} \ - --email ${{ github.event.inputs.microsoft_email }} \ - --name ${{ github.event.inputs.name }} \ + --tag_name "${{ github.event.inputs.tag_name }}" \ + --email "${{ github.event.inputs.microsoft_email }}" \ + --name "${{ github.event.inputs.name }}" \ --pipeline \ --exec_path "$(pwd)" From 58de0a583e5eca2b8c50a382194f62fc0a0ed88e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Fri, 27 Aug 2021 11:03:41 +0300 Subject: [PATCH 7/9] Adds ap-get update --- .github/workflows/build-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-package.yml b/.github/workflows/build-package.yml index 64ea755bb..709a85517 100644 --- a/.github/workflows/build-package.yml +++ b/.github/workflows/build-package.yml @@ -39,7 +39,7 @@ jobs: run: git clone -b v0.8.1 --depth=1 https://github.com/citusdata/tools.git tools - name: Install package dependencies - run: sudo apt install libcurl4-openssl-dev libssl-dev + run: sudo apt-get update && sudo apt install libcurl4-openssl-dev libssl-dev - name: Install python requirements run: sudo apt-get update && python -m pip install -r tools/packaging_automation/requirements.txt From 44d296597e067a7ee71d2a687819f1e889b0c6ff Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 27 Aug 2021 11:41:54 +0300 Subject: [PATCH 8/9] Removes ubuntu/xenial from travis --- .travis.yml | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index f8e239ea3..011996ba9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ sudo: required -services: [ docker ] +services: [docker] language: perl perl: "5.18" env: @@ -27,7 +27,6 @@ env: # We do not have Ubuntu Focal repos yet on packages.microsoft.com - TARGET_PLATFORM=ubuntu/focal - TARGET_PLATFORM=ubuntu/bionic - - TARGET_PLATFORM=ubuntu/xenial # Packages on package cloud # Oracle Linux 6 is old and it doesn't work with static assert: # https://github.com/citusdata/pg_auto_failover/issues/200 @@ -53,19 +52,19 @@ before_install: fi install: true script: | - if [ ${TRAVIS_EVENT_TYPE} == 'push' ] || [ ${TRAVIS_EVENT_TYPE} == 'pull_request' ] || [ ${TRAVIS_EVENT_TYPE} == 'api' ] ; then - if [ -z "${UNENCRYPTED_PACKAGE:-}" ]; then - citus_package -p ${TARGET_PLATFORM} 'local' 'release' - else - build_new_release - fi - elif [ ${TRAVIS_EVENT_TYPE} == 'cron' ] ; then - if [ -z "${UNENCRYPTED_PACKAGE:-}" ]; then - citus_package -p ${TARGET_PLATFORM} 'local' 'nightly' - else - build_new_nightly - fi - fi + if [ ${TRAVIS_EVENT_TYPE} == 'push' ] || [ ${TRAVIS_EVENT_TYPE} == 'pull_request' ] || [ ${TRAVIS_EVENT_TYPE} == 'api' ] ; then + if [ -z "${UNENCRYPTED_PACKAGE:-}" ]; then + citus_package -p ${TARGET_PLATFORM} 'local' 'release' + else + build_new_release + fi + elif [ ${TRAVIS_EVENT_TYPE} == 'cron' ] ; then + if [ -z "${UNENCRYPTED_PACKAGE:-}" ]; then + citus_package -p ${TARGET_PLATFORM} 'local' 'nightly' + else + build_new_nightly + fi + fi deploy: - provider: packagecloud From e236ae53d243e613a50f39ee1a0a07ebabb5b33e Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 27 Aug 2021 15:29:58 +0300 Subject: [PATCH 9/9] Removes unnecessary apt-get update --- .github/workflows/build-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-package.yml b/.github/workflows/build-package.yml index 709a85517..e54669d34 100644 --- a/.github/workflows/build-package.yml +++ b/.github/workflows/build-package.yml @@ -42,7 +42,7 @@ jobs: run: sudo apt-get update && sudo apt install libcurl4-openssl-dev libssl-dev - name: Install python requirements - run: sudo apt-get update && python -m pip install -r tools/packaging_automation/requirements.txt + run: python -m pip install -r tools/packaging_automation/requirements.txt - name: Build packages run: |