From bf067bfd9d1ea9d68ccba05aa18a5cef5898722a Mon Sep 17 00:00:00 2001 From: pulpbot Date: Tue, 14 Nov 2023 11:20:27 +0000 Subject: [PATCH] Update CI files [noissue] --- .ci/ansible/Containerfile.j2 | 19 ++-- .github/template_gitref | 2 +- .github/workflows/build.yml | 37 +++++++ .github/workflows/ci.yml | 152 ++------------------------- .github/workflows/lint.yml | 49 +++++++++ .github/workflows/release.yml | 5 - .github/workflows/scripts/install.sh | 8 +- .github/workflows/scripts/release.py | 26 ++--- .github/workflows/test.yml | 129 +++++++++++++++++++++++ 9 files changed, 241 insertions(+), 186 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/test.yml diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index 482ee277..0eb5d5cc 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -8,23 +8,22 @@ ADD ./{{ item.name }} ./{{ item.name }} {% endfor %} # Install python packages -# Hacking botocore (https://github.com/boto/botocore/pull/1990) RUN pip3 install -{%- if s3_test | default(false) -%} -{{ " " }}django-storages[boto3] git+https://github.com/fabricio-aguiar/botocore.git@fix-100-continue -{%- endif -%} -{%- if azure_test | default(false) -%} -{{ " " }}django-storages[azure]>=1.12.2 -{%- endif -%} -{%- if gcp_test | default(false) -%} -{{ " " }}django-storages[google]>=1.13.2 -{%- endif -%} {%- for item in plugins -%} {%- if item.name == "pulp-certguard" -%} {{ " " }}python-dateutil rhsm {%- endif -%} {{ " " }}{{ item.source }} +{%- if item.name == "pulpcore" -%} +{%- if s3_test | default(false) -%} +[s3] +{%- elif azure_test | default(false) -%} +[azure] +{%- elif gcp_test | default(false) -%} +[google] +{%- endif -%} +{%- endif -%} {%- if item.ci_requirements | default(false) -%} {{ " " }}-r ./{{ item.name }}/ci_requirements.txt {%- endif -%} diff --git a/.github/template_gitref b/.github/template_gitref index e0764316..dfea88fd 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-248-g4bfc3e1 +2021.08.26-253-gb267834 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..82eaa2eb --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,37 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_ostree' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: Build +on: + workflow_call: + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + - uses: actions/setup-python@v4 + with: + python-version: "3.8" + - name: Install python dependencies + run: | + echo ::group::PYDEPS + pip install packaging wheel + echo ::endgroup:: + - name: Build package + run: python3 setup.py sdist bdist_wheel --python-tag py3 + - name: 'Upload Package whl' + uses: actions/upload-artifact@v3 + with: + name: plugin_package + path: dist/ + if-no-files-found: error + retention-days: 5 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf52069e..0111e73e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,155 +42,15 @@ jobs: run: python .ci/scripts/check_requirements.py lint: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - uses: actions/setup-python@v4 - with: - python-version: "3.8" - # lint_requirements contains tools needed for flake8, etc. - - name: Install requirements - run: pip3 install -r lint_requirements.txt + uses: "./.github/workflows/lint.yml" - # run black separately from flake8 to get a diff - - name: Run black - run: | - black --version - black --check --diff . - - # Lint code. - - name: Run flake8 - run: flake8 - - - name: Run extra lint checks - run: "[ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh" - - # check for any files unintentionally left out of MANIFEST.in - - name: Check manifest - run: check-manifest - - - name: Check for pulpcore imports outside of pulpcore.plugin - run: sh .ci/scripts/check_pulpcore_imports.sh - - - name: Check for gettext problems - run: sh .ci/scripts/check_gettext.sh - - test: - runs-on: ubuntu-latest - # run only after lint finishes + build: needs: lint - strategy: - fail-fast: false - matrix: - env: - - TEST: pulp - - TEST: docs - - TEST: azure - - TEST: s3 - - TEST: lowerbounds - outputs: - deprecations-pulp: ${{ steps.deprecations.outputs.deprecations-pulp }} - deprecations-azure: ${{ steps.deprecations.outputs.deprecations-azure }} - deprecations-s3: ${{ steps.deprecations.outputs.deprecations-s3 }} - deprecations-lowerbounds: ${{ steps.deprecations.outputs.deprecations-lowerbounds }} - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - - name: Install httpie - run: | - echo ::group::HTTPIE - pip install httpie - echo ::endgroup:: - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV - - - name: Set environment variables - run: | - echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV - - - name: Before Install - run: .github/workflows/scripts/before_install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Install - run: .github/workflows/scripts/install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Before Script - run: .github/workflows/scripts/before_script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - REDIS_DISABLED: ${{ contains('', matrix.env.TEST) }} - - - name: Setting secrets - if: github.event_name != 'pull_request' - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} + uses: "./.github/workflows/build.yml" - - name: Script - run: .github/workflows/scripts/script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Extract Deprecations from Logs - id: deprecations - run: echo deprecations-${{ matrix.env.TEST }}=$(docker logs pulp 2>&1 | grep -i pulpcore.deprecation | base64 -w 0) >> $GITHUB_OUTPUT - - - name: Logs - if: always() - run: | - echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true - docker images || true - docker ps -a || true - docker logs pulp || true - docker exec pulp ls -latr /etc/yum.repos.d/ || true - docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" + test: + needs: build + uses: "./.github/workflows/test.yml" deprecations: runs-on: ubuntu-latest if: github.base_ref == 'main' diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..68eafd4c --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,49 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_ostree' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: Lint +on: + workflow_call: + +jobs: + lint: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + - uses: actions/setup-python@v4 + with: + python-version: "3.8" + # lint_requirements contains tools needed for flake8, etc. + - name: Install requirements + run: pip3 install -r lint_requirements.txt + + # run black separately from flake8 to get a diff + - name: Run black + run: | + black --version + black --check --diff . + + # Lint code. + - name: Run flake8 + run: flake8 + + - name: Run extra lint checks + run: "[ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh" + + # check for any files unintentionally left out of MANIFEST.in + - name: Check manifest + run: check-manifest + + - name: Check for pulpcore imports outside of pulpcore.plugin + run: sh .ci/scripts/check_pulpcore_imports.sh + + - name: Check for gettext problems + run: sh .ci/scripts/check_gettext.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f6d76f64..1c986941 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,7 +12,6 @@ on: inputs: release: description: "Release tag (e.g. 3.2.1)" - required: true before_script: description: | Bash code to run before bindings and docs are built. This should only be used when re-running @@ -287,9 +286,5 @@ jobs: echo "GH Issues $GH_ISSUES" python .ci/scripts/update_github.py - - name: Tweet - continue-on-error: true - run: python .ci/scripts/tweet.py ${{ github.event.inputs.release }} - - name: Create release on GitHub run: bash .github/workflows/scripts/create_release_from_tag.sh ${{ github.event.inputs.release }} diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 933b79c4..3036779b 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -15,6 +15,9 @@ set -euv source .github/workflows/scripts/utils.sh +PLUGIN_VERSION="$(sed -n -e 's/^\s*current_version\s*=\s*//p' .bumpversion.cfg | python -c 'from packaging.version import Version; print(Version(input()))')" +PLUGIN_NAME="./pulp_ostree/dist/pulp_ostree-${PLUGIN_VERSION}-py3-none-any.whl" + export PULP_API_ROOT="/pulp/" PIP_REQUIREMENTS=("pulp-cli-ostree") @@ -33,11 +36,6 @@ fi cd .ci/ansible/ -if [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then - PLUGIN_NAME=./pulp_ostree/dist/pulp_ostree-$PLUGIN_VERSION-py3-none-any.whl -else - PLUGIN_NAME=./pulp_ostree -fi cat >> vars/main.yaml << VARSYAML image: name: pulp diff --git a/.github/workflows/scripts/release.py b/.github/workflows/scripts/release.py index c6410fc5..d20b923b 100755 --- a/.github/workflows/scripts/release.py +++ b/.github/workflows/scripts/release.py @@ -10,6 +10,7 @@ import os import textwrap import requests +import subprocess from git import Repo from pathlib import Path @@ -56,11 +57,7 @@ def create_release_commits(repo, release_version, plugin_path): # Second commit: release version os.system("bump2version release --allow-dirty") - git.add(f"{plugin_path}/pulp_ostree/*") - git.add(f"{plugin_path}/docs/conf.py") - git.add(f"{plugin_path}/setup.py") - git.add(f"{plugin_path}/requirements.txt") - git.add(f"{plugin_path}/.bumpversion.cfg") + git.add(f"{plugin_path}") git.commit("-m", f"Release {release_version}\nGH Issues: {issues}\n\n[noissue]") sha = repo.head.object.hexsha short_sha = git.rev_parse(sha, short=7) @@ -75,11 +72,7 @@ def create_release_commits(repo, release_version, plugin_path): if not new_dev_version: raise RuntimeError("Could not detect new dev version ... aborting.") - git.add(f"{plugin_path}/pulp_ostree/*") - git.add(f"{plugin_path}/docs/conf.py") - git.add(f"{plugin_path}/setup.py") - git.add(f"{plugin_path}/requirements.txt") - git.add(f"{plugin_path}/.bumpversion.cfg") + git.add(f"{plugin_path}") git.commit("-m", f"Bump to {new_dev_version}\n\n[noissue]") print(f"Release commit == {short_sha}") print(f"All changes were committed on branch: release_{release_version}") @@ -143,6 +136,7 @@ def main(): "release_version", type=str, help="The version string for the release.", + nargs="?", ) args = parser.parse_args() @@ -152,20 +146,14 @@ def main(): release_path = os.path.dirname(os.path.abspath(__file__)) plugin_path = release_path.split("/.github")[0] - version = None - with open(f"{plugin_path}/setup.py") as fp: - for line in fp.readlines(): - if "version=" in line: - version = re.split("\"|'", line)[1] - if not version: - raise RuntimeError("Could not detect existing version ... aborting.") - release_version = version.replace(".dev", "") + output = subprocess.check_output(["bump2version", "--dry-run", "--list", "release"]) + release_version = re.findall(r"\nnew_version=([0-9.]*)\n", output.decode())[0] print(f"\n\nRepo path: {plugin_path}") repo = Repo(plugin_path) release_commit = None - if release_version != release_version_arg: + if release_version_arg and release_version != release_version_arg: # Look for a commit with the requested release version for commit in repo.iter_commits(): if f"Release {release_version_arg}\n" in commit.message: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..1bfb2711 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,129 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_ostree' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: Test +on: + workflow_call: + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + env: + - TEST: pulp + - TEST: docs + - TEST: azure + - TEST: s3 + - TEST: lowerbounds + outputs: + deprecations-pulp: ${{ steps.deprecations.outputs.deprecations-pulp }} + deprecations-azure: ${{ steps.deprecations.outputs.deprecations-azure }} + deprecations-s3: ${{ steps.deprecations.outputs.deprecations-s3 }} + deprecations-lowerbounds: ${{ steps.deprecations.outputs.deprecations-lowerbounds }} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - uses: actions/setup-python@v4 + with: + python-version: "3.8" + + - uses: actions/download-artifact@v3 + with: + name: plugin_package + path: dist/ + + - name: Install httpie + run: | + echo ::group::HTTPIE + pip install httpie + echo ::endgroup:: + echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV + + - name: Set environment variables + run: | + echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV + + - name: Before Install + run: .github/workflows/scripts/before_install.sh + shell: bash + env: + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + GITHUB_PULL_REQUEST: ${{ github.event.number }} + GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} + GITHUB_BRANCH: ${{ github.head_ref }} + GITHUB_REPO_SLUG: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} + + - name: Install + run: .github/workflows/scripts/install.sh + shell: bash + env: + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + GITHUB_PULL_REQUEST: ${{ github.event.number }} + GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} + GITHUB_BRANCH: ${{ github.head_ref }} + GITHUB_REPO_SLUG: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} + + - name: Before Script + run: .github/workflows/scripts/before_script.sh + shell: bash + env: + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + GITHUB_PULL_REQUEST: ${{ github.event.number }} + GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} + GITHUB_BRANCH: ${{ github.head_ref }} + GITHUB_REPO_SLUG: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} + REDIS_DISABLED: ${{ contains('', matrix.env.TEST) }} + + - name: Setting secrets + if: github.event_name != 'pull_request' + run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + env: + SECRETS_CONTEXT: ${{ toJson(secrets) }} + + - name: Script + run: .github/workflows/scripts/script.sh + shell: bash + env: + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + GITHUB_PULL_REQUEST: ${{ github.event.number }} + GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} + GITHUB_BRANCH: ${{ github.head_ref }} + GITHUB_REPO_SLUG: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} + + - name: Extract Deprecations from Logs + id: deprecations + run: echo deprecations-${{ matrix.env.TEST }}=$(docker logs pulp 2>&1 | grep -i pulpcore.deprecation | base64 -w 0) >> $GITHUB_OUTPUT + + - name: Logs + if: always() + run: | + echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" + http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true + docker images || true + docker ps -a || true + docker logs pulp || true + docker exec pulp ls -latr /etc/yum.repos.d/ || true + docker exec pulp cat /etc/yum.repos.d/* || true + docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree"