Add unified docs-ci

Related to: pulp/pulp-docs#31

[noissue]

.ci/ansible/Containerfile.j2

@@ -1,4 +1,4 @@
-FROM {{ ci_base | default("ghcr.io/pulp/pulp-ci-centos:" + pulp_container_tag) }}
+FROM {{ ci_base | default(pulp_default_container) }}
 
 # Add source directories to container
 {% for item in plugins %}
@@ -11,7 +11,7 @@ ADD ./{{ item.name }} ./{{ item.name }}
 
 RUN pip3 install
 {%- if s3_test | default(false) -%}
-{{ " " }}git+https://github.com/fabricio-aguiar/botocore.git@fix-100-continue
+{{ " " }}git+https://github.com/gerrod3/botocore.git@fix-100-continue
 {%- endif -%}
 {%- for item in plugins -%}
 {{ " " }}{{ item.source }}

.ci/ansible/start_container.yaml

@@ -83,22 +83,14 @@
           command: "docker logs pulp"
           failed_when: true
 
-    - block:
-        - name: "Check version of component being tested"
-          assert:
-            that:
-              - (result.json.versions | items2dict(key_name="component", value_name="version"))[component_name] | canonical_semver == (component_version | canonical_semver)
-            fail_msg: |
-              Component {{ component_name }} was expected to be installed in version {{ component_version }}.
-              Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[component_name] }}.
-      rescue:
-        - name: "Check version of component being tested (legacy)"
-          assert:
-            that:
-              - (result.json.versions | items2dict(key_name="component", value_name="version"))[legacy_component_name] | canonical_semver == (component_version | canonical_semver)
-            fail_msg: |
-              Component {{ legacy_component_name }} was expected to be installed in version {{ component_version }}.
-              Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[legacy_component_name] }}.
+    - name: "Check version of component being tested"
+      assert:
+        that:
+          - (result.json.versions | items2dict(key_name="component", value_name="version"))[item.app_label] | canonical_semver == (component_version | canonical_semver)
+        fail_msg: |
+          Component {{ item.app_label }} was expected to be installed in version {{ component_version }}.
+          Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[item.app_label] }}.
+      loop: "{{ 'plugins' | ansible.builtin.extract(lookup('ansible.builtin.file', '../../template_config.yml') | from_yaml) }}"
 
     - name: "Set pulp password in .netrc"
       copy:

.ci/scripts/check_release.py

@@ -29,7 +29,7 @@ def main():
         "--branches",
         default="supported",
         help="A comma separated list of branches to check for releases. Can also use keyword: "
-        "'supported'. Defaults to 'supported', see `ci_update_branches` in "
+        "'supported'. Defaults to 'supported', see `supported_release_branches` in "
         "`plugin_template.yml`.",
     )
     opts = parser.parse_args()
@@ -46,12 +46,15 @@ def main():
         if branches == "supported":
             with open(f"{d}/template_config.yml", mode="r") as f:
                 tc = yaml.safe_load(f)
-                branches = tc["ci_update_branches"]
-            branches.append(DEFAULT_BRANCH)
+                branches = set(tc["supported_release_branches"])
+            latest_release_branch = tc["latest_release_branch"]
+            if latest_release_branch is not None:
+                branches.add(latest_release_branch)
+            branches.add(DEFAULT_BRANCH)
         else:
-            branches = branches.split(",")
+            branches = set(branches.split(","))
 
-        if diff := set(branches) - set(available_branches):
+        if diff := branches - set(available_branches):
             print(f"Supplied branches contains non-existent branches! {diff}")
             exit(1)
 
@@ -62,7 +65,7 @@ def main():
             if branch != DEFAULT_BRANCH:
                 # Check if a Z release is needed
                 changes = repo.git.ls_tree("-r", "--name-only", f"origin/{branch}", "CHANGES/")
-                z_release = False
+                z_changelog = False
                 for change in changes.split("\n"):
                     # Check each changelog file to make sure everything checks out
                     _, ext = os.path.splitext(change)
@@ -72,18 +75,48 @@ def main():
                             f"{branch} release branch!"
                         )
                     elif ext in Z_CHANGELOG_EXTS:
-                        z_release = True
-                if z_release:
+                        z_changelog = True
+
+                last_tag = repo.git.describe("--tags", "--abbrev=0", f"origin/{branch}")
+                req_txt_diff = repo.git.diff(
+                    f"{last_tag}", f"origin/{branch}", "--name-only", "--", "requirements.txt"
+                )
+                if z_changelog or req_txt_diff:
                     # Blobless clone does not have file contents for Z branches,
                     # check commit message for last Z bump
                     git_branch = f"origin/{branch}"
-                    next_version = repo.git.log(
-                        "--oneline", "--grep=Bump to", "-n 1", git_branch, "--", ".bumpversion.cfg"
-                    ).split("to")[-1]
+                    next_version = None
+                    bump_commit = repo.git.log(
+                        "--oneline",
+                        "--grep=Bump version",
+                        "-n 1",
+                        git_branch,
+                        "--",
+                        ".bumpversion.cfg",
+                    )
+                    if bump_commit:
+                        next_version = bump_commit.split("→ ")[-1]
+                    # If not found - try old-commit-msg
+                    if not next_version:
+                        bump_commit = repo.git.log(
+                            "--oneline",
+                            "--grep=Bump to",
+                            "-n 1",
+                            git_branch,
+                            "--",
+                            ".bumpversion.cfg",
+                        )
+                        next_version = bump_commit.split("to ")[-1] if bump_commit else None
+
+                    # You could, theoretically, be next_vers==None here - but that's always
+                    # been true for this script.
                     next_version = Version(next_version)
+                    reason = "CHANGES" if z_changelog else "requirements.txt"
                     print(
                         f"A Z-release is needed for {branch}, "
-                        f"New Version: {next_version.base_version}"
+                        f"Prev: {last_tag}, "
+                        f"Next: {next_version.base_version}, "
+                        f"Reason: {reason}"
                     )
                     releases.append(next_version)
             else:

.ci/scripts/collect_changes.py

@@ -11,30 +11,37 @@
 
 import toml
 from git import GitCommandError, Repo
-from pkg_resources import parse_version
+from packaging.version import parse as parse_version
 
 # Read Towncrier settings
 tc_settings = toml.load("pyproject.toml")["tool"]["towncrier"]
 
 CHANGELOG_FILE = tc_settings.get("filename", "NEWS.rst")
 START_STRING = tc_settings.get(
     "start_string",
-    "<!-- towncrier release notes start -->\n"
-    if CHANGELOG_FILE.endswith(".md")
-    else ".. towncrier release notes start\n",
+    (
+        "<!-- towncrier release notes start -->\n"
+        if CHANGELOG_FILE.endswith(".md")
+        else ".. towncrier release notes start\n"
+    ),
 )
 TITLE_FORMAT = tc_settings.get("title_format", "{name} {version} ({project_date})")
 
 
+# Build a regex to find the header of a changelog section.
+# It must have a single capture group to single out the version.
+# see help(re.split) for more info.
 NAME_REGEX = r".*"
-VERSION_REGEX = r"([0-9]+\.[0-9]+\.[0-9][0-9ab]*)"
+VERSION_REGEX = r"[0-9]+\.[0-9]+\.[0-9][0-9ab]*"
+VERSION_CAPTURE_REGEX = rf"({VERSION_REGEX})"
 DATE_REGEX = r"[0-9]{4}-[0-9]{2}-[0-9]{2}"
 TITLE_REGEX = (
     "("
     + re.escape(
         TITLE_FORMAT.format(name="NAME_REGEX", version="VERSION_REGEX", project_date="DATE_REGEX")
     )
     .replace("NAME_REGEX", NAME_REGEX)
+    .replace("VERSION_REGEX", VERSION_CAPTURE_REGEX, 1)
     .replace("VERSION_REGEX", VERSION_REGEX)
     .replace("DATE_REGEX", DATE_REGEX)
     + ")"

.github/template_gitref

@@ -1 +1 @@
-2021.08.26-301-g83f0607
+2021.08.26-327-g520b576

.github/workflows/build.yml

@@ -25,7 +25,7 @@ jobs:
           path: "pulp_gem"
       - uses: "actions/setup-python@v4"
         with:
-          python-version: "3.8"
+          python-version: "3.11"
       - name: "Install python dependencies"
         run: |
           echo ::group::PYDEPS

.github/workflows/changelog.yml

@@ -30,7 +30,7 @@ jobs:
 
       - uses: "actions/setup-python@v4"
         with:
-          python-version: "3.8"
+          python-version: "3.11"
 
       - name: "Install python dependencies"
         run: |

.github/workflows/ci.yml

@@ -27,7 +27,7 @@ jobs:
           path: "pulp_gem"
       - uses: "actions/setup-python@v4"
         with:
-          python-version: "3.8"
+          python-version: "3.11"
       - name: "Install python dependencies"
         run: |
           echo ::group::PYDEPS

.github/workflows/codeql-analysis.yml

@@ -31,7 +31,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v2

.github/workflows/create-branch.yml

@@ -28,12 +28,12 @@ jobs:
 
       - uses: "actions/setup-python@v4"
         with:
-          python-version: "3.8"
+          python-version: "3.11"
 
       - name: "Install python dependencies"
         run: |
           echo ::group::PYDEPS
-          pip install bump2version jinja2 pyyaml
+          pip install bump2version jinja2 pyyaml packaging
           echo ::endgroup::
 
       - name: "Setting secrets"
@@ -72,7 +72,7 @@ jobs:
           find CHANGES -type f -regex ".*\.\(bugfix\|doc\|feature\|misc\|deprecation\|removal\)" -exec git rm {} +
 
       - name: Checkout plugin template
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: pulp/plugin_template
           path: plugin_template
@@ -81,11 +81,11 @@ jobs:
       - name: Update CI branches in template_config
         working-directory: plugin_template
         run: |
-          python3 ./plugin-template pulp_gem --github --add-version-ci-update-branches "${NEW_BRANCH}"
+          python3 ./plugin-template pulp_gem --github --latest-release-branch "${NEW_BRANCH}"
           git add -A
 
       - name: Make a PR with version bump and without CHANGES/*
-        uses: peter-evans/create-pull-request@v4
+        uses: peter-evans/create-pull-request@v6
         with:
           path: pulp_gem
           token: ${{ secrets.RELEASE_TOKEN }}

.github/workflows/kanban.yml

@@ -61,7 +61,7 @@ jobs:
       linked-issues: ${{ steps.linked-issues.outputs.issues }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Get Linked Issues Action
         uses: kin/[email protected]
         id: linked-issues

.github/workflows/lint.yml

@@ -26,7 +26,7 @@ jobs:
 
       - uses: "actions/setup-python@v4"
         with:
-          python-version: "3.8"
+          python-version: "3.11"
 
       - name: "Install python dependencies"
         run: |

.github/workflows/nightly.yml

@@ -45,7 +45,7 @@ jobs:
       - name: "Install python dependencies"
         run: |
           echo ::group::PYDEPS
-          pip install gitpython toml
+          pip install gitpython packaging toml
           echo ::endgroup::
 
       - name: "Configure Git with pulpbot name and email"
@@ -57,7 +57,7 @@ jobs:
         run: python .ci/scripts/collect_changes.py
 
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v4
+        uses: peter-evans/create-pull-request@v6
         with:
           token: ${{ secrets.RELEASE_TOKEN }}
           title: "Update Changelog"
@@ -83,7 +83,7 @@ jobs:
 
       - uses: "actions/setup-python@v4"
         with:
-          python-version: "3.8"
+          python-version: "3.11"
 
       - name: "Install python dependencies"
         run: |