From fd580871d8cf3af4d01a579c6dcf70cfe2496a42 Mon Sep 17 00:00:00 2001 From: a-velasco Date: Mon, 5 Aug 2024 21:51:31 +0200 Subject: [PATCH 01/66] First commit --- .github/workflows/_sync_docs_v2.md | 74 ++++++++++ .github/workflows/_sync_docs_v2.yaml | 51 +++++++ .../download_discourse_topics.py | 137 ++++++++++++++++++ 3 files changed, 262 insertions(+) create mode 100644 .github/workflows/_sync_docs_v2.md create mode 100644 .github/workflows/_sync_docs_v2.yaml create mode 100644 python/cli/data_platform_workflows_cli/download_discourse_topics.py diff --git a/.github/workflows/_sync_docs_v2.md b/.github/workflows/_sync_docs_v2.md new file mode 100644 index 00000000..81573083 --- /dev/null +++ b/.github/workflows/_sync_docs_v2.md @@ -0,0 +1,74 @@ +Workflow file: [_sync_docs_v2.yaml](_sync_docs_v2.yaml) + +> [!WARNING] +> Subject to **breaking changes on patch release**. `_sync_docs_v2.yaml` is experimental & not part of the public interface. + +## Usage +Add `.yaml` file to `.github/workflows/` + +```yaml +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. +name: Sync Discourse docs (v2) + +on: + workflow_dispatch: + schedule: + - cron: # Refer to Run schedule below + +jobs: + sync-docs-v2: + name: Sync docs from Discourse (v2) + uses: canonical/data-platform-workflows/.github/workflows/_sync_docs_2.yaml@main + permissions: + contents: write # Needed to push branch & tag + pull-requests: write # Needed to create PR +``` + +## Run schedule + +Cron job schedules for Data Platform repositories + +### SQL +| repository | run time | cron | +|:-------------------------:|:--------:|:-------------:| +| mysql-k8s-operator | 12:00 AM | `00 00 * * *` | +| mysql-operator | 12:10 AM | `10 00 * * *` | +| mysql-test-app | 12:20 AM | `20 00 * * *` | +| mysql-router-k8s-operator | 12:30 AM | `30 00 * * *` | +| mysql-router-operator | 12:40 AM | `40 00 * * *` | +| postgresql-k8s-operator | 12:50 AM | `50 00 * * *` | +| postgresql-operator | 01:00 AM | `00 01 * * *` | +| postgresql-test-app | 01:10 AM | `10 01 * * *` | +| pgbouncer-k8s-operator | 01:20 AM | `20 01 * * *` | +| pgbouncer-operator | 01:30 AM | `30 01 * * *` | + +### NoSQL +| repository | run time | cron | +|:------------------------------:|:--------:|:-------------:| +| mongodb-k8s-operator | 01:40 AM | `40 01 * * *` | +| mongodb-operator | 01:50 AM | `50 01 * * *` | +| mongos-operator | 02:00 AM | `00 02 * * *` | +| opensearch-k8s-operator | 02:10 AM | `10 02 * * *` | +| opensearch-operator | 02:20 AM | `20 02 * * *` | +| opensearch-dashboards-operator | 02:30 AM | `30 02 * * *` | +| redis-k8s-operator | 02:40 AM | `40 02 * * *` | +| redis-operator | 02:50 AM | `50 02 * * *` | + +### Big Data +| repository | run time | cron | +|:---------------------------------:|:--------:|:-------------:| +| kafka-k8s-operator | 03:00 AM | `00 03 * * *` | +| kafka-operator | 03:10 AM | `10 03 * * *` | +| kafka-test-app | 03:20 AM | `20 03 * * *` | +| zookeeper-k8s-operator | 03:30 AM | `30 03 * * *` | +| zookeeper-operator | 03:40 AM | `40 03 * * *` | +| spark-history-server-k8s-operator | 03:50 AM | `50 03 * * *` | +| spark-client-snap | 04:00 AM | `00 04 * * *` | + +### Other +| repository | run time | cron | +|:------------------:|:--------:|:-------------:| +| data-integrator | 04:10 AM | `10 04 * * *` | +| s3-integrator | 04:20 AM | `20 04 * * *` | +| data-platform-libs | 04:30 AM | `30 04 * * *` | diff --git a/.github/workflows/_sync_docs_v2.yaml b/.github/workflows/_sync_docs_v2.yaml new file mode 100644 index 00000000..e45c5182 --- /dev/null +++ b/.github/workflows/_sync_docs_v2.yaml @@ -0,0 +1,51 @@ +on: + workflow_call: + inputs: + reviewers: + description: Comma separated list of GitHub usernames to request to review pull request (e.g. "canonical/data-platform-engineers,octocat") + required: false + type: string + secrets: + token: + description: | + GitHub App token or personal access token (not GITHUB_TOKEN) + + Permissions needed for App token: + - Access: Read & write for Repository permissions: Pull requests + - Access: Read & write for Repository permissions: Contents + - If GitHub team is requested for pull request review, + Access: Read-only for Organization permissions: Members + + Permissions needed for personal access token: write access to repository, read:org + Personal access tokens with fine grained access are not supported (by GraphQL API, which is used by GitHub CLI). + + The GITHUB_TOKEN can create a pull request or push a branch, but `on: pull_request` workflows will not be triggered. + + Source: https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#triggering-further-workflow-runs + required: true + +jobs: + sync-docs-v2: + name: Sync Discourse docs (v2) + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Get workflow version + id: workflow-version + uses: canonical/get-workflow-version-action@v1 + with: + repository-name: canonical/data-platform-workflows + file-name: update_bundle.yaml + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Install CLI + run: pipx install git+https://github.com/canonical/data-platform-workflows@'${{ steps.workflow-version.outputs.sha }}'#subdirectory=python/cli + - name: Checkout + uses: actions/checkout@v4 + with: + token: ${{ secrets.token }} + - name: Download Discourse docs + id: sync-docs-v2 + run: + - name: Push `sync-docs-v2` branch + + - name: Create pull request \ No newline at end of file diff --git a/python/cli/data_platform_workflows_cli/download_discourse_topics.py b/python/cli/data_platform_workflows_cli/download_discourse_topics.py new file mode 100644 index 00000000..67491bbf --- /dev/null +++ b/python/cli/data_platform_workflows_cli/download_discourse_topics.py @@ -0,0 +1,137 @@ +import csv +import dataclasses +import pathlib +import re + +import requests +import yaml + +NAVTABLE_START_MARKER = "[details=Navigation]" +NAVTABLE_END_MARKER = "[/details]" + +def get_topic(topic_id_: str): + """Get markdown content of a discourse.charmhub.io topic""" + + response = requests.get( + f"https://discourse.charmhub.io/raw/{topic_id_}/1" + ) # "/1" for post 1 + + response.raise_for_status() + return response.text + + +class NoTopicToDownload(Exception): + """No Discourse topic is available to download + + Happens if: + - no "Navlink" is provided (e.g. for a navigation group) + - "Navlink" is an external URL + """ + +class NoTableToParse(Exception): + """No markdown navigation table is available + + Happens if: + - Navtable markers do not exist in the topic (i.e. [details=Navigation][/details]) + - The navtable markers in the topic have a typo + - The table is empty + """ + +@dataclasses.dataclass +class Topic: + """Discourse topic to download""" + + id: str + path: pathlib.Path + + @classmethod + def from_csv_row(cls, row_: dict): + # Example `row_`: {'Level': '2', 'Path': 't-overview', 'Navlink': '[Overview](/t/9707)'} + + # Extract Discourse topic ID from "Navlink" + # Example `link`: "/t/9707" + link = re.fullmatch(r"\[.*?]\((.*?)\)", row_["Navlink"]).group(1) + if link == "": + raise NoTopicToDownload + elif link.startswith("http") and "charmhub.io" not in link: + # Ignore external links (e.g. "https://canonical.com/data/docs/postgresql/iaas") + raise NoTopicToDownload + + match = re.fullmatch(r"/t/([0-9]+)", link) + if not match: + raise ValueError( + f'Invalid navlink "{link}". Expected something like "/t/9707"' + ) + # Example `topic_id`: "9707" + topic_id = match.group(1) + + # Determine local path to download Markdown file + # Example `topic_slug`: "t-overview" + topic_slug = row_["Path"] + diataxis_directory = { + "t-": "tutorial", + "h-": "how-to", + "r-": "reference", + "e-": "explanation", + }[topic_slug[:2]] + + # Example `path`: "docs/tutorial/t-overview.md" + path = pathlib.Path("docs/") / diataxis_directory / f"{topic_slug}.md" + + return cls(topic_id, path) + +def main(): + # Example `overview_topic_link`: "https://discourse.charmhub.io/t/charmed-postgresql-documentation/9710" + overview_topic_link: str = yaml.safe_load(pathlib.Path("disc2github/metadata.yaml").read_text())["docs"] + assert overview_topic_link.startswith("https://discourse.charmhub.io/") + + # Example `topic_id`: "9710" + topic_id = overview_topic_link.split("/")[-1] + overview_topic_markdown = get_topic(topic_id) + + # Example: + # | Level | Path | Navlink | + # |--------|--------|-------------| + # | 1 | tutorial | [Tutorial]() | + # | 2 | t-overview | [Overview](/t/9707) | + # | 2 | t-set-up | [1. Set up the environment](/t/9709) | + # | 2 | t-deploy | [2. Deploy PostgreSQL](/t/9697) | + # | 1 | search | [Search](https://canonical.com/data/docs/postgresql/iaas) | + + # Search for table delimiters NAVTABLE_START_MARKER and NAVTABLE_END_MARKER + start_index = overview_topic_markdown.find(NAVTABLE_START_MARKER) + if start_index == -1: + raise NoTableToParse("Could not find Navtable start marker " + NAVTABLE_START_MARKER + " in the overview topic") + + end_index = overview_topic_markdown.find(NAVTABLE_END_MARKER) + if end_index == -1: + raise NoTableToParse("Could not find Navtable end marker " + NAVTABLE_END_MARKER + " in the overview topic") + + start_index += len(NAVTABLE_START_MARKER) + end_index = overview_topic_markdown.find(NAVTABLE_END_MARKER, start_index) + + table_raw = overview_topic_markdown[start_index:end_index].strip() # remove leading and trailing whitespace + if table_raw == "": + raise NoTableToParse + + # Convert Markdown table to list[dict[str, str]] + # (https://stackoverflow.com/a/78254495) + rows: list[dict] = list(csv.DictReader(table_raw.split("\n"), delimiter="|")) + + # Remove first row (e.g. "|--------|--------|-------------|") + rows = rows[2:] + rows: list[dict[str, str]] = [ + {key.strip(): value.strip() for key, value in row.items() if key != ''} + for row in rows + ] + + # Example `row`: {'Level': '2', 'Path': 't-overview', 'Navlink': '[Overview](/t/9707)'} + for row in rows: + try: + topic = Topic.from_csv_row(row) + except NoTopicToDownload: + continue + + # Download topic markdown to `topic.path` + topic.path.parent.mkdir(parents=True, exist_ok=True) + topic.path.write_text(get_topic(topic_id)) \ No newline at end of file From 8821fbcbdd03b340e5f6b8ad06868b9a68ac923d Mon Sep 17 00:00:00 2001 From: a-velasco Date: Wed, 7 Aug 2024 19:51:16 +0200 Subject: [PATCH 02/66] Misc PR review fixes --- .github/workflows/_sync_docs_v2.yaml | 55 ++++++++++++------- README.md | 27 ++++----- ...ad_discourse_topics.py => sync_docs_v2.py} | 31 ++++------- python/cli/pyproject.toml | 1 + 4 files changed, 63 insertions(+), 51 deletions(-) rename python/cli/data_platform_workflows_cli/{download_discourse_topics.py => sync_docs_v2.py} (82%) diff --git a/.github/workflows/_sync_docs_v2.yaml b/.github/workflows/_sync_docs_v2.yaml index e45c5182..80f31695 100644 --- a/.github/workflows/_sync_docs_v2.yaml +++ b/.github/workflows/_sync_docs_v2.yaml @@ -30,22 +30,39 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - name: Get workflow version - id: workflow-version - uses: canonical/get-workflow-version-action@v1 - with: - repository-name: canonical/data-platform-workflows - file-name: update_bundle.yaml - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install CLI - run: pipx install git+https://github.com/canonical/data-platform-workflows@'${{ steps.workflow-version.outputs.sha }}'#subdirectory=python/cli - - name: Checkout - uses: actions/checkout@v4 - with: - token: ${{ secrets.token }} - - name: Download Discourse docs - id: sync-docs-v2 - run: - - name: Push `sync-docs-v2` branch - - - name: Create pull request \ No newline at end of file + - name: Get workflow version + id: workflow-version + uses: canonical/get-workflow-version-action@v1 + with: + repository-name: canonical/data-platform-workflows + file-name: _sync_docs_v2.yaml + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Install CLI + run: pipx install git+https://github.com/canonical/data-platform-workflows@'${{ steps.workflow-version.outputs.sha }}'#subdirectory=python/cli + - name: Checkout + uses: actions/checkout@v4 + with: + token: ${{ secrets.token }} + - name: Download Discourse docs + id: sync-docs-v2 + run: + - name: Push `sync-docs` branch + run: | + git checkout -b sync-docs + git add docs/ + git config user.name "GitHub Actions" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git commit -m "Sync docs from Discourse" + git push origin sync-docs -f + - name: Create pull request + run: | + # Capture output in variable so that step fails if `gh pr list` exits with non-zero code + prs=$(gh pr list --head sync-docs --state open --json number) + if [[ $prs != "[]" ]] + then + echo Open pull request already exists + exit 0 + fi + gh pr create --head sync-docs --title "Sync docs from Discourse" --body "Sync charm docs from https://discourse.charmhub.io" --reviewer '${{ inputs.reviewers }}' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/README.md b/README.md index 098b7739..d690e827 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,19 @@ ## Usage ### Workflows -| Name | Description | -|----------------------------------------------------------------------------|----------------------------------------------------------------------------| -| [lint.yaml](.github/workflows/lint.md) | Lint GitHub Actions workflows (`.github/workflows/`) and `tox run -e lint` | -| [integration_test_charm.yaml](.github/workflows/integration_test_charm.md) | Integration test charm | -| [build_snap.yaml](.github/workflows/build_snap.md) | Build snap | -| [build_rock.yaml](.github/workflows/build_rock.md) | Build rock | -| [build_charm.yaml](.github/workflows/build_charm.md) | Build charm | -| [release_snap.yaml](.github/workflows/release_snap.md) | Release snap to Snap Store | -| [release_rock.yaml](.github/workflows/release_rock.md) | Release rock to GitHub Container Registry | -| [release_charm.yaml](.github/workflows/release_charm.md) | Release charm to Charmhub | -| [update_bundle.yaml](.github/workflows/update_bundle.md) | Update charm revisions in bundle | -| [sync_issue_to_jira.yaml](.github/workflows/sync_issue_to_jira.md) | Sync GitHub issues to Jira issues | -| [_sync_docs.yaml](.github/workflows/_sync_docs.md) | **Experimental** Sync Discourse documentation to GitHub | +| Name | Description | +|----------------------------------------------------------------------------|-----------------------------------------------------------------------------| +| [lint.yaml](.github/workflows/lint.md) | Lint GitHub Actions workflows (`.github/workflows/`) and `tox run -e lint` | +| [integration_test_charm.yaml](.github/workflows/integration_test_charm.md) | Integration test charm | +| [build_snap.yaml](.github/workflows/build_snap.md) | Build snap | +| [build_rock.yaml](.github/workflows/build_rock.md) | Build rock | +| [build_charm.yaml](.github/workflows/build_charm.md) | Build charm | +| [release_snap.yaml](.github/workflows/release_snap.md) | Release snap to Snap Store | +| [release_rock.yaml](.github/workflows/release_rock.md) | Release rock to GitHub Container Registry | +| [release_charm.yaml](.github/workflows/release_charm.md) | Release charm to Charmhub | +| [update_bundle.yaml](.github/workflows/update_bundle.md) | Update charm revisions in bundle | +| [sync_issue_to_jira.yaml](.github/workflows/sync_issue_to_jira.md) | Sync GitHub issues to Jira issues | +| [_sync_docs.yaml](.github/workflows/_sync_docs.md) | **Experimental** Sync Discourse documentation to GitHub (gatekeeper action) | +| [_sync_docs_v2.yaml](.github/workflows/_sync_docs_v2.md) | **Experimental** Sync Discourse documentation to GitHub (custom script) | ### Version Recommendation: pin the latest version (e.g. `v1.0.0`) and use [Renovate](https://docs.renovatebot.com/) to stay up-to-date. diff --git a/python/cli/data_platform_workflows_cli/download_discourse_topics.py b/python/cli/data_platform_workflows_cli/sync_docs_v2.py similarity index 82% rename from python/cli/data_platform_workflows_cli/download_discourse_topics.py rename to python/cli/data_platform_workflows_cli/sync_docs_v2.py index 67491bbf..883c8b0d 100644 --- a/python/cli/data_platform_workflows_cli/download_discourse_topics.py +++ b/python/cli/data_platform_workflows_cli/sync_docs_v2.py @@ -2,7 +2,7 @@ import dataclasses import pathlib import re - +import shutil import requests import yaml @@ -27,15 +27,6 @@ class NoTopicToDownload(Exception): - no "Navlink" is provided (e.g. for a navigation group) - "Navlink" is an external URL """ - -class NoTableToParse(Exception): - """No markdown navigation table is available - - Happens if: - - Navtable markers do not exist in the topic (i.e. [details=Navigation][/details]) - - The navtable markers in the topic have a typo - - The table is empty - """ @dataclasses.dataclass class Topic: @@ -53,7 +44,7 @@ def from_csv_row(cls, row_: dict): link = re.fullmatch(r"\[.*?]\((.*?)\)", row_["Navlink"]).group(1) if link == "": raise NoTopicToDownload - elif link.startswith("http") and "charmhub.io" not in link: + elif link.startswith("http") and "discourse.charmhub.io" not in link: # Ignore external links (e.g. "https://canonical.com/data/docs/postgresql/iaas") raise NoTopicToDownload @@ -81,15 +72,17 @@ def from_csv_row(cls, row_: dict): return cls(topic_id, path) def main(): + """Download Discourse documentation topics to docs/ directory""" + # Example `overview_topic_link`: "https://discourse.charmhub.io/t/charmed-postgresql-documentation/9710" - overview_topic_link: str = yaml.safe_load(pathlib.Path("disc2github/metadata.yaml").read_text())["docs"] + overview_topic_link: str = yaml.safe_load(pathlib.Path("metadata.yaml").read_text())["docs"] assert overview_topic_link.startswith("https://discourse.charmhub.io/") # Example `topic_id`: "9710" topic_id = overview_topic_link.split("/")[-1] overview_topic_markdown = get_topic(topic_id) - # Example: + # Example of an expected markdown table: # | Level | Path | Navlink | # |--------|--------|-------------| # | 1 | tutorial | [Tutorial]() | @@ -101,32 +94,32 @@ def main(): # Search for table delimiters NAVTABLE_START_MARKER and NAVTABLE_END_MARKER start_index = overview_topic_markdown.find(NAVTABLE_START_MARKER) if start_index == -1: - raise NoTableToParse("Could not find Navtable start marker " + NAVTABLE_START_MARKER + " in the overview topic") + raise ValueError("Could not find Navtable start marker " + NAVTABLE_START_MARKER + " in the overview topic") end_index = overview_topic_markdown.find(NAVTABLE_END_MARKER) if end_index == -1: - raise NoTableToParse("Could not find Navtable end marker " + NAVTABLE_END_MARKER + " in the overview topic") + raise ValueError("Could not find Navtable end marker " + NAVTABLE_END_MARKER + " in the overview topic") start_index += len(NAVTABLE_START_MARKER) end_index = overview_topic_markdown.find(NAVTABLE_END_MARKER, start_index) table_raw = overview_topic_markdown[start_index:end_index].strip() # remove leading and trailing whitespace if table_raw == "": - raise NoTableToParse + raise ValueError("Could not find a valid table") # Convert Markdown table to list[dict[str, str]] # (https://stackoverflow.com/a/78254495) rows: list[dict] = list(csv.DictReader(table_raw.split("\n"), delimiter="|")) - # Remove first row (e.g. "|--------|--------|-------------|") rows = rows[2:] rows: list[dict[str, str]] = [ {key.strip(): value.strip() for key, value in row.items() if key != ''} for row in rows ] - - # Example `row`: {'Level': '2', 'Path': 't-overview', 'Navlink': '[Overview](/t/9707)'} + shutil.rmtree(pathlib.Path("docs/")) + for row in rows: + # Example `row`: {'Level': '2', 'Path': 't-overview', 'Navlink': '[Overview](/t/9707)'} try: topic = Topic.from_csv_row(row) except NoTopicToDownload: diff --git a/python/cli/pyproject.toml b/python/cli/pyproject.toml index 86833597..9a17aa86 100644 --- a/python/cli/pyproject.toml +++ b/python/cli/pyproject.toml @@ -21,6 +21,7 @@ parse-snap-version = "data_platform_workflows_cli.parse_snap_version:main" allure-add-default-for-missing-results = "data_platform_workflows_cli.allure_add_default_for_missing_results:main" add-ssh-keys = "data_platform_workflows_cli.add_ssh_keys:main" tee-log-for-all-models = "data_platform_workflows_cli.tee_log_for_all_models:main" +sync-docs-v2 = "data_platform_workflows_cli.sync_docs_v2:main" [tool.poetry.dependencies] python = "^3.10" From 71f818ce49f52f7e5f7853628ab269cf16f6a9d4 Mon Sep 17 00:00:00 2001 From: a-velasco Date: Wed, 7 Aug 2024 19:53:42 +0200 Subject: [PATCH 03/66] Removing run schedule since there are no longer API constraints --- .github/workflows/_sync_docs_v2.md | 50 +----------------------------- 1 file changed, 1 insertion(+), 49 deletions(-) diff --git a/.github/workflows/_sync_docs_v2.md b/.github/workflows/_sync_docs_v2.md index 81573083..b598a4f8 100644 --- a/.github/workflows/_sync_docs_v2.md +++ b/.github/workflows/_sync_docs_v2.md @@ -14,7 +14,7 @@ name: Sync Discourse docs (v2) on: workflow_dispatch: schedule: - - cron: # Refer to Run schedule below + - cron: jobs: sync-docs-v2: @@ -24,51 +24,3 @@ jobs: contents: write # Needed to push branch & tag pull-requests: write # Needed to create PR ``` - -## Run schedule - -Cron job schedules for Data Platform repositories - -### SQL -| repository | run time | cron | -|:-------------------------:|:--------:|:-------------:| -| mysql-k8s-operator | 12:00 AM | `00 00 * * *` | -| mysql-operator | 12:10 AM | `10 00 * * *` | -| mysql-test-app | 12:20 AM | `20 00 * * *` | -| mysql-router-k8s-operator | 12:30 AM | `30 00 * * *` | -| mysql-router-operator | 12:40 AM | `40 00 * * *` | -| postgresql-k8s-operator | 12:50 AM | `50 00 * * *` | -| postgresql-operator | 01:00 AM | `00 01 * * *` | -| postgresql-test-app | 01:10 AM | `10 01 * * *` | -| pgbouncer-k8s-operator | 01:20 AM | `20 01 * * *` | -| pgbouncer-operator | 01:30 AM | `30 01 * * *` | - -### NoSQL -| repository | run time | cron | -|:------------------------------:|:--------:|:-------------:| -| mongodb-k8s-operator | 01:40 AM | `40 01 * * *` | -| mongodb-operator | 01:50 AM | `50 01 * * *` | -| mongos-operator | 02:00 AM | `00 02 * * *` | -| opensearch-k8s-operator | 02:10 AM | `10 02 * * *` | -| opensearch-operator | 02:20 AM | `20 02 * * *` | -| opensearch-dashboards-operator | 02:30 AM | `30 02 * * *` | -| redis-k8s-operator | 02:40 AM | `40 02 * * *` | -| redis-operator | 02:50 AM | `50 02 * * *` | - -### Big Data -| repository | run time | cron | -|:---------------------------------:|:--------:|:-------------:| -| kafka-k8s-operator | 03:00 AM | `00 03 * * *` | -| kafka-operator | 03:10 AM | `10 03 * * *` | -| kafka-test-app | 03:20 AM | `20 03 * * *` | -| zookeeper-k8s-operator | 03:30 AM | `30 03 * * *` | -| zookeeper-operator | 03:40 AM | `40 03 * * *` | -| spark-history-server-k8s-operator | 03:50 AM | `50 03 * * *` | -| spark-client-snap | 04:00 AM | `00 04 * * *` | - -### Other -| repository | run time | cron | -|:------------------:|:--------:|:-------------:| -| data-integrator | 04:10 AM | `10 04 * * *` | -| s3-integrator | 04:20 AM | `20 04 * * *` | -| data-platform-libs | 04:30 AM | `30 04 * * *` | From 330982e50d11a90ffb7d9eb8b19edfbcbd7b9baa Mon Sep 17 00:00:00 2001 From: a-velasco Date: Wed, 7 Aug 2024 20:00:24 +0200 Subject: [PATCH 04/66] Using GITHUB_TOKEN instead of PA TOKEN --- .github/workflows/_sync_docs_v2.yaml | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/.github/workflows/_sync_docs_v2.yaml b/.github/workflows/_sync_docs_v2.yaml index 80f31695..1bdb5ffa 100644 --- a/.github/workflows/_sync_docs_v2.yaml +++ b/.github/workflows/_sync_docs_v2.yaml @@ -5,24 +5,6 @@ on: description: Comma separated list of GitHub usernames to request to review pull request (e.g. "canonical/data-platform-engineers,octocat") required: false type: string - secrets: - token: - description: | - GitHub App token or personal access token (not GITHUB_TOKEN) - - Permissions needed for App token: - - Access: Read & write for Repository permissions: Pull requests - - Access: Read & write for Repository permissions: Contents - - If GitHub team is requested for pull request review, - Access: Read-only for Organization permissions: Members - - Permissions needed for personal access token: write access to repository, read:org - Personal access tokens with fine grained access are not supported (by GraphQL API, which is used by GitHub CLI). - - The GITHUB_TOKEN can create a pull request or push a branch, but `on: pull_request` workflows will not be triggered. - - Source: https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#triggering-further-workflow-runs - required: true jobs: sync-docs-v2: @@ -42,7 +24,7 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - token: ${{ secrets.token }} + token: ${{ secrets.GITHUB_TOKEN }} - name: Download Discourse docs id: sync-docs-v2 run: From a2e04639248a85fb591de9c537f41e44205a79f3 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Thu, 8 Aug 2024 11:22:47 +0200 Subject: [PATCH 05/66] Update .github/workflows/_sync_docs_v2.yaml Co-authored-by: Carl Csaposs --- .github/workflows/_sync_docs_v2.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/_sync_docs_v2.yaml b/.github/workflows/_sync_docs_v2.yaml index 1bdb5ffa..826310bc 100644 --- a/.github/workflows/_sync_docs_v2.yaml +++ b/.github/workflows/_sync_docs_v2.yaml @@ -23,8 +23,6 @@ jobs: run: pipx install git+https://github.com/canonical/data-platform-workflows@'${{ steps.workflow-version.outputs.sha }}'#subdirectory=python/cli - name: Checkout uses: actions/checkout@v4 - with: - token: ${{ secrets.GITHUB_TOKEN }} - name: Download Discourse docs id: sync-docs-v2 run: From 6b8e22710ba763ce4dd9735fc4e50f68a94a463e Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Thu, 8 Aug 2024 11:22:56 +0200 Subject: [PATCH 06/66] Update .github/workflows/_sync_docs_v2.yaml Co-authored-by: Carl Csaposs --- .github/workflows/_sync_docs_v2.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_sync_docs_v2.yaml b/.github/workflows/_sync_docs_v2.yaml index 826310bc..7eadfbd7 100644 --- a/.github/workflows/_sync_docs_v2.yaml +++ b/.github/workflows/_sync_docs_v2.yaml @@ -11,7 +11,7 @@ jobs: name: Sync Discourse docs (v2) runs-on: ubuntu-latest timeout-minutes: 5 - steps: + steps: - name: Get workflow version id: workflow-version uses: canonical/get-workflow-version-action@v1 From 0561751e80a528cac7b7061328333c486766e4b9 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Thu, 8 Aug 2024 11:23:06 +0200 Subject: [PATCH 07/66] Update .github/workflows/_sync_docs_v2.yaml Co-authored-by: Carl Csaposs --- .github/workflows/_sync_docs_v2.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_sync_docs_v2.yaml b/.github/workflows/_sync_docs_v2.yaml index 7eadfbd7..ad34d184 100644 --- a/.github/workflows/_sync_docs_v2.yaml +++ b/.github/workflows/_sync_docs_v2.yaml @@ -45,4 +45,4 @@ jobs: fi gh pr create --head sync-docs --title "Sync docs from Discourse" --body "Sync charm docs from https://discourse.charmhub.io" --reviewer '${{ inputs.reviewers }}' env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From ac42f67a10ade65a49593e6efeb320c9bdfb9329 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Thu, 8 Aug 2024 16:19:55 +0200 Subject: [PATCH 08/66] Update .github/workflows/_sync_docs_v2.md Co-authored-by: Carl Csaposs --- .github/workflows/_sync_docs_v2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_sync_docs_v2.md b/.github/workflows/_sync_docs_v2.md index b598a4f8..42f6ec95 100644 --- a/.github/workflows/_sync_docs_v2.md +++ b/.github/workflows/_sync_docs_v2.md @@ -14,7 +14,7 @@ name: Sync Discourse docs (v2) on: workflow_dispatch: schedule: - - cron: + - cron: '53 0 * * *' # Daily at 00:53 UTC jobs: sync-docs-v2: From ee2866b3aacbad615b9208826d26afd96f543426 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Thu, 8 Aug 2024 16:20:15 +0200 Subject: [PATCH 09/66] Update python/cli/data_platform_workflows_cli/sync_docs_v2.py Co-authored-by: Carl Csaposs --- python/cli/data_platform_workflows_cli/sync_docs_v2.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/cli/data_platform_workflows_cli/sync_docs_v2.py b/python/cli/data_platform_workflows_cli/sync_docs_v2.py index 883c8b0d..90c8a9d9 100644 --- a/python/cli/data_platform_workflows_cli/sync_docs_v2.py +++ b/python/cli/data_platform_workflows_cli/sync_docs_v2.py @@ -116,7 +116,10 @@ def main(): {key.strip(): value.strip() for key, value in row.items() if key != ''} for row in rows ] - shutil.rmtree(pathlib.Path("docs/")) + try: + shutil.rmtree(pathlib.Path("docs/")) + except FileNotFoundError: + pass for row in rows: # Example `row`: {'Level': '2', 'Path': 't-overview', 'Navlink': '[Overview](/t/9707)'} From c92536f5a65cbae3469a9c7bc417a61e71f3fad0 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Thu, 8 Aug 2024 16:20:26 +0200 Subject: [PATCH 10/66] Update python/cli/data_platform_workflows_cli/sync_docs_v2.py Co-authored-by: Carl Csaposs --- .../sync_docs_v2.py | 27 +++++++------------ 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/python/cli/data_platform_workflows_cli/sync_docs_v2.py b/python/cli/data_platform_workflows_cli/sync_docs_v2.py index 90c8a9d9..ea7e2cc9 100644 --- a/python/cli/data_platform_workflows_cli/sync_docs_v2.py +++ b/python/cli/data_platform_workflows_cli/sync_docs_v2.py @@ -82,7 +82,15 @@ def main(): topic_id = overview_topic_link.split("/")[-1] overview_topic_markdown = get_topic(topic_id) - # Example of an expected markdown table: + # Extract navigation table from Markdown + match = re.search( + r"\[details=Navigation]\n(.*?)\n\[/details]", + overview_topic_markdown, + flags=re.DOTALL, + ) + if not match: + raise ValueError("Unable to find navigation table") + # Example `table`: # | Level | Path | Navlink | # |--------|--------|-------------| # | 1 | tutorial | [Tutorial]() | @@ -90,22 +98,7 @@ def main(): # | 2 | t-set-up | [1. Set up the environment](/t/9709) | # | 2 | t-deploy | [2. Deploy PostgreSQL](/t/9697) | # | 1 | search | [Search](https://canonical.com/data/docs/postgresql/iaas) | - - # Search for table delimiters NAVTABLE_START_MARKER and NAVTABLE_END_MARKER - start_index = overview_topic_markdown.find(NAVTABLE_START_MARKER) - if start_index == -1: - raise ValueError("Could not find Navtable start marker " + NAVTABLE_START_MARKER + " in the overview topic") - - end_index = overview_topic_markdown.find(NAVTABLE_END_MARKER) - if end_index == -1: - raise ValueError("Could not find Navtable end marker " + NAVTABLE_END_MARKER + " in the overview topic") - - start_index += len(NAVTABLE_START_MARKER) - end_index = overview_topic_markdown.find(NAVTABLE_END_MARKER, start_index) - - table_raw = overview_topic_markdown[start_index:end_index].strip() # remove leading and trailing whitespace - if table_raw == "": - raise ValueError("Could not find a valid table") + table = match.group(1).strip() # Convert Markdown table to list[dict[str, str]] # (https://stackoverflow.com/a/78254495) From 3c43657175db301ee635c8acffb3e21fae9e5744 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Thu, 8 Aug 2024 16:21:48 +0200 Subject: [PATCH 11/66] Update .github/workflows/_sync_docs_v2.yaml Co-authored-by: Carl Csaposs --- .github/workflows/_sync_docs_v2.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/_sync_docs_v2.yaml b/.github/workflows/_sync_docs_v2.yaml index ad34d184..94b8529d 100644 --- a/.github/workflows/_sync_docs_v2.yaml +++ b/.github/workflows/_sync_docs_v2.yaml @@ -24,7 +24,6 @@ jobs: - name: Checkout uses: actions/checkout@v4 - name: Download Discourse docs - id: sync-docs-v2 run: - name: Push `sync-docs` branch run: | From 64f0a611e69cd310a46956bcf9eb8632276b9593 Mon Sep 17 00:00:00 2001 From: a-velasco Date: Thu, 8 Aug 2024 16:48:58 +0200 Subject: [PATCH 12/66] Small edits --- python/cli/data_platform_workflows_cli/sync_docs_v2.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/python/cli/data_platform_workflows_cli/sync_docs_v2.py b/python/cli/data_platform_workflows_cli/sync_docs_v2.py index ea7e2cc9..0814e824 100644 --- a/python/cli/data_platform_workflows_cli/sync_docs_v2.py +++ b/python/cli/data_platform_workflows_cli/sync_docs_v2.py @@ -9,6 +9,8 @@ NAVTABLE_START_MARKER = "[details=Navigation]" NAVTABLE_END_MARKER = "[/details]" +DOCS_LOCAL_PATH = pathlib.Path("docs/") + def get_topic(topic_id_: str): """Get markdown content of a discourse.charmhub.io topic""" @@ -67,7 +69,7 @@ def from_csv_row(cls, row_: dict): }[topic_slug[:2]] # Example `path`: "docs/tutorial/t-overview.md" - path = pathlib.Path("docs/") / diataxis_directory / f"{topic_slug}.md" + path = DOCS_LOCAL_PATH / diataxis_directory / f"{topic_slug}.md" return cls(topic_id, path) @@ -90,6 +92,7 @@ def main(): ) if not match: raise ValueError("Unable to find navigation table") + # Example `table`: # | Level | Path | Navlink | # |--------|--------|-------------| @@ -102,7 +105,7 @@ def main(): # Convert Markdown table to list[dict[str, str]] # (https://stackoverflow.com/a/78254495) - rows: list[dict] = list(csv.DictReader(table_raw.split("\n"), delimiter="|")) + rows: list[dict] = list(csv.DictReader(table.split("\n"), delimiter="|")) # Remove first row (e.g. "|--------|--------|-------------|") rows = rows[2:] rows: list[dict[str, str]] = [ @@ -110,7 +113,7 @@ def main(): for row in rows ] try: - shutil.rmtree(pathlib.Path("docs/")) + shutil.rmtree(DOCS_LOCAL_PATH) except FileNotFoundError: pass From 17e4327a0cd6174da71c7ff76ca7a4423a5508c5 Mon Sep 17 00:00:00 2001 From: a-velasco Date: Thu, 8 Aug 2024 16:58:56 +0200 Subject: [PATCH 13/66] Replaced original sync_docs with v2 and removed experimental file prefixes --- .github/workflows/_sync_docs.md | 74 ------------------- .github/workflows/_sync_docs.yaml | 31 -------- .github/workflows/_sync_docs_v2.md | 26 ------- .github/workflows/sync_docs.md | 24 ++++++ .../{_sync_docs_v2.yaml => sync_docs.yaml} | 18 +++-- README.md | 3 +- .../{sync_docs_v2.py => sync_docs.py} | 0 python/cli/pyproject.toml | 2 +- 8 files changed, 37 insertions(+), 141 deletions(-) delete mode 100644 .github/workflows/_sync_docs.md delete mode 100644 .github/workflows/_sync_docs.yaml delete mode 100644 .github/workflows/_sync_docs_v2.md create mode 100644 .github/workflows/sync_docs.md rename .github/workflows/{_sync_docs_v2.yaml => sync_docs.yaml} (82%) rename python/cli/data_platform_workflows_cli/{sync_docs_v2.py => sync_docs.py} (100%) diff --git a/.github/workflows/_sync_docs.md b/.github/workflows/_sync_docs.md deleted file mode 100644 index 43593941..00000000 --- a/.github/workflows/_sync_docs.md +++ /dev/null @@ -1,74 +0,0 @@ -Workflow file: [_sync_docs.yaml](_sync_docs.yaml) - -> [!WARNING] -> Subject to **breaking changes on patch release**. `_sync_docs.yaml` is experimental & not part of the public interface. - -## Usage -Add `.yaml` file to `.github/workflows/` - -```yaml -# Copyright 2024 Canonical Ltd. -# See LICENSE file for licensing details. -name: Sync docs from Discourse - -on: - workflow_dispatch: - schedule: - - cron: # Refer to Run schedule below - -jobs: - sync-docs: - name: Sync docs from Discourse - uses: canonical/data-platform-workflows/.github/workflows/_sync_docs.yaml@v0.0.0 - secrets: - discourse-api-user: ${{ secrets.DISCOURSE_API_USERNAME }} - discourse-api-key: ${{ secrets.DISCOURSE_API_KEY }} - permissions: - contents: write # Needed to push branch & tag - pull-requests: write # Needed to create PR -``` - -## Run schedule -### SQL -| repository | run time | cron | -|:-------------------------:|:--------:|:-------------:| -| mysql-k8s-operator | 12:00 AM | `00 00 * * *` | -| mysql-operator | 12:10 AM | `10 00 * * *` | -| mysql-test-app | 12:20 AM | `20 00 * * *` | -| mysql-router-k8s-operator | 12:30 AM | `30 00 * * *` | -| mysql-router-operator | 12:40 AM | `40 00 * * *` | -| postgresql-k8s-operator | 12:50 AM | `50 00 * * *` | -| postgresql-operator | 01:00 AM | `00 01 * * *` | -| postgresql-test-app | 01:10 AM | `10 01 * * *` | -| pgbouncer-k8s-operator | 01:20 AM | `20 01 * * *` | -| pgbouncer-operator | 01:30 AM | `30 01 * * *` | - -### NoSQL -| repository | run time | cron | -|:------------------------------:|:--------:|:-------------:| -| mongodb-k8s-operator | 01:40 AM | `40 01 * * *` | -| mongodb-operator | 01:50 AM | `50 01 * * *` | -| mongos-operator | 02:00 AM | `00 02 * * *` | -| opensearch-k8s-operator | 02:10 AM | `10 02 * * *` | -| opensearch-operator | 02:20 AM | `20 02 * * *` | -| opensearch-dashboards-operator | 02:30 AM | `30 02 * * *` | -| redis-k8s-operator | 02:40 AM | `40 02 * * *` | -| redis-operator | 02:50 AM | `50 02 * * *` | - -### Big Data -| repository | run time | cron | -|:---------------------------------:|:--------:|:-------------:| -| kafka-k8s-operator | 03:00 AM | `00 03 * * *` | -| kafka-operator | 03:10 AM | `10 03 * * *` | -| kafka-test-app | 03:20 AM | `20 03 * * *` | -| zookeeper-k8s-operator | 03:30 AM | `30 03 * * *` | -| zookeeper-operator | 03:40 AM | `40 03 * * *` | -| spark-history-server-k8s-operator | 03:50 AM | `50 03 * * *` | -| spark-client-snap | 04:00 AM | `00 04 * * *` | - -### Other -| repository | run time | cron | -|:------------------:|:--------:|:-------------:| -| data-integrator | 04:10 AM | `10 04 * * *` | -| s3-integrator | 04:20 AM | `20 04 * * *` | -| data-platform-libs | 04:30 AM | `30 04 * * *` | diff --git a/.github/workflows/_sync_docs.yaml b/.github/workflows/_sync_docs.yaml deleted file mode 100644 index 5176b01b..00000000 --- a/.github/workflows/_sync_docs.yaml +++ /dev/null @@ -1,31 +0,0 @@ -on: - workflow_call: - secrets: - discourse-api-user: - description: Discourse API username - required: true - discourse-api-key: - description: Discourse API key - required: true - -jobs: - sync-docs: - name: Sync Discourse docs - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Open PR with changes to Discourse topics - uses: canonical/discourse-gatekeeper@main - id: docs-pr - with: - discourse_host: discourse.charmhub.io - discourse_api_username: ${{ secrets.discourse-api-user }} - discourse_api_key: ${{ secrets.discourse-api-key }} - github_token: ${{ secrets.GITHUB_TOKEN }} - dry_run: "true" - - name: Show migrate output - run: echo '${{ steps.docs-pr.outputs.migrate }}' - - name: Show reconcile output - run: echo '${{ steps.docs-pr.outputs.reconcile }}' diff --git a/.github/workflows/_sync_docs_v2.md b/.github/workflows/_sync_docs_v2.md deleted file mode 100644 index 42f6ec95..00000000 --- a/.github/workflows/_sync_docs_v2.md +++ /dev/null @@ -1,26 +0,0 @@ -Workflow file: [_sync_docs_v2.yaml](_sync_docs_v2.yaml) - -> [!WARNING] -> Subject to **breaking changes on patch release**. `_sync_docs_v2.yaml` is experimental & not part of the public interface. - -## Usage -Add `.yaml` file to `.github/workflows/` - -```yaml -# Copyright 2024 Canonical Ltd. -# See LICENSE file for licensing details. -name: Sync Discourse docs (v2) - -on: - workflow_dispatch: - schedule: - - cron: '53 0 * * *' # Daily at 00:53 UTC - -jobs: - sync-docs-v2: - name: Sync docs from Discourse (v2) - uses: canonical/data-platform-workflows/.github/workflows/_sync_docs_2.yaml@main - permissions: - contents: write # Needed to push branch & tag - pull-requests: write # Needed to create PR -``` diff --git a/.github/workflows/sync_docs.md b/.github/workflows/sync_docs.md new file mode 100644 index 00000000..35ad0a33 --- /dev/null +++ b/.github/workflows/sync_docs.md @@ -0,0 +1,24 @@ +Workflow file: [sync_docs.yaml](sync_docs.yaml) + + +## Usage +Add `.yaml` file to `.github/workflows/` + +```yaml +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. +name: Sync Discourse docs + +on: + workflow_dispatch: + schedule: + - cron: '53 0 * * *' # Daily at 00:53 UTC + +jobs: + sync-docs: + name: Sync docs from Discourse + uses: canonical/data-platform-workflows/.github/workflows/sync_docs.yaml@main + permissions: + contents: write # Needed to push branch & tag + pull-requests: write # Needed to create PR +``` diff --git a/.github/workflows/_sync_docs_v2.yaml b/.github/workflows/sync_docs.yaml similarity index 82% rename from .github/workflows/_sync_docs_v2.yaml rename to .github/workflows/sync_docs.yaml index ad34d184..e6e2d32b 100644 --- a/.github/workflows/_sync_docs_v2.yaml +++ b/.github/workflows/sync_docs.yaml @@ -7,8 +7,8 @@ on: type: string jobs: - sync-docs-v2: - name: Sync Discourse docs (v2) + sync-docs: + name: Sync Discourse docs runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -17,25 +17,29 @@ jobs: uses: canonical/get-workflow-version-action@v1 with: repository-name: canonical/data-platform-workflows - file-name: _sync_docs_v2.yaml + file-name: sync_docs.yaml github-token: ${{ secrets.GITHUB_TOKEN }} - name: Install CLI run: pipx install git+https://github.com/canonical/data-platform-workflows@'${{ steps.workflow-version.outputs.sha }}'#subdirectory=python/cli - name: Checkout uses: actions/checkout@v4 - name: Download Discourse docs - id: sync-docs-v2 run: - - name: Push `sync-docs` branch + - name: Push `sync-docs` branch & create pull request run: | git checkout -b sync-docs git add docs/ + # Check if changes staged + if git diff --cached --quiet + then + echo 'No changes to docs/ from default git branch' + exit 0 + fi git config user.name "GitHub Actions" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git commit -m "Sync docs from Discourse" git push origin sync-docs -f - - name: Create pull request - run: | + # Create pull request # Capture output in variable so that step fails if `gh pr list` exits with non-zero code prs=$(gh pr list --head sync-docs --state open --json number) if [[ $prs != "[]" ]] diff --git a/README.md b/README.md index d690e827..c3f5dcff 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,7 @@ | [release_charm.yaml](.github/workflows/release_charm.md) | Release charm to Charmhub | | [update_bundle.yaml](.github/workflows/update_bundle.md) | Update charm revisions in bundle | | [sync_issue_to_jira.yaml](.github/workflows/sync_issue_to_jira.md) | Sync GitHub issues to Jira issues | -| [_sync_docs.yaml](.github/workflows/_sync_docs.md) | **Experimental** Sync Discourse documentation to GitHub (gatekeeper action) | -| [_sync_docs_v2.yaml](.github/workflows/_sync_docs_v2.md) | **Experimental** Sync Discourse documentation to GitHub (custom script) | +| [sync_docs.yaml](.github/workflows/sync_docs.md) | Sync Discourse documentation to GitHub (custom script) | ### Version Recommendation: pin the latest version (e.g. `v1.0.0`) and use [Renovate](https://docs.renovatebot.com/) to stay up-to-date. diff --git a/python/cli/data_platform_workflows_cli/sync_docs_v2.py b/python/cli/data_platform_workflows_cli/sync_docs.py similarity index 100% rename from python/cli/data_platform_workflows_cli/sync_docs_v2.py rename to python/cli/data_platform_workflows_cli/sync_docs.py diff --git a/python/cli/pyproject.toml b/python/cli/pyproject.toml index 9a17aa86..0904c913 100644 --- a/python/cli/pyproject.toml +++ b/python/cli/pyproject.toml @@ -21,7 +21,7 @@ parse-snap-version = "data_platform_workflows_cli.parse_snap_version:main" allure-add-default-for-missing-results = "data_platform_workflows_cli.allure_add_default_for_missing_results:main" add-ssh-keys = "data_platform_workflows_cli.add_ssh_keys:main" tee-log-for-all-models = "data_platform_workflows_cli.tee_log_for_all_models:main" -sync-docs-v2 = "data_platform_workflows_cli.sync_docs_v2:main" +sync-docs = "data_platform_workflows_cli.sync_docs:main" [tool.poetry.dependencies] python = "^3.10" From bd3c59c70f41f1caff03b28f3eee0a098e4c2ed6 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Thu, 8 Aug 2024 17:01:29 +0200 Subject: [PATCH 14/66] Update sync_docs.md --- .github/workflows/sync_docs.md | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/sync_docs.md b/.github/workflows/sync_docs.md index 35ad0a33..269a876e 100644 --- a/.github/workflows/sync_docs.md +++ b/.github/workflows/sync_docs.md @@ -1,6 +1,5 @@ Workflow file: [sync_docs.yaml](sync_docs.yaml) - ## Usage Add `.yaml` file to `.github/workflows/` From 0d196b59d6a9082bbc59c7cfc0a620b093918d9c Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Thu, 8 Aug 2024 17:02:04 +0200 Subject: [PATCH 15/66] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c3f5dcff..603ee2a2 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ | [release_charm.yaml](.github/workflows/release_charm.md) | Release charm to Charmhub | | [update_bundle.yaml](.github/workflows/update_bundle.md) | Update charm revisions in bundle | | [sync_issue_to_jira.yaml](.github/workflows/sync_issue_to_jira.md) | Sync GitHub issues to Jira issues | -| [sync_docs.yaml](.github/workflows/sync_docs.md) | Sync Discourse documentation to GitHub (custom script) | +| [sync_docs.yaml](.github/workflows/sync_docs.md) | Sync Discourse documentation to GitHub | ### Version Recommendation: pin the latest version (e.g. `v1.0.0`) and use [Renovate](https://docs.renovatebot.com/) to stay up-to-date. @@ -72,4 +72,4 @@ Workflows that begin with one underscore (e.g. `_foo.yaml`) are internal and are Workflows that begin with two underscores (e.g. `__foo.yaml`) are for this repository only. They may only be (triggered by an event on this repository or) called by workflows in this repository that begin with two underscores. ## Contributing -See [CONTRIBUTING.md](CONTRIBUTING.md) \ No newline at end of file +See [CONTRIBUTING.md](CONTRIBUTING.md) From 908c13746243821d006825a926f4b30247c16733 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Thu, 8 Aug 2024 17:03:11 +0200 Subject: [PATCH 16/66] Update sync_docs.yaml For consistent phrasing --- .github/workflows/sync_docs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync_docs.yaml b/.github/workflows/sync_docs.yaml index e6e2d32b..381bc419 100644 --- a/.github/workflows/sync_docs.yaml +++ b/.github/workflows/sync_docs.yaml @@ -8,7 +8,7 @@ on: jobs: sync-docs: - name: Sync Discourse docs + name: Sync docs from Discourse runs-on: ubuntu-latest timeout-minutes: 5 steps: From 67f50a4ade7520b676f64a9c82147bcdd7916c19 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Fri, 9 Aug 2024 11:58:34 +0200 Subject: [PATCH 17/66] Update .github/workflows/sync_docs.md Co-authored-by: Carl Csaposs --- .github/workflows/sync_docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync_docs.md b/.github/workflows/sync_docs.md index 269a876e..b974ebcc 100644 --- a/.github/workflows/sync_docs.md +++ b/.github/workflows/sync_docs.md @@ -16,7 +16,7 @@ on: jobs: sync-docs: name: Sync docs from Discourse - uses: canonical/data-platform-workflows/.github/workflows/sync_docs.yaml@main + uses: canonical/data-platform-workflows/.github/workflows/sync_docs.yaml@v0.0.0 permissions: contents: write # Needed to push branch & tag pull-requests: write # Needed to create PR From ee6932f24bbbbc73e543e268051bc8b9a38a939e Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Fri, 9 Aug 2024 11:59:40 +0200 Subject: [PATCH 18/66] Update python/cli/data_platform_workflows_cli/sync_docs.py Co-authored-by: Carl Csaposs --- python/cli/data_platform_workflows_cli/sync_docs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/cli/data_platform_workflows_cli/sync_docs.py b/python/cli/data_platform_workflows_cli/sync_docs.py index 0814e824..2633910c 100644 --- a/python/cli/data_platform_workflows_cli/sync_docs.py +++ b/python/cli/data_platform_workflows_cli/sync_docs.py @@ -112,6 +112,7 @@ def main(): {key.strip(): value.strip() for key, value in row.items() if key != ''} for row in rows ] + try: shutil.rmtree(DOCS_LOCAL_PATH) except FileNotFoundError: From 5731b971cb7882f4af8aa71f2369d87c08a7e38a Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Fri, 9 Aug 2024 12:00:06 +0200 Subject: [PATCH 19/66] Update .github/workflows/sync_docs.yaml Co-authored-by: Carl Csaposs --- .github/workflows/sync_docs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync_docs.yaml b/.github/workflows/sync_docs.yaml index 381bc419..d2f72f9a 100644 --- a/.github/workflows/sync_docs.yaml +++ b/.github/workflows/sync_docs.yaml @@ -24,7 +24,7 @@ jobs: - name: Checkout uses: actions/checkout@v4 - name: Download Discourse docs - run: + run: sync-docs - name: Push `sync-docs` branch & create pull request run: | git checkout -b sync-docs From 9f46bd4b6fecf331f0071307875933afff150d3f Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Fri, 9 Aug 2024 12:00:42 +0200 Subject: [PATCH 20/66] Update .github/workflows/sync_docs.yaml Co-authored-by: Carl Csaposs --- .github/workflows/sync_docs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync_docs.yaml b/.github/workflows/sync_docs.yaml index d2f72f9a..a1d8f677 100644 --- a/.github/workflows/sync_docs.yaml +++ b/.github/workflows/sync_docs.yaml @@ -23,7 +23,7 @@ jobs: run: pipx install git+https://github.com/canonical/data-platform-workflows@'${{ steps.workflow-version.outputs.sha }}'#subdirectory=python/cli - name: Checkout uses: actions/checkout@v4 - - name: Download Discourse docs + - name: Update Discourse docs run: sync-docs - name: Push `sync-docs` branch & create pull request run: | From d645e0235378eaf74edd24aeec63965637ec1846 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Fri, 9 Aug 2024 12:04:00 +0200 Subject: [PATCH 21/66] Update README.md --- README.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 603ee2a2..a626ba0a 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,18 @@ ## Usage ### Workflows -| Name | Description | -|----------------------------------------------------------------------------|-----------------------------------------------------------------------------| -| [lint.yaml](.github/workflows/lint.md) | Lint GitHub Actions workflows (`.github/workflows/`) and `tox run -e lint` | -| [integration_test_charm.yaml](.github/workflows/integration_test_charm.md) | Integration test charm | -| [build_snap.yaml](.github/workflows/build_snap.md) | Build snap | -| [build_rock.yaml](.github/workflows/build_rock.md) | Build rock | -| [build_charm.yaml](.github/workflows/build_charm.md) | Build charm | -| [release_snap.yaml](.github/workflows/release_snap.md) | Release snap to Snap Store | -| [release_rock.yaml](.github/workflows/release_rock.md) | Release rock to GitHub Container Registry | -| [release_charm.yaml](.github/workflows/release_charm.md) | Release charm to Charmhub | -| [update_bundle.yaml](.github/workflows/update_bundle.md) | Update charm revisions in bundle | -| [sync_issue_to_jira.yaml](.github/workflows/sync_issue_to_jira.md) | Sync GitHub issues to Jira issues | -| [sync_docs.yaml](.github/workflows/sync_docs.md) | Sync Discourse documentation to GitHub | +| Name | Description | +|----------------------------------------------------------------------------|----------------------------------------------------------------------------| +| [lint.yaml](.github/workflows/lint.md) | Lint GitHub Actions workflows (`.github/workflows/`) and `tox run -e lint` | +| [integration_test_charm.yaml](.github/workflows/integration_test_charm.md) | Integration test charm | +| [build_snap.yaml](.github/workflows/build_snap.md) | Build snap | +| [build_rock.yaml](.github/workflows/build_rock.md) | Build rock | +| [build_charm.yaml](.github/workflows/build_charm.md) | Build charm | +| [release_snap.yaml](.github/workflows/release_snap.md) | Release snap to Snap Store | +| [release_rock.yaml](.github/workflows/release_rock.md) | Release rock to GitHub Container Registry | +| [release_charm.yaml](.github/workflows/release_charm.md) | Release charm to Charmhub | +| [update_bundle.yaml](.github/workflows/update_bundle.md) | Update charm revisions in bundle | +| [sync_issue_to_jira.yaml](.github/workflows/sync_issue_to_jira.md) | Sync GitHub issues to Jira issues | +| [sync_docs.yaml](.github/workflows/sync_docs.md) | Sync Discourse documentation to GitHub | ### Version Recommendation: pin the latest version (e.g. `v1.0.0`) and use [Renovate](https://docs.renovatebot.com/) to stay up-to-date. From ca1164a3d3ba09a27a1a5483daba80987247e6eb Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Fri, 9 Aug 2024 12:04:41 +0200 Subject: [PATCH 22/66] Update python/cli/data_platform_workflows_cli/sync_docs.py Co-authored-by: Carl Csaposs --- python/cli/data_platform_workflows_cli/sync_docs.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/python/cli/data_platform_workflows_cli/sync_docs.py b/python/cli/data_platform_workflows_cli/sync_docs.py index 2633910c..51272c03 100644 --- a/python/cli/data_platform_workflows_cli/sync_docs.py +++ b/python/cli/data_platform_workflows_cli/sync_docs.py @@ -6,9 +6,6 @@ import requests import yaml -NAVTABLE_START_MARKER = "[details=Navigation]" -NAVTABLE_END_MARKER = "[/details]" - DOCS_LOCAL_PATH = pathlib.Path("docs/") def get_topic(topic_id_: str): From 64605850479adc8fd14f9227e3a1f8d801d895af Mon Sep 17 00:00:00 2001 From: a-velasco Date: Fri, 9 Aug 2024 13:05:30 +0200 Subject: [PATCH 23/66] Formatted with isort --- python/cli/data_platform_workflows_cli/sync_docs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/cli/data_platform_workflows_cli/sync_docs.py b/python/cli/data_platform_workflows_cli/sync_docs.py index 51272c03..2be1b5fa 100644 --- a/python/cli/data_platform_workflows_cli/sync_docs.py +++ b/python/cli/data_platform_workflows_cli/sync_docs.py @@ -3,6 +3,7 @@ import pathlib import re import shutil + import requests import yaml From d05d5ffc9d4b43d6181cf14b94b23686ec693fb7 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Fri, 9 Aug 2024 13:11:39 +0200 Subject: [PATCH 24/66] Update python/cli/data_platform_workflows_cli/sync_docs.py Co-authored-by: Carl Csaposs --- python/cli/data_platform_workflows_cli/sync_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cli/data_platform_workflows_cli/sync_docs.py b/python/cli/data_platform_workflows_cli/sync_docs.py index 2be1b5fa..01cc6f6a 100644 --- a/python/cli/data_platform_workflows_cli/sync_docs.py +++ b/python/cli/data_platform_workflows_cli/sync_docs.py @@ -125,4 +125,4 @@ def main(): # Download topic markdown to `topic.path` topic.path.parent.mkdir(parents=True, exist_ok=True) - topic.path.write_text(get_topic(topic_id)) \ No newline at end of file + topic.path.write_text(get_topic(topic.id)) \ No newline at end of file From 5316d895cee05fe3a6c11decfa7582574b7d3515 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Fri, 9 Aug 2024 13:38:58 +0200 Subject: [PATCH 25/66] Update sync_docs.md --- .github/workflows/sync_docs.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/.github/workflows/sync_docs.md b/.github/workflows/sync_docs.md index b974ebcc..21ff5af2 100644 --- a/.github/workflows/sync_docs.md +++ b/.github/workflows/sync_docs.md @@ -21,3 +21,35 @@ jobs: contents: write # Needed to push branch & tag pull-requests: write # Needed to create PR ``` + +## Documentation requirements + +Requirement: The overview topic contains a navigation table with the columns "Level", "Path", and "Navlink" + +Requirement: The navigation table of the overview page is wrapped by the `[details]` HTML/markdown element as follows: + +``` +# Navigation + +[details=Navigation] + + + +[/details] + +``` + +> [!NOTE] +> There should be a white space before and after the `[details]` and `[/details]` lines. Otherwise, Discourse/Charmhub may not parse them correctly. + +Requirement: Links in the `Navlink` column are formatted as `[Text](/t/)`. + +They are **not** formatted as `[Text](/t//)`. + +### Examples +The following documentation sets fulfill the above requirements and have been tested: +* [PostgreSQL VM](https://discourse.charmhub.io/t/charmed-postgresql-documentation/9710) +* [PostgreSQL K8s](https://discourse.charmhub.io/t/charmed-postgresql-k8s-documentation/9307) +* [PgBouncer VM](https://discourse.charmhub.io/t/pgbouncer-documentation/12133) +* [PgBouncer K8s](https://discourse.charmhub.io/t/pgbouncer-k8s-documentation/12132) +* [MySQL Router K8s](https://discourse.charmhub.io/t/mysql-router-k8s-documentation/12130) From aa3c9ede25f9ab101d133c6d04584a241e829866 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Fri, 9 Aug 2024 15:04:15 +0200 Subject: [PATCH 26/66] Update sync_docs.md --- .github/workflows/sync_docs.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/sync_docs.md b/.github/workflows/sync_docs.md index 21ff5af2..3207c1cd 100644 --- a/.github/workflows/sync_docs.md +++ b/.github/workflows/sync_docs.md @@ -24,9 +24,9 @@ jobs: ## Documentation requirements -Requirement: The overview topic contains a navigation table with the columns "Level", "Path", and "Navlink" +Requirement: The overview topic must contain a navigation table with the columns "Level", "Path", and "Navlink" -Requirement: The navigation table of the overview page is wrapped by the `[details]` HTML/markdown element as follows: +Requirement: The navigation table of the overview page must be wrapped by the `[details]` HTML/markdown element as follows: ``` # Navigation @@ -42,9 +42,9 @@ Requirement: The navigation table of the overview page is wrapped by the `[detai > [!NOTE] > There should be a white space before and after the `[details]` and `[/details]` lines. Otherwise, Discourse/Charmhub may not parse them correctly. -Requirement: Links in the `Navlink` column are formatted as `[Text](/t/)`. +Requirement: Links in the `Navlink` column must be formatted as `[Text](/t/)`. -They are **not** formatted as `[Text](/t//)`. +They **must not** be formatted as `[Text](/t//)`. ### Examples The following documentation sets fulfill the above requirements and have been tested: From 7a8e20111072ee3aa71814317a232c7c12194f83 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Fri, 9 Aug 2024 15:05:46 +0200 Subject: [PATCH 27/66] Update sync_docs.md --- .github/workflows/sync_docs.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/sync_docs.md b/.github/workflows/sync_docs.md index 3207c1cd..cb54c635 100644 --- a/.github/workflows/sync_docs.md +++ b/.github/workflows/sync_docs.md @@ -26,7 +26,7 @@ jobs: Requirement: The overview topic must contain a navigation table with the columns "Level", "Path", and "Navlink" -Requirement: The navigation table of the overview page must be wrapped by the `[details]` HTML/markdown element as follows: +Requirement: The navigation table of the overview topic must be wrapped by the `[details]` HTML/markdown element as follows: ``` # Navigation @@ -40,11 +40,11 @@ Requirement: The navigation table of the overview page must be wrapped by the `[ ``` > [!NOTE] -> There should be a white space before and after the `[details]` and `[/details]` lines. Otherwise, Discourse/Charmhub may not parse them correctly. +> Make sure to leave a white space before and after the `[details]` and `[/details]` lines. Otherwise, Discourse/Charmhub may not parse them correctly. -Requirement: Links in the `Navlink` column must be formatted as `[Text](/t/)`. +Requirement: Links to topics in the `Navlink` column must be formatted as `[Text](/t/)`. -They **must not** be formatted as `[Text](/t//)`. +They **must not** be formatted as `[Text](/t//)`. ### Examples The following documentation sets fulfill the above requirements and have been tested: From eafdffdc7d1cb1cb2d2a55d39107e6b004c427a2 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Fri, 9 Aug 2024 15:06:50 +0200 Subject: [PATCH 28/66] Update sync_docs.py --- python/cli/data_platform_workflows_cli/sync_docs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cli/data_platform_workflows_cli/sync_docs.py b/python/cli/data_platform_workflows_cli/sync_docs.py index 01cc6f6a..9ba800bc 100644 --- a/python/cli/data_platform_workflows_cli/sync_docs.py +++ b/python/cli/data_platform_workflows_cli/sync_docs.py @@ -72,7 +72,7 @@ def from_csv_row(cls, row_: dict): return cls(topic_id, path) def main(): - """Download Discourse documentation topics to docs/ directory""" + """Update Discourse documentation topics to docs/ directory""" # Example `overview_topic_link`: "https://discourse.charmhub.io/t/charmed-postgresql-documentation/9710" overview_topic_link: str = yaml.safe_load(pathlib.Path("metadata.yaml").read_text())["docs"] @@ -125,4 +125,4 @@ def main(): # Download topic markdown to `topic.path` topic.path.parent.mkdir(parents=True, exist_ok=True) - topic.path.write_text(get_topic(topic.id)) \ No newline at end of file + topic.path.write_text(get_topic(topic.id)) From 917485de411d2c2b814f0e395b484ef5c07b803d Mon Sep 17 00:00:00 2001 From: a-velasco Date: Fri, 9 Aug 2024 15:09:37 +0200 Subject: [PATCH 29/66] Reformatted with black --- .../data_platform_workflows_cli/sync_docs.py | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/python/cli/data_platform_workflows_cli/sync_docs.py b/python/cli/data_platform_workflows_cli/sync_docs.py index 9ba800bc..04b4b73e 100644 --- a/python/cli/data_platform_workflows_cli/sync_docs.py +++ b/python/cli/data_platform_workflows_cli/sync_docs.py @@ -9,13 +9,14 @@ DOCS_LOCAL_PATH = pathlib.Path("docs/") + def get_topic(topic_id_: str): """Get markdown content of a discourse.charmhub.io topic""" - + response = requests.get( f"https://discourse.charmhub.io/raw/{topic_id_}/1" ) # "/1" for post 1 - + response.raise_for_status() return response.text @@ -28,6 +29,7 @@ class NoTopicToDownload(Exception): - "Navlink" is an external URL """ + @dataclasses.dataclass class Topic: """Discourse topic to download""" @@ -71,11 +73,14 @@ def from_csv_row(cls, row_: dict): return cls(topic_id, path) + def main(): """Update Discourse documentation topics to docs/ directory""" - + # Example `overview_topic_link`: "https://discourse.charmhub.io/t/charmed-postgresql-documentation/9710" - overview_topic_link: str = yaml.safe_load(pathlib.Path("metadata.yaml").read_text())["docs"] + overview_topic_link: str = yaml.safe_load( + pathlib.Path("metadata.yaml").read_text() + )["docs"] assert overview_topic_link.startswith("https://discourse.charmhub.io/") # Example `topic_id`: "9710" @@ -90,7 +95,7 @@ def main(): ) if not match: raise ValueError("Unable to find navigation table") - + # Example `table`: # | Level | Path | Navlink | # |--------|--------|-------------| @@ -107,7 +112,7 @@ def main(): # Remove first row (e.g. "|--------|--------|-------------|") rows = rows[2:] rows: list[dict[str, str]] = [ - {key.strip(): value.strip() for key, value in row.items() if key != ''} + {key.strip(): value.strip() for key, value in row.items() if key != ""} for row in rows ] @@ -115,14 +120,14 @@ def main(): shutil.rmtree(DOCS_LOCAL_PATH) except FileNotFoundError: pass - + for row in rows: - # Example `row`: {'Level': '2', 'Path': 't-overview', 'Navlink': '[Overview](/t/9707)'} + # Example `row`: {'Level': '2', 'Path': 't-overview', 'Navlink': '[Overview](/t/9707)'} try: topic = Topic.from_csv_row(row) except NoTopicToDownload: continue - + # Download topic markdown to `topic.path` topic.path.parent.mkdir(parents=True, exist_ok=True) topic.path.write_text(get_topic(topic.id)) From 785629c06ab3217de254717832fd2ebf45295112 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Fri, 9 Aug 2024 17:45:32 +0200 Subject: [PATCH 30/66] Update python/cli/data_platform_workflows_cli/sync_docs.py Co-authored-by: Carl Csaposs --- python/cli/data_platform_workflows_cli/sync_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cli/data_platform_workflows_cli/sync_docs.py b/python/cli/data_platform_workflows_cli/sync_docs.py index 04b4b73e..3f10317e 100644 --- a/python/cli/data_platform_workflows_cli/sync_docs.py +++ b/python/cli/data_platform_workflows_cli/sync_docs.py @@ -75,7 +75,7 @@ def from_csv_row(cls, row_: dict): def main(): - """Update Discourse documentation topics to docs/ directory""" + """Update Discourse documentation topics in docs/ directory""" # Example `overview_topic_link`: "https://discourse.charmhub.io/t/charmed-postgresql-documentation/9710" overview_topic_link: str = yaml.safe_load( From d9d979ad5dc25ddb18929517db1f13c62314a552 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Fri, 9 Aug 2024 17:56:48 +0200 Subject: [PATCH 31/66] Update sync_docs.md --- .github/workflows/sync_docs.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/sync_docs.md b/.github/workflows/sync_docs.md index cb54c635..fc39ad00 100644 --- a/.github/workflows/sync_docs.md +++ b/.github/workflows/sync_docs.md @@ -17,6 +17,8 @@ jobs: sync-docs: name: Sync docs from Discourse uses: canonical/data-platform-workflows/.github/workflows/sync_docs.yaml@v0.0.0 + with: + reviewers: canonical/data-platform-technical-authors,octocat permissions: contents: write # Needed to push branch & tag pull-requests: write # Needed to create PR From e051a0051ac63044cc7cbcb7ccb1deed929b2a5a Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Mon, 12 Aug 2024 11:32:52 +0200 Subject: [PATCH 32/66] Update sync_docs.yaml Delete existing branch and create a new one to account for cases where a change in an existing branch is reverted on Discourse. --- .github/workflows/sync_docs.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/sync_docs.yaml b/.github/workflows/sync_docs.yaml index a1d8f677..32b43df6 100644 --- a/.github/workflows/sync_docs.yaml +++ b/.github/workflows/sync_docs.yaml @@ -33,6 +33,7 @@ jobs: if git diff --cached --quiet then echo 'No changes to docs/ from default git branch' + git push origin --delete sync-docs exit 0 fi git config user.name "GitHub Actions" From 69c58feb651d88d5fb6720d01d980fbf9dd405d8 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Mon, 12 Aug 2024 12:01:55 +0200 Subject: [PATCH 33/66] Update sync_docs.yaml Check branch exists before deleting --- .github/workflows/sync_docs.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sync_docs.yaml b/.github/workflows/sync_docs.yaml index 32b43df6..f1a743ab 100644 --- a/.github/workflows/sync_docs.yaml +++ b/.github/workflows/sync_docs.yaml @@ -33,7 +33,9 @@ jobs: if git diff --cached --quiet then echo 'No changes to docs/ from default git branch' - git push origin --delete sync-docs + if git branch --list sync-docs + then + git push origin --delete sync-docs exit 0 fi git config user.name "GitHub Actions" From e3cd611c908ff1cf8eea45ceb0271f9b0ab53cb9 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Mon, 12 Aug 2024 12:07:33 +0200 Subject: [PATCH 34/66] test --- .github/workflows/sync_docs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync_docs.yaml b/.github/workflows/sync_docs.yaml index f1a743ab..c64dc80d 100644 --- a/.github/workflows/sync_docs.yaml +++ b/.github/workflows/sync_docs.yaml @@ -33,7 +33,7 @@ jobs: if git diff --cached --quiet then echo 'No changes to docs/ from default git branch' - if git branch --list sync-docs + if git branch -r --list sync-docs then git push origin --delete sync-docs exit 0 From 3225cd9261b780c33cbe86c53e6e69bcc450b3c6 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Mon, 12 Aug 2024 12:17:46 +0200 Subject: [PATCH 35/66] Update sync_docs.yaml --- .github/workflows/sync_docs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync_docs.yaml b/.github/workflows/sync_docs.yaml index c64dc80d..8ca3ec34 100644 --- a/.github/workflows/sync_docs.yaml +++ b/.github/workflows/sync_docs.yaml @@ -33,7 +33,7 @@ jobs: if git diff --cached --quiet then echo 'No changes to docs/ from default git branch' - if git branch -r --list sync-docs + if [[ "$(git branch --remotes --list origin/sync-docs)" != "" ]] then git push origin --delete sync-docs exit 0 From c369e5c7577be086c0b8e0f09f321769fc009473 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Mon, 12 Aug 2024 12:29:09 +0200 Subject: [PATCH 36/66] Update sync_docs.yaml --- .github/workflows/sync_docs.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/sync_docs.yaml b/.github/workflows/sync_docs.yaml index 8ca3ec34..81c6805d 100644 --- a/.github/workflows/sync_docs.yaml +++ b/.github/workflows/sync_docs.yaml @@ -36,6 +36,7 @@ jobs: if [[ "$(git branch --remotes --list origin/sync-docs)" != "" ]] then git push origin --delete sync-docs + fi exit 0 fi git config user.name "GitHub Actions" From c7e6000362cba1a0fbe38c78c5c19dec2fa8ca15 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 12 Aug 2024 12:45:09 +0200 Subject: [PATCH 37/66] Add logging --- python/cli/data_platform_workflows_cli/sync_docs.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/cli/data_platform_workflows_cli/sync_docs.py b/python/cli/data_platform_workflows_cli/sync_docs.py index 3f10317e..4015f54d 100644 --- a/python/cli/data_platform_workflows_cli/sync_docs.py +++ b/python/cli/data_platform_workflows_cli/sync_docs.py @@ -1,12 +1,15 @@ import csv import dataclasses +import logging import pathlib import re import shutil +import sys import requests import yaml +logging.basicConfig(level=logging.INFO, stream=sys.stdout) DOCS_LOCAL_PATH = pathlib.Path("docs/") @@ -118,6 +121,7 @@ def main(): try: shutil.rmtree(DOCS_LOCAL_PATH) + logging.info(f"Deleted {DOCS_LOCAL_PATH} directory") except FileNotFoundError: pass @@ -131,3 +135,4 @@ def main(): # Download topic markdown to `topic.path` topic.path.parent.mkdir(parents=True, exist_ok=True) topic.path.write_text(get_topic(topic.id)) + logging.info(f"Downloaded {topic=}") From 46f8dbcac48c2dbc1c0082d381bd6f7854d88118 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Mon, 12 Aug 2024 12:53:54 +0200 Subject: [PATCH 38/66] Fixed bug where first line of navtable was getting automatically filtered out --- python/cli/data_platform_workflows_cli/sync_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cli/data_platform_workflows_cli/sync_docs.py b/python/cli/data_platform_workflows_cli/sync_docs.py index 4015f54d..dbd35d2c 100644 --- a/python/cli/data_platform_workflows_cli/sync_docs.py +++ b/python/cli/data_platform_workflows_cli/sync_docs.py @@ -113,7 +113,7 @@ def main(): # (https://stackoverflow.com/a/78254495) rows: list[dict] = list(csv.DictReader(table.split("\n"), delimiter="|")) # Remove first row (e.g. "|--------|--------|-------------|") - rows = rows[2:] + rows = rows[1:] rows: list[dict[str, str]] = [ {key.strip(): value.strip() for key, value in row.items() if key != ""} for row in rows From 4e33b34d5c9be07d4ff5c22c9981f8edcb08545b Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Mon, 12 Aug 2024 13:12:58 +0200 Subject: [PATCH 39/66] Update .github/workflows/sync_docs.yaml Co-authored-by: Carl Csaposs --- .github/workflows/sync_docs.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/sync_docs.yaml b/.github/workflows/sync_docs.yaml index 81c6805d..0079a466 100644 --- a/.github/workflows/sync_docs.yaml +++ b/.github/workflows/sync_docs.yaml @@ -33,6 +33,8 @@ jobs: if git diff --cached --quiet then echo 'No changes to docs/ from default git branch' + # Delete branch on GitHub if it exists + # (GitHub will automatically close PR if it exists) if [[ "$(git branch --remotes --list origin/sync-docs)" != "" ]] then git push origin --delete sync-docs From 6cf1cdd26af01f753668040f8dfacacadb9d6685 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Mon, 12 Aug 2024 13:14:09 +0200 Subject: [PATCH 40/66] temp (debug) --- .github/workflows/sync_docs.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/sync_docs.yaml b/.github/workflows/sync_docs.yaml index 0079a466..0a839990 100644 --- a/.github/workflows/sync_docs.yaml +++ b/.github/workflows/sync_docs.yaml @@ -35,6 +35,7 @@ jobs: echo 'No changes to docs/ from default git branch' # Delete branch on GitHub if it exists # (GitHub will automatically close PR if it exists) + git branch --remotes --list origin/sync-docs if [[ "$(git branch --remotes --list origin/sync-docs)" != "" ]] then git push origin --delete sync-docs From 7bdf96904fa7d09c237609b2b6a80123b5a47984 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Mon, 12 Aug 2024 13:17:11 +0200 Subject: [PATCH 41/66] temp (debug) --- .github/workflows/sync_docs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync_docs.yaml b/.github/workflows/sync_docs.yaml index 0a839990..2021586e 100644 --- a/.github/workflows/sync_docs.yaml +++ b/.github/workflows/sync_docs.yaml @@ -35,7 +35,7 @@ jobs: echo 'No changes to docs/ from default git branch' # Delete branch on GitHub if it exists # (GitHub will automatically close PR if it exists) - git branch --remotes --list origin/sync-docs + echo "remote branches:\n$(git branch --remotes --list origin/sync-docs)" if [[ "$(git branch --remotes --list origin/sync-docs)" != "" ]] then git push origin --delete sync-docs From 221171152f49be076fc46c094f5662e087006e40 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Mon, 12 Aug 2024 13:19:09 +0200 Subject: [PATCH 42/66] temp (debug) --- .github/workflows/sync_docs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync_docs.yaml b/.github/workflows/sync_docs.yaml index 2021586e..b9588d25 100644 --- a/.github/workflows/sync_docs.yaml +++ b/.github/workflows/sync_docs.yaml @@ -35,7 +35,7 @@ jobs: echo 'No changes to docs/ from default git branch' # Delete branch on GitHub if it exists # (GitHub will automatically close PR if it exists) - echo "remote branches:\n$(git branch --remotes --list origin/sync-docs)" + echo "remote branches:\n $(git branch --remotes --list)" if [[ "$(git branch --remotes --list origin/sync-docs)" != "" ]] then git push origin --delete sync-docs From ac7d9dc5b913f3d4ddc874e6a8ef9a5ee7a5abfa Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Mon, 12 Aug 2024 13:20:15 +0200 Subject: [PATCH 43/66] Update sync_docs.yaml --- .github/workflows/sync_docs.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/sync_docs.yaml b/.github/workflows/sync_docs.yaml index b9588d25..0079a466 100644 --- a/.github/workflows/sync_docs.yaml +++ b/.github/workflows/sync_docs.yaml @@ -35,7 +35,6 @@ jobs: echo 'No changes to docs/ from default git branch' # Delete branch on GitHub if it exists # (GitHub will automatically close PR if it exists) - echo "remote branches:\n $(git branch --remotes --list)" if [[ "$(git branch --remotes --list origin/sync-docs)" != "" ]] then git push origin --delete sync-docs From fd0a2ed5382c028fe17388d2056f087f3328bcf5 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Mon, 12 Aug 2024 13:23:04 +0200 Subject: [PATCH 44/66] Update sync_docs.yaml --- .github/workflows/sync_docs.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/sync_docs.yaml b/.github/workflows/sync_docs.yaml index 0079a466..93e4e28a 100644 --- a/.github/workflows/sync_docs.yaml +++ b/.github/workflows/sync_docs.yaml @@ -35,6 +35,7 @@ jobs: echo 'No changes to docs/ from default git branch' # Delete branch on GitHub if it exists # (GitHub will automatically close PR if it exists) + git fetch origin if [[ "$(git branch --remotes --list origin/sync-docs)" != "" ]] then git push origin --delete sync-docs From ba261b1b51f973ba54a2ed8a336384b68a08d953 Mon Sep 17 00:00:00 2001 From: a-velasco Date: Mon, 12 Aug 2024 13:59:13 +0200 Subject: [PATCH 45/66] Added overview topic download and support for valid non-diataxis topics --- .../data_platform_workflows_cli/sync_docs.py | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/python/cli/data_platform_workflows_cli/sync_docs.py b/python/cli/data_platform_workflows_cli/sync_docs.py index dbd35d2c..5248aec8 100644 --- a/python/cli/data_platform_workflows_cli/sync_docs.py +++ b/python/cli/data_platform_workflows_cli/sync_docs.py @@ -42,7 +42,7 @@ class Topic: @classmethod def from_csv_row(cls, row_: dict): - # Example `row_`: {'Level': '2', 'Path': 't-overview', 'Navlink': '[Overview](/t/9707)'} + # Example `row_`: {'Level': '2', 'Path': 't-introduction', 'Navlink': '[Introduction](/t/9707)'} # Extract Discourse topic ID from "Navlink" # Example `link`: "/t/9707" @@ -62,16 +62,19 @@ def from_csv_row(cls, row_: dict): topic_id = match.group(1) # Determine local path to download Markdown file - # Example `topic_slug`: "t-overview" + # Example `topic_slug`: "t-introduction" topic_slug = row_["Path"] - diataxis_directory = { - "t-": "tutorial", - "h-": "how-to", - "r-": "reference", - "e-": "explanation", - }[topic_slug[:2]] - - # Example `path`: "docs/tutorial/t-overview.md" + try: + diataxis_directory = { + "t-": "tutorial", + "h-": "how-to", + "r-": "reference", + "e-": "explanation", + }[topic_slug[:2]] + except KeyError: + diataxis_directory = "" + + # Example `path`: "docs/tutorial/t-introduction.md" path = DOCS_LOCAL_PATH / diataxis_directory / f"{topic_slug}.md" return cls(topic_id, path) @@ -86,9 +89,9 @@ def main(): )["docs"] assert overview_topic_link.startswith("https://discourse.charmhub.io/") - # Example `topic_id`: "9710" - topic_id = overview_topic_link.split("/")[-1] - overview_topic_markdown = get_topic(topic_id) + # Example `overview_topic_id`: "9710" + overview_topic_id = overview_topic_link.split("/")[-1] + overview_topic_markdown = get_topic(overview_topic_id) # Extract navigation table from Markdown match = re.search( @@ -103,7 +106,7 @@ def main(): # | Level | Path | Navlink | # |--------|--------|-------------| # | 1 | tutorial | [Tutorial]() | - # | 2 | t-overview | [Overview](/t/9707) | + # | 2 | t-introduction | [Introduction](/t/9707) | # | 2 | t-set-up | [1. Set up the environment](/t/9709) | # | 2 | t-deploy | [2. Deploy PostgreSQL](/t/9697) | # | 1 | search | [Search](https://canonical.com/data/docs/postgresql/iaas) | @@ -125,8 +128,11 @@ def main(): except FileNotFoundError: pass + # Manually create and append a row for the overview topic, since its not part of the navtable + rows.append({'Level' : '1', 'Path' : 'overview', 'Navlink' : f"[Overview](/t/{overview_topic_id})"}) + for row in rows: - # Example `row`: {'Level': '2', 'Path': 't-overview', 'Navlink': '[Overview](/t/9707)'} + # Example `row`: {'Level': '2', 'Path': 't-introduction', 'Navlink': '[Introduction](/t/9707)'} try: topic = Topic.from_csv_row(row) except NoTopicToDownload: From 3a81e0cc53866b01ccdf65871444a289dd6dd5fe Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Mon, 12 Aug 2024 14:11:12 +0200 Subject: [PATCH 46/66] Update .github/workflows/sync_docs.yaml Co-authored-by: Carl Csaposs --- .github/workflows/sync_docs.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/sync_docs.yaml b/.github/workflows/sync_docs.yaml index 93e4e28a..7194f1ba 100644 --- a/.github/workflows/sync_docs.yaml +++ b/.github/workflows/sync_docs.yaml @@ -35,8 +35,7 @@ jobs: echo 'No changes to docs/ from default git branch' # Delete branch on GitHub if it exists # (GitHub will automatically close PR if it exists) - git fetch origin - if [[ "$(git branch --remotes --list origin/sync-docs)" != "" ]] + if git ls-remote --exit-code origin sync-docs then git push origin --delete sync-docs fi From f7c2c0373c4babd033f5c809d5bb1156c9eb6b87 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Mon, 12 Aug 2024 14:11:40 +0200 Subject: [PATCH 47/66] Update python/cli/data_platform_workflows_cli/sync_docs.py Co-authored-by: Carl Csaposs --- .../cli/data_platform_workflows_cli/sync_docs.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/python/cli/data_platform_workflows_cli/sync_docs.py b/python/cli/data_platform_workflows_cli/sync_docs.py index 5248aec8..918b20f3 100644 --- a/python/cli/data_platform_workflows_cli/sync_docs.py +++ b/python/cli/data_platform_workflows_cli/sync_docs.py @@ -64,15 +64,12 @@ def from_csv_row(cls, row_: dict): # Determine local path to download Markdown file # Example `topic_slug`: "t-introduction" topic_slug = row_["Path"] - try: - diataxis_directory = { - "t-": "tutorial", - "h-": "how-to", - "r-": "reference", - "e-": "explanation", - }[topic_slug[:2]] - except KeyError: - diataxis_directory = "" + diataxis_directory = { + "t-": "tutorial", + "h-": "how-to", + "r-": "reference", + "e-": "explanation", + }.get(topic_slug[:2], "") # Example `path`: "docs/tutorial/t-introduction.md" path = DOCS_LOCAL_PATH / diataxis_directory / f"{topic_slug}.md" From 2adc0dbd44bbddc28df0072a8b30f5973facfb25 Mon Sep 17 00:00:00 2001 From: a-velasco Date: Mon, 12 Aug 2024 14:46:13 +0200 Subject: [PATCH 48/66] Moved download logic into Topic class --- .../data_platform_workflows_cli/sync_docs.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/python/cli/data_platform_workflows_cli/sync_docs.py b/python/cli/data_platform_workflows_cli/sync_docs.py index 5248aec8..291b480d 100644 --- a/python/cli/data_platform_workflows_cli/sync_docs.py +++ b/python/cli/data_platform_workflows_cli/sync_docs.py @@ -79,6 +79,12 @@ def from_csv_row(cls, row_: dict): return cls(topic_id, path) + def download_to_path(self): + # Download topic markdown to `topic.path` + self.path.parent.mkdir(parents=True, exist_ok=True) + self.path.write_text(get_topic(self.id)) + logging.info(f"Downloaded {self=}") + def main(): """Update Discourse documentation topics in docs/ directory""" @@ -128,9 +134,11 @@ def main(): except FileNotFoundError: pass - # Manually create and append a row for the overview topic, since its not part of the navtable - rows.append({'Level' : '1', 'Path' : 'overview', 'Navlink' : f"[Overview](/t/{overview_topic_id})"}) + # Manually create and download the overview topic, since its not part of the navtable + overview_topic = Topic(overview_topic_id, DOCS_LOCAL_PATH / "overview.md") + overview_topic.download_to_path() + # Convert navtable rows into topics, then download for row in rows: # Example `row`: {'Level': '2', 'Path': 't-introduction', 'Navlink': '[Introduction](/t/9707)'} try: @@ -138,7 +146,4 @@ def main(): except NoTopicToDownload: continue - # Download topic markdown to `topic.path` - topic.path.parent.mkdir(parents=True, exist_ok=True) - topic.path.write_text(get_topic(topic.id)) - logging.info(f"Downloaded {topic=}") + topic.download_to_path() From 663ffc009e4d0a8c69000b4b5fc8975f9f3f7fe9 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Mon, 12 Aug 2024 15:02:51 +0200 Subject: [PATCH 49/66] Update sync_docs.md --- .github/workflows/sync_docs.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/sync_docs.md b/.github/workflows/sync_docs.md index fc39ad00..3f008909 100644 --- a/.github/workflows/sync_docs.md +++ b/.github/workflows/sync_docs.md @@ -24,6 +24,16 @@ jobs: pull-requests: write # Needed to create PR ``` +## Behavior + +Downloads all Discourse topics in a charm's Charmhub documentation to a `/docs` folder in the charm's repository. If the folder doesn't exist, it creates one. + +The topics are determined by the navigation table in the charm's overview page - i.e. the page linked in the `metadata.yaml` `docs:` field. + +If there is a change on Discourse, the workflow will open a PR in the `sync-docs` branch. If a new change is made before the PR is merged, the PR will simply be updated. + +Each Discourse topic is downloaded to either: `/docs`, `/docs/tutorial`, `/docs/how-to`, `/docs/reference`, or `/docs/explanation`. Sub-categories without a topic of their own will not be added as an additional directory. + ## Documentation requirements Requirement: The overview topic must contain a navigation table with the columns "Level", "Path", and "Navlink" From 0bde819bf79b50dc088e845c94f4ed3bb1660884 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Mon, 12 Aug 2024 15:15:37 +0200 Subject: [PATCH 50/66] Update python/cli/data_platform_workflows_cli/sync_docs.py Co-authored-by: Carl Csaposs --- python/cli/data_platform_workflows_cli/sync_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cli/data_platform_workflows_cli/sync_docs.py b/python/cli/data_platform_workflows_cli/sync_docs.py index 001c8ac9..afdd216c 100644 --- a/python/cli/data_platform_workflows_cli/sync_docs.py +++ b/python/cli/data_platform_workflows_cli/sync_docs.py @@ -77,7 +77,7 @@ def from_csv_row(cls, row_: dict): return cls(topic_id, path) def download_to_path(self): - # Download topic markdown to `topic.path` + """Download topic markdown to path""" self.path.parent.mkdir(parents=True, exist_ok=True) self.path.write_text(get_topic(self.id)) logging.info(f"Downloaded {self=}") From 9b77eace6e38a5f6fadc12c7075663e67bda3829 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Mon, 12 Aug 2024 15:15:53 +0200 Subject: [PATCH 51/66] Update python/cli/data_platform_workflows_cli/sync_docs.py Co-authored-by: Carl Csaposs --- python/cli/data_platform_workflows_cli/sync_docs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/cli/data_platform_workflows_cli/sync_docs.py b/python/cli/data_platform_workflows_cli/sync_docs.py index afdd216c..9e494cfa 100644 --- a/python/cli/data_platform_workflows_cli/sync_docs.py +++ b/python/cli/data_platform_workflows_cli/sync_docs.py @@ -132,8 +132,7 @@ def main(): pass # Manually create and download the overview topic, since its not part of the navtable - overview_topic = Topic(overview_topic_id, DOCS_LOCAL_PATH / "overview.md") - overview_topic.download_to_path() + Topic(overview_topic_id, DOCS_LOCAL_PATH / "overview.md").download_to_path() # Convert navtable rows into topics, then download for row in rows: From 53944e40edc8abab19bc4ff479d19222d40938d5 Mon Sep 17 00:00:00 2001 From: a-velasco Date: Mon, 12 Aug 2024 15:36:06 +0200 Subject: [PATCH 52/66] Rename topic download function --- python/cli/data_platform_workflows_cli/sync_docs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/cli/data_platform_workflows_cli/sync_docs.py b/python/cli/data_platform_workflows_cli/sync_docs.py index 9e494cfa..bc01ed12 100644 --- a/python/cli/data_platform_workflows_cli/sync_docs.py +++ b/python/cli/data_platform_workflows_cli/sync_docs.py @@ -76,7 +76,7 @@ def from_csv_row(cls, row_: dict): return cls(topic_id, path) - def download_to_path(self): + def download(self): """Download topic markdown to path""" self.path.parent.mkdir(parents=True, exist_ok=True) self.path.write_text(get_topic(self.id)) @@ -132,7 +132,7 @@ def main(): pass # Manually create and download the overview topic, since its not part of the navtable - Topic(overview_topic_id, DOCS_LOCAL_PATH / "overview.md").download_to_path() + Topic(overview_topic_id, DOCS_LOCAL_PATH / "overview.md").download() # Convert navtable rows into topics, then download for row in rows: @@ -142,4 +142,4 @@ def main(): except NoTopicToDownload: continue - topic.download_to_path() + topic.download() From 80b83476b32f3d7336c1378709f9a5ad14a3a820 Mon Sep 17 00:00:00 2001 From: a-velasco Date: Mon, 12 Aug 2024 15:36:28 +0200 Subject: [PATCH 53/66] Some rephrasing --- .github/workflows/sync_docs.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sync_docs.md b/.github/workflows/sync_docs.md index 3f008909..4b0d871f 100644 --- a/.github/workflows/sync_docs.md +++ b/.github/workflows/sync_docs.md @@ -26,7 +26,10 @@ jobs: ## Behavior -Downloads all Discourse topics in a charm's Charmhub documentation to a `/docs` folder in the charm's repository. If the folder doesn't exist, it creates one. +Downloads all Discourse topics in the charm's Charmhub documentation to `/docs` directory in the charm's repository. + +>[!NOTE] +> Any content in the `/docs` folder that is not part of this workflow will get removed. The topics are determined by the navigation table in the charm's overview page - i.e. the page linked in the `metadata.yaml` `docs:` field. From 9c0d0f22d6be70eb3a1f882a4b72cc65a309e2e4 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Mon, 12 Aug 2024 17:45:00 +0200 Subject: [PATCH 54/66] Update python/cli/data_platform_workflows_cli/sync_docs.py Co-authored-by: Carl Csaposs --- python/cli/data_platform_workflows_cli/sync_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cli/data_platform_workflows_cli/sync_docs.py b/python/cli/data_platform_workflows_cli/sync_docs.py index bc01ed12..8930dc1f 100644 --- a/python/cli/data_platform_workflows_cli/sync_docs.py +++ b/python/cli/data_platform_workflows_cli/sync_docs.py @@ -131,7 +131,7 @@ def main(): except FileNotFoundError: pass - # Manually create and download the overview topic, since its not part of the navtable + # Download the overview topic since it's not part of the navtable Topic(overview_topic_id, DOCS_LOCAL_PATH / "overview.md").download() # Convert navtable rows into topics, then download From c6d1a5f4586bd4e26e9fd8a20c3534ad511ed9dc Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Mon, 12 Aug 2024 17:45:22 +0200 Subject: [PATCH 55/66] Update python/cli/data_platform_workflows_cli/sync_docs.py Co-authored-by: Carl Csaposs --- python/cli/data_platform_workflows_cli/sync_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cli/data_platform_workflows_cli/sync_docs.py b/python/cli/data_platform_workflows_cli/sync_docs.py index 8930dc1f..102c78d6 100644 --- a/python/cli/data_platform_workflows_cli/sync_docs.py +++ b/python/cli/data_platform_workflows_cli/sync_docs.py @@ -134,7 +134,7 @@ def main(): # Download the overview topic since it's not part of the navtable Topic(overview_topic_id, DOCS_LOCAL_PATH / "overview.md").download() - # Convert navtable rows into topics, then download + # Download topics in navtable for row in rows: # Example `row`: {'Level': '2', 'Path': 't-introduction', 'Navlink': '[Introduction](/t/9707)'} try: From fdeaf62e79030a41d7bf6e2c86b22ba4b04df4b9 Mon Sep 17 00:00:00 2001 From: a-velasco Date: Mon, 12 Aug 2024 18:18:21 +0200 Subject: [PATCH 56/66] Update sync_docs.md --- .github/workflows/sync_docs.md | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sync_docs.md b/.github/workflows/sync_docs.md index 4b0d871f..b02998a0 100644 --- a/.github/workflows/sync_docs.md +++ b/.github/workflows/sync_docs.md @@ -33,9 +33,34 @@ Downloads all Discourse topics in the charm's Charmhub documentation to `/docs` The topics are determined by the navigation table in the charm's overview page - i.e. the page linked in the `metadata.yaml` `docs:` field. -If there is a change on Discourse, the workflow will open a PR in the `sync-docs` branch. If a new change is made before the PR is merged, the PR will simply be updated. +When the workflow is triggered, it downloads all Discourse topics in their latest state and compares them to the `/docs` folder in the `main` branch. +* If the contents match, GitHub is up to date with Discourse. Nothing happens. +* If the contents do not match, Discourse is ahead of GitHub. The workflow opens or updates a PR for the `sync-docs` branch with the diff between Discourse and GitHub. -Each Discourse topic is downloaded to either: `/docs`, `/docs/tutorial`, `/docs/how-to`, `/docs/reference`, or `/docs/explanation`. Sub-categories without a topic of their own will not be added as an additional directory. +Each Discourse topic is downloaded to either: `/docs`, `/docs/tutorial`, `/docs/how-to`, `/docs/reference`, or `/docs/explanation` depending on their slug prefix. Sub-categories without a topic of their own will not be added as an additional directory. + +``` +# Example navtable: + +| Level | Path | Navlink | +|-------|----------------|---------------------------| +| 1 | t-landing | [Tutorial](/t/123) | +| 2 | t-introduction | [Introduction](/t/124) | +| 1 | h-landing | [How To]() | +| 2 | h-set-up | [Set up](/t/125) | +| 3 | h-deploy-lxd | [Deploy on LXD](/t/126) | +| 2 | h-monitoring | [Monitoring]() | +| 3 | h-alert | [Add alert rules](/t/127) | + +# Expected /docs output: + +/docs/overview.md +/docs/tutorial/t-landing.md +/docs/tutorial/t-introduction.md +/docs/how-to/h-set-up.md +/docs/how-to/h-deploy-lxd.md +/docs/how-to/h-alert.md +``` ## Documentation requirements From 71981b58df589d7b935e9ce58398189ec0187c9e Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Mon, 12 Aug 2024 18:22:41 +0200 Subject: [PATCH 57/66] Formatting --- .github/workflows/sync_docs.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sync_docs.md b/.github/workflows/sync_docs.md index b02998a0..d9af4bdf 100644 --- a/.github/workflows/sync_docs.md +++ b/.github/workflows/sync_docs.md @@ -64,9 +64,9 @@ Each Discourse topic is downloaded to either: `/docs`, `/docs/tutorial`, `/docs/ ## Documentation requirements -Requirement: The overview topic must contain a navigation table with the columns "Level", "Path", and "Navlink" +**Requirement**: The overview topic must contain a navigation table with the columns "Level", "Path", and "Navlink" -Requirement: The navigation table of the overview topic must be wrapped by the `[details]` HTML/markdown element as follows: +**Requirement**: The navigation table of the overview topic must be wrapped by the `[details]` HTML/markdown element as follows: ``` # Navigation @@ -82,7 +82,7 @@ Requirement: The navigation table of the overview topic must be wrapped by the ` > [!NOTE] > Make sure to leave a white space before and after the `[details]` and `[/details]` lines. Otherwise, Discourse/Charmhub may not parse them correctly. -Requirement: Links to topics in the `Navlink` column must be formatted as `[Text](/t/)`. +**Requirement**: Links to topics in the `Navlink` column must be formatted as `[Text](/t/)`. They **must not** be formatted as `[Text](/t//)`. From 982fce95a24d8425630723b6633de17b57abd70e Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Tue, 13 Aug 2024 09:36:28 +0200 Subject: [PATCH 58/66] Update .github/workflows/sync_docs.md Co-authored-by: Carl Csaposs --- .github/workflows/sync_docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync_docs.md b/.github/workflows/sync_docs.md index d9af4bdf..d25ffadb 100644 --- a/.github/workflows/sync_docs.md +++ b/.github/workflows/sync_docs.md @@ -33,7 +33,7 @@ Downloads all Discourse topics in the charm's Charmhub documentation to `/docs` The topics are determined by the navigation table in the charm's overview page - i.e. the page linked in the `metadata.yaml` `docs:` field. -When the workflow is triggered, it downloads all Discourse topics in their latest state and compares them to the `/docs` folder in the `main` branch. +When the workflow is triggered, it downloads all Discourse topics in their latest state and compares them to the `/docs` folder in the default branch (e.g. `main`). * If the contents match, GitHub is up to date with Discourse. Nothing happens. * If the contents do not match, Discourse is ahead of GitHub. The workflow opens or updates a PR for the `sync-docs` branch with the diff between Discourse and GitHub. From f121d08c2dbd2761914a9179433173be8d7312c8 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Tue, 13 Aug 2024 09:36:47 +0200 Subject: [PATCH 59/66] Update .github/workflows/sync_docs.md Co-authored-by: Carl Csaposs --- .github/workflows/sync_docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync_docs.md b/.github/workflows/sync_docs.md index d25ffadb..6327c8f3 100644 --- a/.github/workflows/sync_docs.md +++ b/.github/workflows/sync_docs.md @@ -26,7 +26,7 @@ jobs: ## Behavior -Downloads all Discourse topics in the charm's Charmhub documentation to `/docs` directory in the charm's repository. +Downloads all Discourse topics in the charm's Charmhub documentation to `/docs/` directory in the charm's repository. >[!NOTE] > Any content in the `/docs` folder that is not part of this workflow will get removed. From d1b71fbc004c30e72d00aafcc012c6a2b53c96ac Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Tue, 13 Aug 2024 09:37:20 +0200 Subject: [PATCH 60/66] Update sync_docs.md --- .github/workflows/sync_docs.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sync_docs.md b/.github/workflows/sync_docs.md index 6327c8f3..b45fb533 100644 --- a/.github/workflows/sync_docs.md +++ b/.github/workflows/sync_docs.md @@ -29,11 +29,11 @@ jobs: Downloads all Discourse topics in the charm's Charmhub documentation to `/docs/` directory in the charm's repository. >[!NOTE] -> Any content in the `/docs` folder that is not part of this workflow will get removed. +> Any content in the `/docs` directory that is not part of this workflow will get removed. The topics are determined by the navigation table in the charm's overview page - i.e. the page linked in the `metadata.yaml` `docs:` field. -When the workflow is triggered, it downloads all Discourse topics in their latest state and compares them to the `/docs` folder in the default branch (e.g. `main`). +When the workflow is triggered, it downloads all Discourse topics in their latest state and compares them to the `/docs` directory in the default branch (e.g. `main`). * If the contents match, GitHub is up to date with Discourse. Nothing happens. * If the contents do not match, Discourse is ahead of GitHub. The workflow opens or updates a PR for the `sync-docs` branch with the diff between Discourse and GitHub. From 46b523821617b145049f03dfb3955d3c67f6a029 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Tue, 13 Aug 2024 10:04:48 +0200 Subject: [PATCH 61/66] Update .github/workflows/sync_docs.md Co-authored-by: Carl Csaposs --- .github/workflows/sync_docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync_docs.md b/.github/workflows/sync_docs.md index b45fb533..117da896 100644 --- a/.github/workflows/sync_docs.md +++ b/.github/workflows/sync_docs.md @@ -29,7 +29,7 @@ jobs: Downloads all Discourse topics in the charm's Charmhub documentation to `/docs/` directory in the charm's repository. >[!NOTE] -> Any content in the `/docs` directory that is not part of this workflow will get removed. +> Any content in the `/docs/` directory that is not part of this workflow will get removed. The topics are determined by the navigation table in the charm's overview page - i.e. the page linked in the `metadata.yaml` `docs:` field. From d33bea54c16f8085072ca5e938f03d8b8c7593d8 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Tue, 13 Aug 2024 10:04:55 +0200 Subject: [PATCH 62/66] Update .github/workflows/sync_docs.md Co-authored-by: Carl Csaposs --- .github/workflows/sync_docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync_docs.md b/.github/workflows/sync_docs.md index 117da896..af181d97 100644 --- a/.github/workflows/sync_docs.md +++ b/.github/workflows/sync_docs.md @@ -37,7 +37,7 @@ When the workflow is triggered, it downloads all Discourse topics in their lates * If the contents match, GitHub is up to date with Discourse. Nothing happens. * If the contents do not match, Discourse is ahead of GitHub. The workflow opens or updates a PR for the `sync-docs` branch with the diff between Discourse and GitHub. -Each Discourse topic is downloaded to either: `/docs`, `/docs/tutorial`, `/docs/how-to`, `/docs/reference`, or `/docs/explanation` depending on their slug prefix. Sub-categories without a topic of their own will not be added as an additional directory. +Each Discourse topic is downloaded to either: `/docs/`, `/docs/tutorial/`, `/docs/how-to/`, `/docs/reference/`, or `/docs/explanation/` depending on their slug prefix. Sub-categories without a topic of their own will not be added as an additional directory. ``` # Example navtable: From dccdd38e013d43d069f2b9352310eeea472078e9 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Tue, 13 Aug 2024 10:05:07 +0200 Subject: [PATCH 63/66] Update .github/workflows/sync_docs.md Co-authored-by: Carl Csaposs --- .github/workflows/sync_docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync_docs.md b/.github/workflows/sync_docs.md index af181d97..0990f00a 100644 --- a/.github/workflows/sync_docs.md +++ b/.github/workflows/sync_docs.md @@ -33,7 +33,7 @@ Downloads all Discourse topics in the charm's Charmhub documentation to `/docs/` The topics are determined by the navigation table in the charm's overview page - i.e. the page linked in the `metadata.yaml` `docs:` field. -When the workflow is triggered, it downloads all Discourse topics in their latest state and compares them to the `/docs` directory in the default branch (e.g. `main`). +When the workflow is triggered, it downloads all Discourse topics in their latest state and compares them to the `/docs/` directory in the default branch (e.g. `main`). * If the contents match, GitHub is up to date with Discourse. Nothing happens. * If the contents do not match, Discourse is ahead of GitHub. The workflow opens or updates a PR for the `sync-docs` branch with the diff between Discourse and GitHub. From d048543c1eb824b9037de1c91b0d4b4f166bcf45 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Tue, 13 Aug 2024 10:06:41 +0200 Subject: [PATCH 64/66] Update sync_docs.md --- .github/workflows/sync_docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync_docs.md b/.github/workflows/sync_docs.md index 0990f00a..7b565223 100644 --- a/.github/workflows/sync_docs.md +++ b/.github/workflows/sync_docs.md @@ -37,7 +37,7 @@ When the workflow is triggered, it downloads all Discourse topics in their lates * If the contents match, GitHub is up to date with Discourse. Nothing happens. * If the contents do not match, Discourse is ahead of GitHub. The workflow opens or updates a PR for the `sync-docs` branch with the diff between Discourse and GitHub. -Each Discourse topic is downloaded to either: `/docs/`, `/docs/tutorial/`, `/docs/how-to/`, `/docs/reference/`, or `/docs/explanation/` depending on their slug prefix. Sub-categories without a topic of their own will not be added as an additional directory. +Each Discourse topic is downloaded to either: `/docs/`, `/docs/tutorial/`, `/docs/how-to/`, `/docs/reference/`, or `/docs/explanation/` depending on their slug prefix. Sub-categories will not add an additional directory. ``` # Example navtable: From e88588927f59f72f75a6700738b935209588c207 Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Tue, 13 Aug 2024 10:07:05 +0200 Subject: [PATCH 65/66] Update sync_docs.md --- .github/workflows/sync_docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync_docs.md b/.github/workflows/sync_docs.md index 7b565223..0335f2f5 100644 --- a/.github/workflows/sync_docs.md +++ b/.github/workflows/sync_docs.md @@ -37,7 +37,7 @@ When the workflow is triggered, it downloads all Discourse topics in their lates * If the contents match, GitHub is up to date with Discourse. Nothing happens. * If the contents do not match, Discourse is ahead of GitHub. The workflow opens or updates a PR for the `sync-docs` branch with the diff between Discourse and GitHub. -Each Discourse topic is downloaded to either: `/docs/`, `/docs/tutorial/`, `/docs/how-to/`, `/docs/reference/`, or `/docs/explanation/` depending on their slug prefix. Sub-categories will not add an additional directory. +Each Discourse topic is downloaded to either: `/docs/`, `/docs/tutorial/`, `/docs/how-to/`, `/docs/reference/`, or `/docs/explanation/` depending on their slug prefix. Sub-categories will not create an additional directory. ``` # Example navtable: From 5a80297f1fd79d4ffa440ff079ddce4b066195dd Mon Sep 17 00:00:00 2001 From: Andreia Velasco Date: Tue, 13 Aug 2024 13:27:42 +0200 Subject: [PATCH 66/66] Fixed bad indentation in yaml template --- .github/workflows/sync_docs.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sync_docs.md b/.github/workflows/sync_docs.md index 0335f2f5..f7a854d8 100644 --- a/.github/workflows/sync_docs.md +++ b/.github/workflows/sync_docs.md @@ -17,8 +17,8 @@ jobs: sync-docs: name: Sync docs from Discourse uses: canonical/data-platform-workflows/.github/workflows/sync_docs.yaml@v0.0.0 - with: - reviewers: canonical/data-platform-technical-authors,octocat + with: + reviewers: canonical/data-platform-technical-authors,octocat permissions: contents: write # Needed to push branch & tag pull-requests: write # Needed to create PR