From 6ea9e4507ffe9b80a5c191f3ee9e744d915cab30 Mon Sep 17 00:00:00 2001 From: nodiscc Date: Sun, 23 Jul 2023 02:22:37 +0200 Subject: [PATCH 1/3] finalize automation/CI workflows: - build markdown/HTML versions, use separate makefile targets/hecat configuration files for each export format/push task - add non-free.md generation step to markdown export hecat configuration file - add makefile targets to push exports to https://github.com/awesome-selfhosted/awesome-selfhosted and https://github.com/nodiscc/awesome-selfhosted-html-preview - install: install sphinx <7 directly using pip since it can't be installed from setup.py https://github.com/sphinx-doc/sphinx/issues/11130 - pin sphinx to version <7, sphinx-design is not compatible with sphinx 7 https://github.com/executablebooks/sphinx-design/issues/130 - add sphinx configuration file - tools/makefile: monkeypatch furo theme to display build tools and license link in footer (furo/sphinx does not provide a way to customize this https://github.com/pradyunsg/furo/discussions/612) - fix dependencies between makefile targets - setup/document SSH deploy keys usage - each repository requires a different deploy key, trying to add a deploy key to repo B, which has already been added to repo A results in error 'this key already exists' - set git remote URL to use ssh URIs before push (required to use SSH deploy keys) - when referencing an environment secret, environment: key must be defined explicitly for the step (https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment#referencing-an-environment) - use a single SSH key file/no ssh-agent, else git push will try to use the first registered key and fail with permission denied when trying to push to the HTML repository - use variables to identify target repos - use temporary repositories as push destination --- .github/workflows/build.yml | 27 +++++++++++++- .github/workflows/daily-checks.yml | 2 +- .github/workflows/pull-request.yml | 2 +- .gitignore | 2 ++ .hecat/{export.yml => export-html.yml} | 12 +++---- .hecat/export-markdown.yml | 46 ++++++++++++++++++++++++ Makefile | 45 ++++++++++++++++++----- README.md | 5 ++- conf.py | 49 ++++++++++++++++++++++++++ 9 files changed, 171 insertions(+), 19 deletions(-) rename .hecat/{export.yml => export-html.yml} (51%) create mode 100644 .hecat/export-markdown.yml create mode 100644 conf.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d9e1e0f3a3..eaa540158d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,10 +9,35 @@ concurrency: group: build-${{ github.ref }} cancel-in-progress: true +# This job requires deploy keys to be added to target repositories (https://docs.github.com/en/authentication/connecting-to-github-with-ssh/managing-deploy-keys) +# Generate deploy keys locally: +# $ ssh-keygen -t rsa -f awesome-selfhosted-deploy-key -C awesome-selfhosted-deploy-key +# $ ssh-keygen -t rsa -f awesome-selfhosted-html-deploy-key -C awesome-selfhosted-html-deploy-key +# Paste the contents of awesome-selfhosted-deploy-key.pub to https://github.com/nodiscc/awesome-selfhosted/settings/keys/new, name: awesome-selfhosted-deploy-key, allow write access +# Paste the contents of awesome-selfhosted-html-deploy-key.pub to https://github.com/nodiscc/awesome-selfhosted-html-preview/settings/keys/new, name: awesome-selfhosted-html-deploy-key +# Access https://github.com/awesome-selfhosted/awesome-selfhosted-data/settings/environments, create new environment with name: production +# - deployment branches: selected branches +# - add deployment branch rule: branch name pattern: master +# - environment secrets: add secret with name: SSH_MARKDOWN_DEPLOY_KEY, and value: paste the contents of awesome-selfhosted-deploy-key +# - environment secrets: add secret with name: SSH_HTML_DEPLOY_KEY, and value: paste the contents of awesome-selfhosted-html-deploy-key + jobs: build: + environment: production runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - run: make awesome_lint - - run: make export + - run: make export_markdown export_html + - name: setup markdown repository SSH deploy key + run: | + mkdir -p ~/.ssh + echo "${{ secrets.SSH_MARKDOWN_DEPLOY_KEY }}" > ~/.ssh/id_rsa + chmod 0600 ~/.ssh/id_rsa + - run: make push_markdown + - name: setup HTML repository SSH deploy key + run: | + mkdir -p ~/.ssh + echo "${{ secrets.SSH_HTML_DEPLOY_KEY }}" > ~/.ssh/id_rsa + chmod 0600 ~/.ssh/id_rsa + - run: make push_html diff --git a/.github/workflows/daily-checks.yml b/.github/workflows/daily-checks.yml index 5dbf47929a..e585dddc14 100644 --- a/.github/workflows/daily-checks.yml +++ b/.github/workflows/daily-checks.yml @@ -1,4 +1,4 @@ -name: dead links and unmaintained projects checks +name: dead links/unmaintained projects checks on: schedule: diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 4888bf787b..d5e37f086c 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -14,4 +14,4 @@ jobs: steps: - uses: actions/checkout@v3 - run: make awesome_lint - - run: make export + - run: make export_markdown diff --git a/.gitignore b/.gitignore index f2df0872e1..96c38f6ea5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ awesome-selfhosted/ +awesome-selfhosted-html-preview/ +html/ .venv/ diff --git a/.hecat/export.yml b/.hecat/export-html.yml similarity index 51% rename from .hecat/export.yml rename to .hecat/export-html.yml index faf3e08a84..e42c419eba 100644 --- a/.hecat/export.yml +++ b/.hecat/export-html.yml @@ -1,12 +1,9 @@ steps: - - name: export YAML data to awesome-selfhosted markdown - module: exporters/markdown_singlepage + - name: export YAML data to multi-page markdown site + module: exporters/markdown_multipage module_options: source_directory: ./ - output_directory: awesome-selfhosted - output_file: README.md - back_to_top_url: '#awesome-selfhosted' - # authors_file: AUTHORS.md + output_directory: html exclude_licenses: - '⊘ Proprietary' - 'BUSL-1.1' @@ -16,3 +13,6 @@ steps: - 'Commons-Clause' - 'DPL' - 'SSPL-1.0' + - 'DPL' + - 'Elastic-1.0' + - 'Elastic-2.0' diff --git a/.hecat/export-markdown.yml b/.hecat/export-markdown.yml new file mode 100644 index 0000000000..49d708bbd1 --- /dev/null +++ b/.hecat/export-markdown.yml @@ -0,0 +1,46 @@ +steps: + - name: export YAML data to single-page markdown + module: exporters/markdown_singlepage + module_options: + source_directory: ./ + output_directory: awesome-selfhosted + output_file: README.md + markdown_header: markdown/header.md + markdown_footer: markdown/footer.md + back_to_top_url: '#awesome-selfhosted' + exclude_licenses: + - '⊘ Proprietary' + - 'BUSL-1.1' + - 'CC-BY-NC-4.0' + - 'CC-BY-NC-SA-3.0' + - 'CC-BY-ND-3.0' + - 'Commons-Clause' + - 'DPL' + - 'SSPL-1.0' + - 'DPL' + - 'Elastic-1.0' + - 'Elastic-2.0' + + - name: export YAML data to single-page markdown (non-free) + module: exporters/markdown_singlepage + module_options: + source_directory: ./ + output_directory: awesome-selfhosted + output_file: non-free.md + markdown_header: markdown/non-free-header.md + licenses_file: licenses-nonfree.yml + back_to_top_url: '##awesome-selfhosted---non-free-software' + render_empty_categories: False + render_category_headers: False + include_licenses: + - '⊘ Proprietary' + - 'BUSL-1.1' + - 'CC-BY-NC-4.0' + - 'CC-BY-NC-SA-3.0' + - 'CC-BY-ND-3.0' + - 'Commons-Clause' + - 'DPL' + - 'SSPL-1.0' + - 'DPL' + - 'Elastic-1.0' + - 'Elastic-2.0' diff --git a/Makefile b/Makefile index a09ffab9b4..1ec750da54 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,18 @@ +# requirements: git bash make python3-pip python3-venv SHELL := /bin/bash +MARKDOWN_REPOSITORY = nodiscc/awesome-selfhosted +HTML_REPOSITORY = nodiscc/awesome-selfhosted-html-preview .PHONY: install # install build tools in a virtualenv install: python3 -m venv .venv source .venv/bin/activate && \ pip3 install wheel && \ - pip3 install --force git+https://github.com/nodiscc/hecat.git@master + pip3 install --force git+https://github.com/nodiscc/hecat.git@master 'sphinx<7' .PHONY: import # import data from original list at https://github.com/awesome-selfhosted/awesome-selfhosted -import: install - rm -rf awesome-selfhosted && git clone --depth=1 https://github.com/awesome-selfhosted/awesome-selfhosted +import: clean install + git clone --depth=1 https://github.com/awesome-selfhosted/awesome-selfhosted cp awesome-selfhosted/.github/.mailmap .mailmap cp awesome-selfhosted/AUTHORS AUTHORS rm -rf tags/ software/ platforms/ @@ -27,13 +30,37 @@ awesome_lint: install source .venv/bin/activate && \ hecat --config .hecat/awesome-lint.yml -.PHONY: export # export markdown singlepage document from yaml data -export: install - rm -rf awesome-selfhosted && git clone https://github.com/awesome-selfhosted/awesome-selfhosted - source .venv/bin/activate && \ - hecat --config .hecat/export.yml +.PHONY: export_markdown # render markdown export from YAML data (https://github.com/awesome-selfhosted/awesome-selfhosted) +export_markdown: install + rm -rf awesome-selfhosted/ + git clone https://github.com/$(MARKDOWN_REPOSITORY) + source .venv/bin/activate && hecat --config .hecat/export-markdown.yml cd awesome-selfhosted && git diff --color=always +.PHONY: export_html # render HTML export from YAML data (https://nodiscc.github.io/awesome-selfhosted-html-preview/) +export_html: install + rm -rf awesome-selfhosted-html-preview/ html/ + git clone https://github.com/$(HTML_REPOSITORY) + mkdir html && source .venv/bin/activate && hecat --config .hecat/export-html.yml + sed -i 's|Furo|hecat, sphinx and furo. Content under CC-BY-SA 3.0 license.|' .venv/lib/python*/site-packages/furo/theme/furo/page.html + source .venv/bin/activate && sphinx-build -b html -c ./ html/md/ html/html/ + rm -rf html/html/.buildinfo html/html/objects.inv html/html/.doctrees awesome-selfhosted-html-preview/* + +.PHONY: push_markdown # commit and push changes to the markdown repository +push_markdown: + cd awesome-selfhosted && git remote set-url origin git@github.com:$(MARKDOWN_REPOSITORY) + cd awesome-selfhosted && git config user.name awesome-selfhosted-bot && git config user.email github-actions@github.com + cd awesome-selfhosted && git add . && (git diff-index --quiet HEAD || git commit -m "[bot] build markdown repository") + cd awesome-selfhosted && git push -f + +.PHONY: push_html # commit and push changes to the HTML site repository (amend previous commit and force-push) +push_html: + mv html/html/* awesome-selfhosted-html-preview/ + cd awesome-selfhosted-html-preview/ && git remote set-url origin git@github.com:$(HTML_REPOSITORY) + cd awesome-selfhosted-html-preview/ && git config user.name awesome-selfhosted-bot && git config user.email github-actions@github.com + cd awesome-selfhosted-html-preview/ && git add . && (git diff-index --quiet HEAD || git commit --amend -m "[bot] build HTML site") + cd awesome-selfhosted-html-preview/ && git push -f + .PHONY: url_check # check URLs for dead links or other connection problems url_check: install source .venv/bin/activate && \ @@ -46,7 +73,7 @@ authors: .PHONY: clean # clean files generated by automated tasks clean: - rm -rf awesome-selfhosted + rm -rf awesome-selfhosted/ awesome-selfhosted-html-preview/ html/ .PHONY: help # generate list of targets with descriptions help: diff --git a/README.md b/README.md index e15a14f8bc..4ec053b396 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,10 @@ install install build tools in a virtualenv import import data from original list at https://github.com/awesome-selfhosted/awesome-selfhosted update_metadata update metadata from project repositories/API awesome_lint check data against awesome-selfhosted guidelines -export export markdown singlepage document from yaml data +export_markdown render markdown export from YAML data (https://github.com/awesome-selfhosted/awesome-selfhosted) +export_html render HTML export from YAML data (https://nodiscc.github.io/awesome-selfhosted-html-preview/) +push_markdown commit and push changes to the markdown repository +push_html commit and push changes to the HTML site repository (amend previous commit and force-push) url_check check URLs for dead links or other connection problems authors update the AUTHORS file clean clean files generated by automated tasks diff --git a/conf.py b/conf.py new file mode 100644 index 0000000000..877d25155e --- /dev/null +++ b/conf.py @@ -0,0 +1,49 @@ +# Configuration file for the Sphinx documentation builder. +# https://www.sphinx-doc.org/en/master/usage/configuration.html +project = 'awesome-selfhosted' +author = 'awesome-selfhosted community' +version = '1.0.0' +release = '1.0.0' +copyright = '2015-2023, the awesome-selfhosted community' +language = 'en' +html_title = 'awesome-selfhosted' +html_theme = 'furo' +html_show_sphinx = False +html_show_search_summary = True +html_copy_source = False +html_show_copyright = True +html_use_opensearch = 'https://nodiscc.github.io/awesome-selfhosted-html-preview/' +html_favicon = '_static/favicon.ico' +html_logo = '_static/logo.svg' +extensions = ['myst_parser', 'sphinx_design'] +source_suffix = ['.md'] +templates_path = ['_templates'] +exclude_patterns = [] +html_static_path = ['_static'] + +# myst-parser configuration (https://myst-parser.readthedocs.io/en/latest/configuration.html) +myst_enable_extensions = ['fieldlist'] +myst_html_meta = { + "description lang=en": "A list of Free Software network services and web applications which can be hosted on your own servers", + "charset": "UTF-8" +} + +# theme configuration (https://pradyunsg.me/furo/customisation/) +html_theme_options = { + "top_of_page_button": None, + # "announcement": "Example announcement!" + "source_repository": "https://github.com/awesome-selfhosted/awesome-selfhosted-data", + "source_branch": "master", + "footer_icons": [ + { + "name": "GitHub", + "url": "https://github.com/awesome-selfhosted/awesome-selfhosted-data", + "html": """ + + + + """, + "class": "", + }, + ] +} From ce9f441df62cbfe2a9cf421f21d83232e2bc2a46 Mon Sep 17 00:00:00 2001 From: nodiscc Date: Mon, 24 Jul 2023 16:06:35 +0200 Subject: [PATCH 2/3] tools/makefile: exclude awesome-selfhosted-bot from authors list --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1ec750da54..6e1b72935b 100644 --- a/Makefile +++ b/Makefile @@ -69,7 +69,7 @@ url_check: install .PHONY: authors # update the AUTHORS file authors: printf "Commits|Author\n-------|---------------------------------------------------\n" > AUTHORS - git shortlog -sne >> AUTHORS + git shortlog -sne | grep -v awesome-selfhosted-bot >> AUTHORS .PHONY: clean # clean files generated by automated tasks clean: From a6c26971734f46b3cad349429a81910704326b3f Mon Sep 17 00:00:00 2001 From: nodiscc Date: Mon, 24 Jul 2023 12:55:00 +0200 Subject: [PATCH 3/3] markdown: update header from https://github.com/awesome-selfhosted/awesome-selfhosted --- markdown/header.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/markdown/header.md b/markdown/header.md index 4f6ca83f9f..12b2731056 100644 --- a/markdown/header.md +++ b/markdown/header.md @@ -1,6 +1,6 @@ # Awesome-Selfhosted -[![Awesome](_static/awesome.png)](https://github.com/sindresorhus/awesome) [![](https://github.com/awesome-selfhosted/awesome-selfhosted-data/actions/workflows/ci.yml/badge.svg)](https://github.com/awesome-selfhosted/awesome-selfhosted/issues/3558) +[![Awesome](_static/awesome.png)](https://github.com/sindresorhus/awesome) [![](https://github.com/awesome-selfhosted/awesome-selfhosted-data/actions/workflows/daily-checks.yml/badge.svg)](https://github.com/awesome-selfhosted/awesome-selfhosted/issues/3558) Self-hosting is the practice of hosting and managing applications on your own server(s) instead of consuming from [SaaSS](https://www.gnu.org/philosophy/who-does-that-server-really-serve.html) providers.