diff --git a/.github/workflows/build_publish_image_release.yml b/.github/workflows/build_publish_image_release.yml index a2890fcdc..9f676775d 100644 --- a/.github/workflows/build_publish_image_release.yml +++ b/.github/workflows/build_publish_image_release.yml @@ -8,7 +8,9 @@ on: - 'wdqs@*' - 'wdqs-frontend@*' - 'wdqs-proxy@*' - - 'wikibase@*' + - 'wikibase@*' + branches-ignore: + - '**' # weird workaround to trigger on multiple tags for a single commit permissions: contents: write @@ -19,17 +21,18 @@ jobs: runs-on: ubuntu-latest steps: - - name: Extract imageName and confirm it is a valid semantic version tag + - name: Extract image name and confirm it is a valid semantic version tag id: extract_tag run: | + set -x # show commands + # Extract the tag name TAG_NAME=$(echo ${{ github.ref }} | sed 's/refs\/tags\///') # Use regex to extract the imageName before the @ and confirm the version is valid semver version tag if [[ "$TAG_NAME" =~ ^([^@]+)@([0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?)$ ]]; then - # Sets and passes the imageName output from this step, which is used in the subsequent step - # e.g. `steps.extract_tag.outputs.imageName` gets set to "wikibase" matched from "wikibase@1.0.1") - echo "{imageName}=${BASH_REMATCH[1]}" >> $GITHUB_OUTPUT + # Store image name in environment for later steps + echo "IMAGE_NAME=${BASH_REMATCH[1]}" >> $GITHUB_ENV else echo "Tag does not match the pattern {imageName}@{semantic-version}" exit 1 @@ -52,7 +55,4 @@ jobs: set -e # abort on error set -x # show commands - git config --global user.name 'wikibase suite github actions bot' - git config --global user.email 'wikibase-suite-github-actions-bot@users.noreply.github.com' - - ./nx release publish -p ${{ steps.extract_tag.outputs.imageName }} + ./nx release publish -p $IMAGE_NAME diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index b84c58972..f3f32eeef 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -18,7 +18,7 @@ on: - wikibase - All projects (images and deploy) with unreleased changes dry_run: - description: "Only log what will be done, don't do it yet." + description: "Dry run, don't do it yet." type: boolean required: true default: true @@ -33,24 +33,22 @@ jobs: steps: - uses: actions/checkout@v4 + with: + fetch-tags: true + fetch-depth: 0 + token: ${{ secrets.GH_WBS_BOT_TOKEN }} - uses: ./.github/actions/setup-environment with: github_token: ${{ secrets.GITHUB_TOKEN }} - - uses: docker/login-action@v3 - with: - # implicitly docker hub - username: wmdetravisbot - password: ${{ secrets.WBS_PUBLISH_TOKEN }} - - - name: Git tag release + - name: Create release run: | set -e # abort on error set -x # show commands - git config --global user.name 'wikibase suite github actions bot' - git config --global user.email 'wikibase-suite-github-actions-bot@users.noreply.github.com' + git config --global user.name 'wikibase-suite-bot' + git config --global user.email 'wikibase-suite-engineers@wikimedia.de' if [ "${{ inputs.dry_run }}" == "true" ]; then DRY_RUN_FLAG="--dry-run" @@ -64,4 +62,10 @@ jobs: PROJECT_ARG="-p ${{ inputs.image_name }}" fi - ./nx release $PROJECT_ARG $DRY_RUN_FLAG + ./nx release $PROJECT_ARG $DRY_RUN_FLAG --skip-publish --verbose + + # Temporary workaround for nx issue: https://github.com/nrwl/nx/issues/22073# + if [ $? -eq 0 ] && [ -z "$DRY_RUN_FLAG" ]; then + git push origin main --tags + fi + diff --git a/README.md b/README.md index 8c9d1731d..25ade4ee8 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,12 @@ $ ./nx test -- repo # Runs the `repo` test suite with a specific spec file (paths to spec files are rooted in the `test` directory) $ ./nx test -- repo --spec specs/repo/special-item.ts +# Start with a headed browser +$ ./nx test -- repo --headed + +# Start a specific spec only +./nx test -- repo --spec specs/repo/queryservice.ts + # Start and leave up the test environment for a given test suite without running tests $ ./nx test -- repo --setup ``` diff --git a/build/build.sh b/build/build.sh index 5df0686a7..3ca79ac4f 100755 --- a/build/build.sh +++ b/build/build.sh @@ -76,7 +76,7 @@ IMAGE_NAME=$(jq -r '.name' package.json) # publish to Dockerhub if [ "$PUBLISH" == true ]; then - IMAGE_REGISTRY=dockerhub.io + # IMAGE_REGISTRY implies dockerhub if empty IMAGE_NAMESPACE=wikibase # build/test in CI elif [ "$GITHUB_ACTIONS" == true ]; then diff --git a/build/elasticsearch/CHANGELOG.md b/build/elasticsearch/CHANGELOG.md new file mode 100644 index 000000000..0e189b6b1 --- /dev/null +++ b/build/elasticsearch/CHANGELOG.md @@ -0,0 +1,5 @@ +## **elasticsearch@1.0.1** (2024-10-09) + +### 📖 Documentation + +- Switch from `.example.com` to `.example` diff --git a/build/elasticsearch/package.json b/build/elasticsearch/package.json index 0fbd46466..01bc9cabf 100644 --- a/build/elasticsearch/package.json +++ b/build/elasticsearch/package.json @@ -1,6 +1,6 @@ { "name": "elasticsearch", - "version": "1.0.0", + "version": "1.0.1", "nx": { "targets": { "lint": {}, diff --git a/build/quickstatements/CHANGELOG.md b/build/quickstatements/CHANGELOG.md new file mode 100644 index 000000000..285580764 --- /dev/null +++ b/build/quickstatements/CHANGELOG.md @@ -0,0 +1,5 @@ +## **quickstatements@1.0.1** (2024-10-09) + +### 📖 Documentation + +- Switch from `.example.com` to `.example` diff --git a/build/quickstatements/package.json b/build/quickstatements/package.json index 6d6dd3c38..3157cd5e6 100644 --- a/build/quickstatements/package.json +++ b/build/quickstatements/package.json @@ -1,6 +1,6 @@ { "name": "quickstatements", - "version": "1.0.0", + "version": "1.0.1", "nx": { "targets": { "lint": {}, diff --git a/build/wdqs-frontend/CHANGELOG.md b/build/wdqs-frontend/CHANGELOG.md new file mode 100644 index 000000000..1be944c36 --- /dev/null +++ b/build/wdqs-frontend/CHANGELOG.md @@ -0,0 +1,9 @@ +## **wdqs-frontend@1.0.1** (2024-10-09) + +### 📖 Documentation + +- Switch from `.example.com` to `.example` + +### 🩹 Fixes + +- Link to Query Builder now points to the source code repository ([#664](https://github.com/wmde/wikibase-release-pipeline/issues/664)) diff --git a/build/wdqs-frontend/README.md b/build/wdqs-frontend/README.md index 086b52d82..cd933d0da 100644 --- a/build/wdqs-frontend/README.md +++ b/build/wdqs-frontend/README.md @@ -2,6 +2,14 @@ Frontend for the [Wikidata Query Service (WDQS)](https://www.mediawiki.org/wiki/Wikidata_Query_Service). +To interact with the WDQS frontend, navigate to the URL corresponding to the port allocated for it. In the example below, the WDQS frontend is available at `http://localhost:8834`. + +When writing queries using the frontend interface, click "Code" to view the corresponding URL. + +For general instructions on using WDQS, building SPARQL queries, and additional resources, see: +- [Wikidata Query Service User Manual](https://www.mediawiki.org/wiki/Wikidata_Query_Service/User_Manual) +- [What is SPARQL](https://www.wikidata.org/wiki/Wikidata:SPARQL_query_service) + > 💡 This image is part of Wikibase Suite (WBS). [WBS Deploy](https://github.com/wmde/wikibase-release-pipeline/deploy/README.md) provides everything you need to self-host a Wikibase instance out of the box. ## Requirements @@ -21,11 +29,11 @@ We suggest using the [WBS Wikibase image](https://hub.docker.com/r/wikibase/wiki ### WDQS as server -We suggest using the [WBS Wikibase image](https://hub.docker.com/r/wikibase/wdqs). +We suggest using the [WBS WDQS image](https://hub.docker.com/r/wikibase/wdqs). ### WDQS as updater -We suggest using the [WBS Wikibase image](https://hub.docker.com/r/wikibase/wdqs), the same as used for WDQS server. Check out the [documentation](https://wikitech.wikimedia.org/wiki/Wikidata_Query_Service) to learn how to run it in updater mode. +We suggest using the [WBS WDQS image](https://hub.docker.com/r/wikibase/wdqs), the same as used for WDQS server. Check out the [documentation](https://wikitech.wikimedia.org/wiki/Wikidata_Query_Service) to learn how to run it in updater mode. ### Reverse proxy @@ -185,7 +193,7 @@ volumes: Official releases of this image can be found on [Docker Hub wikibase/wdqs-frontend](https://hub.docker.com/r/wikibase/wdqs-frontend). -## Tags and Versioning +## Tags and versioning This WDQS Frontend image is using [semantic versioning](https://semver.org/spec/v2.0.0.html). @@ -206,4 +214,4 @@ This image is built from this [Dockerfile](https://github.com/wmde/wikibase-rele This image is maintained by the Wikibase Suite Team at [Wikimedia Germany (WMDE)](https://wikimedia.de). -If you have questions not listed above or need help, use this [bug report form](https://phabricator.wikimedia.org/maniphest/task/edit/form/129/) to start a conversation with the engineering team. +If you have questions not covered above or need further help, use this [bug report form](https://phabricator.wikimedia.org/maniphest/task/edit/form/129/) to start a conversation with the engineering team. diff --git a/build/wdqs-frontend/custom-config.json b/build/wdqs-frontend/custom-config.json index 978069f73..bff91daf3 100644 --- a/build/wdqs-frontend/custom-config.json +++ b/build/wdqs-frontend/custom-config.json @@ -12,7 +12,10 @@ "pageTitle": "Wikidata:SPARQL_query_service/queries/examples", "pagePathElement": "wiki/" }, - "urlShortener": "tinyurl" + "urlShortener": "tinyurl", + "query-builder": { + "server": "https://gerrit.wikimedia.org/g/wikidata/query-builder" + } }, "brand": { "title": "$BRAND_TITLE", diff --git a/build/wdqs-frontend/package.json b/build/wdqs-frontend/package.json index 7c856a4cd..5eacbb164 100644 --- a/build/wdqs-frontend/package.json +++ b/build/wdqs-frontend/package.json @@ -1,6 +1,6 @@ { "name": "wdqs-frontend", - "version": "1.0.0", + "version": "1.0.1", "nx": { "targets": { "lint": {}, diff --git a/build/wdqs-proxy/CHANGELOG.md b/build/wdqs-proxy/CHANGELOG.md new file mode 100644 index 000000000..b2feb6e26 --- /dev/null +++ b/build/wdqs-proxy/CHANGELOG.md @@ -0,0 +1,5 @@ +## **wdqs-proxy@1.0.1** (2024-10-09) + +### 📖 Documentation + +- Switch from `.example.com` to `.example` diff --git a/build/wdqs-proxy/package.json b/build/wdqs-proxy/package.json index ffce24b6d..9e8bf48f4 100644 --- a/build/wdqs-proxy/package.json +++ b/build/wdqs-proxy/package.json @@ -1,6 +1,6 @@ { "name": "wdqs-proxy", - "version": "1.0.0", + "version": "1.0.1", "nx": { "targets": { "lint": {}, diff --git a/build/wdqs/CHANGELOG.md b/build/wdqs/CHANGELOG.md new file mode 100644 index 000000000..1dac2c8a8 --- /dev/null +++ b/build/wdqs/CHANGELOG.md @@ -0,0 +1,7 @@ +## **wdqs@1.0.1** (2024-10-09) + +### 📖 Documentation + +- Switch from `.example.com` to `.example` +- Add comments in "Updater keeps restarting" FAQ entry +- Correct typos diff --git a/build/wdqs/README.md b/build/wdqs/README.md index 92a964492..0c62475eb 100644 --- a/build/wdqs/README.md +++ b/build/wdqs/README.md @@ -23,6 +23,9 @@ We suggest using the [WBS Wikibase image](https://hub.docker.com/r/wikibase/wiki You'll need one instance of the image to execute the actual WDQS daemon started using `/runBlazegraph.sh`. +You can send `GET` requests with your SPARQL query to the WDQS endpoint (following the example below): +`http://wdqs:9999/bigdata/namespace/wdq/sparql?query={SPARQL}` + ### WDQS as updater You'll need one instance of the image to execute the updater started using `/runUpdate.sh`. This polls changes from Wikibase. diff --git a/build/wdqs/package.json b/build/wdqs/package.json index bd178fad7..da8047b21 100644 --- a/build/wdqs/package.json +++ b/build/wdqs/package.json @@ -1,6 +1,6 @@ { "name": "wdqs", - "version": "1.0.0", + "version": "1.0.1", "nx": { "targets": { "lint": {}, diff --git a/build/wikibase/CHANGELOG.md b/build/wikibase/CHANGELOG.md new file mode 100644 index 000000000..bc7146b9a --- /dev/null +++ b/build/wikibase/CHANGELOG.md @@ -0,0 +1,11 @@ +## **wikibase@3.0.1** (2024-10-09) + +### 📖 Documentation + +- Link to MediaWiki bundled extensions +- Switch from `.example.com` to `.example` + +### 🏡 Chore + +- Removes re-installation of extensions already packaged with MediaWiki (ConfirmEdit, Nuke, Scribunto, SyntaxHighlight_Geshi, VisualEditor) +- Updates Wikibase EDTF extension diff --git a/build/wikibase/package.json b/build/wikibase/package.json index 496db5f27..fe09e8095 100644 --- a/build/wikibase/package.json +++ b/build/wikibase/package.json @@ -1,6 +1,6 @@ { "name": "wikibase", - "version": "3.0.0", + "version": "3.0.1", "nx": { "targets": { "lint": {}, diff --git a/deploy/CHANGELOG.md b/deploy/CHANGELOG.md new file mode 100644 index 000000000..6ab1ae47e --- /dev/null +++ b/deploy/CHANGELOG.md @@ -0,0 +1,17 @@ +## **deploy@3.0.1** (2024-10-09) + +### 🩹 Fixes + +- `*_PUBLIC_HOST` URLs default to `.example` TLD from `example.com` ([T372455](https://phabricator.wikimedia.org/T372455)) +- Removes published ports for wikibase, quickstatements, and wdqs-frontend services ([T372455](https://phabricator.wikimedia.org/T372455)) +- Wikibase waits for elasticsearch service to be up ([T371162](https://phabricator.wikimedia.org/T371162)) + +### 📖 Documentation + +- Do not use `--wait` in `docker compose up` in order to see logs +- Link to WDQS updater crash FAQ entry +- Corrected typos + +### 🏡 Chore + +- Adds default Traefik LetsEncrypt configuration for services diff --git a/deploy/README.md b/deploy/README.md index 2ec862a22..f23f998df 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -146,12 +146,13 @@ To back up your data, shut down the instance and dump the contents of all Docker docker compose down for v in \ - wikibase-suite-wikibase_image-data \ - wikibase-suite_mysql-data \ - wikibase-suite_wdqs-data \ - wikibase-suite_elasticsearch-data \ - wikibase-suite_quickstatements-data \ - wikibase-suite_traefik-letsencrypt-data; do + wbs-deploy_elasticsearch-data \ + wbs-deploy_mysql-data \ + wbs-deploy_quickstatements-data \ + wbs-deploy_traefik-letsencrypt-data \ + wbs-deploy_wdqs-data \ + wbs-deploy_wikibase-image-data \ + ; do docker run --rm --volume $v:/backup debian:12-slim tar cz backup > $v.tar.gz done ``` @@ -164,12 +165,13 @@ To restore the volume backups, ensure your instance has been shut down by runnin docker compose down for v in \ - wikibase-suite-wikibase_image-data \ - wikibase-suite_mysql-data \ - wikibase-suite_wdqs-data \ - wikibase-suite_elasticsearch-data \ - wikibase-suite_quickstatements-data \ - wikibase-suite_traefik-letsencrypt-data; do + wbs-deploy_elasticsearch-data \ + wbs-deploy_mysql-data \ + wbs-deploy_quickstatements-data \ + wbs-deploy_traefik-letsencrypt-data \ + wbs-deploy_wdqs-data \ + wbs-deploy_wikibase-image-data \ + ; do docker volume rm $v 2> /dev/null docker volume create $v docker run -i --rm --volume $v:/backup debian:12-slim tar xz < $v.tar.gz @@ -303,6 +305,15 @@ Removing the `traefik-letsencrypt-data` volume will request a new certificate fr ```yml --certificatesresolvers.letsencrypt.acme.caserver=https://acme-staging-v02.api.letsencrypt.org/directory ``` + +## WDQS Frontend + +To interact with the WDQS frontend, navigate to the URL defined as `WDQS_FRONTEND_PUBLIC_HOST` in the `.env` file. By default, this is set to `wdqs-frontend.example`. + +Alternatively, send `GET` requests with your SPARQL query to the WDQS frontend endpoint: +`https://wdqs-frontend.example.com/proxy/wdqs/bigdata/namespace/wdq/sparql?query={SPARQL}` + + ## FAQ ### Can I host WBS Deploy locally? @@ -324,7 +335,7 @@ It is possible to migrate an existing Wikibase installation to WBS Deploy. The g ### My WDQS Updater keeps crashing, what can I do? -Check out the known issue in the [WDQS README](../build/WDQS/README.md#Known-issues). You may find your solution there in the form of a workaround. +Check out the known issue in the [WDQS README](../build/wdqs/README.md#Known-issues). You may find your solution there in the form of a workaround. ### Do you recommend any VPS hosting providers? diff --git a/deploy/package.json b/deploy/package.json index eabeef09f..5ee26ddc7 100644 --- a/deploy/package.json +++ b/deploy/package.json @@ -1,6 +1,6 @@ { "name": "deploy", - "version": "3.0.0", + "version": "3.0.1", "nx": { "targets": { "lint": {}, diff --git a/docs/conventional-commits.md b/docs/conventional-commits.md new file mode 100644 index 000000000..7d1860e60 --- /dev/null +++ b/docs/conventional-commits.md @@ -0,0 +1,87 @@ +# Conventional commits + +[Conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) are human and machine readable commit messages. We use them to automatically generate changelog items and [semantic version (SemVer)](https://semver.org/) bumps on release using [NX](https://nx.dev). Here is an example of a commit message describing a bug fix in a backup script. Releasing this change would bump the project's patch version number: + +``` +fix(backup-script): report error if no space left on device +``` + +On release of a project, NX collects all commit messages from git for the project's directory (e.g. `build/wdqs`) since last release (e.g. `wdqs@1.0.1`). For this to work well we need to follow a certain procedure in our Pull requests: + +- Use conventional commit syntax for the PR title, then squash merge the PR, so that the squashed commit will contain the conventional commit message from the PR title on the target branch. The PR body will become the commit message body. +- Or: Merge with merge commit so that all commits from the PR are retained on the target branch. All commits in the PR should follow conventional commit syntax then. + +## Supported types + +NX supports [a number of](https://github.com/nrwl/nx/blob/db10812da789cd48d3a722628a00feda9d0e3810/packages/nx/src/command-line/release/config/conventional-commits.ts) conventional commit types. We use the following for changelog generation as configured in our [nx.json](https://github.com/wmde/wikibase-release-pipeline/blob/main/nx.json): + +| Type | SemVer Bump | Changelog Title | +| -------- | ----------- | ---------------- | +| build | patch | 📦 Build | +| chore | patch | 🏡 Chore | +| ci | patch | 🤖 CI | +| docs | patch | 📖 Documentation | +| feat | minor | 🚀 Features | +| fix | patch | 🩹 Fixes | +| perf | patch | 🔥 Performance | +| refactor | patch | 💅 Refactors | +| style | patch | 🎨 Styles | +| test | patch | ✅ Tests | + +## Examples + +### Feature + +A new feature implementation bumping the minor version: + +``` +feat: added support for cats +``` + +Will generate the following changelog entry: + +#### 🚀 Features + +- added support for cats + +### Documentation update + +Some update to the docs bumping a minor version: + +``` +docs: describe how to work around sleeping cats +``` + +Will generate the following changelog entry: + +#### 📖 Documentation + +- describe how to work around sleeping cats + +### Breaking change + +All types can be breaking changes. Here is an example for a breaking refactor, bumping a major version. + +``` +refactor!: remove support for sabre-toothed tiger +``` + +Will generate the following changelog entry: + +#### 💅 Refactors + +- remove support for sabre-toothed tiger + +### Performance change for a certain component + +Noting that a change is specific to a component or subsystem in a project. + +``` +pref(food-dispenser): improved speed +``` + +Will generate the following changelog entry: + +#### 🔥 Performance + +- food-dispenser: improved speed diff --git a/nx.json b/nx.json index 4408d021c..d86ca0ec0 100644 --- a/nx.json +++ b/nx.json @@ -8,6 +8,20 @@ } }, "release": { + "conventionalCommits": { + "types": { + "build": true, + "chore": true, + "ci": true, + "docs": true, + "feat": true, + "fix": true, + "perf": true, + "refactor": true, + "style": true, + "test": true + } + }, "groups": { "images": { "projects": [ "build/*" ] @@ -22,7 +36,14 @@ }, "changelog": { "projectChangelogs": { - "createRelease": "github" + "renderOptions": { + "authors": false + } + }, + "workspaceChangelog": { + "renderOptions": { + "authors": false + } } } },