Merge branch 'main' into wdqs-drop-privileges-during-build

.github/workflows/build_publish_image_release.yml

@@ -8,7 +8,9 @@ on:
       - 'wdqs@*'
       - 'wdqs-frontend@*'
       - 'wdqs-proxy@*'
-      - 'wikibase@*'     
+      - 'wikibase@*'
+    branches-ignore:
+      - '**' # weird workaround to trigger on multiple tags for a single commit
 
 permissions:
   contents: write
@@ -19,17 +21,18 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Extract imageName and confirm it is a valid semantic version tag
+      - name: Extract image name and confirm it is a valid semantic version tag
         id: extract_tag
         run: |
+          set -x # show commands
+
           # Extract the tag name
           TAG_NAME=$(echo ${{ github.ref }} | sed 's/refs\/tags\///')
 
           # Use regex to extract the imageName before the @ and confirm the version is valid semver version tag
           if [[ "$TAG_NAME" =~ ^([^@]+)@([0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?)$ ]]; then
-            # Sets and passes the imageName output from this step, which is used in the subsequent step
-            # e.g. `steps.extract_tag.outputs.imageName` gets set to "wikibase" matched from "[email protected]")
-            echo "{imageName}=${BASH_REMATCH[1]}" >> $GITHUB_OUTPUT
+            # Store image name in environment for later steps
+            echo "IMAGE_NAME=${BASH_REMATCH[1]}" >> $GITHUB_ENV
           else
             echo "Tag does not match the pattern {imageName}@{semantic-version}"
             exit 1
@@ -52,7 +55,4 @@ jobs:
           set -e # abort on error
           set -x # show commands
 
-          git config --global user.name 'wikibase suite github actions bot'
-          git config --global user.email '[email protected]'
-          
-          ./nx release publish -p ${{ steps.extract_tag.outputs.imageName }}
+          ./nx release publish -p $IMAGE_NAME

.github/workflows/create_release.yml

@@ -18,7 +18,7 @@ on:
           - wikibase
           - All projects (images and deploy) with unreleased changes
       dry_run:
-        description: "Only log what will be done, don't do it yet."
+        description: "Dry run, don't do it yet."
         type: boolean
         required: true
         default: true
@@ -33,24 +33,22 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-tags: true
+          fetch-depth: 0
+          token: ${{ secrets.GH_WBS_BOT_TOKEN }}
 
       - uses: ./.github/actions/setup-environment
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
-      - uses: docker/login-action@v3
-        with:
-          # implicitly docker hub
-          username: wmdetravisbot
-          password: ${{ secrets.WBS_PUBLISH_TOKEN }}
-
-      - name: Git tag release
+      - name: Create release
         run: |
           set -e # abort on error
           set -x # show commands
 
-          git config --global user.name 'wikibase suite github actions bot'
-          git config --global user.email 'wikibase-suite-[email protected]'
+          git config --global user.name 'wikibase-suite-bot'
+          git config --global user.email 'wikibase-suite-[email protected]'
 
           if [ "${{ inputs.dry_run }}" == "true" ]; then
             DRY_RUN_FLAG="--dry-run"
@@ -64,4 +62,10 @@ jobs:
             PROJECT_ARG="-p ${{ inputs.image_name }}"
           fi
 
-          ./nx release $PROJECT_ARG $DRY_RUN_FLAG
+          ./nx release $PROJECT_ARG $DRY_RUN_FLAG --skip-publish --verbose
+
+          # Temporary workaround for nx issue: https://github.com/nrwl/nx/issues/22073#
+          if [ $? -eq 0 ] && [ -z "$DRY_RUN_FLAG" ]; then
+            git push origin main --tags
+          fi
+

README.md

@@ -48,6 +48,12 @@ $ ./nx test -- repo
 # Runs the `repo` test suite with a specific spec file (paths to spec files are rooted in the `test` directory)
 $ ./nx test -- repo --spec specs/repo/special-item.ts
 
+# Start with a headed browser
+$ ./nx test -- repo --headed
+
+# Start a specific spec only
+./nx test -- repo --spec specs/repo/queryservice.ts
+
 # Start and leave up the test environment for a given test suite without running tests
 $ ./nx test -- repo --setup
 ```

build/build.sh

@@ -76,7 +76,7 @@ IMAGE_NAME=$(jq -r '.name' package.json)
 
 # publish to Dockerhub
 if [ "$PUBLISH" == true ]; then
-	IMAGE_REGISTRY=dockerhub.io
+	# IMAGE_REGISTRY implies dockerhub if empty
 	IMAGE_NAMESPACE=wikibase
 # build/test in CI
 elif [ "$GITHUB_ACTIONS" == true ]; then

build/elasticsearch/CHANGELOG.md

@@ -0,0 +1,5 @@
+## **[email protected]** (2024-10-09)
+
+### 📖 Documentation
+
+- Switch from `.example.com` to `.example`

build/elasticsearch/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "elasticsearch",
-	"version": "1.0.0",
+	"version": "1.0.1",
 	"nx": {
 		"targets": {
 			"lint": {},

build/quickstatements/CHANGELOG.md

@@ -0,0 +1,5 @@
+## **[email protected]** (2024-10-09)
+
+### 📖 Documentation
+
+- Switch from `.example.com` to `.example`

build/quickstatements/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "quickstatements",
-	"version": "1.0.0",
+	"version": "1.0.1",
 	"nx": {
 		"targets": {
 			"lint": {},

build/wdqs-frontend/CHANGELOG.md

@@ -0,0 +1,9 @@
+## **[email protected]** (2024-10-09)
+
+### 📖 Documentation
+
+- Switch from `.example.com` to `.example`
+
+### 🩹 Fixes
+
+- Link to Query Builder now points to the source code repository ([#664](https://github.com/wmde/wikibase-release-pipeline/issues/664))

build/wdqs-frontend/README.md

@@ -2,6 +2,14 @@
 
 Frontend for the [Wikidata Query Service (WDQS)](https://www.mediawiki.org/wiki/Wikidata_Query_Service).
 
+To interact with the WDQS frontend, navigate to the URL corresponding to the port allocated for it. In the example below, the WDQS frontend is available at `http://localhost:8834`.
+
+When writing queries using the frontend interface, click "Code" to view the corresponding URL.
+
+For general instructions on using WDQS, building SPARQL queries, and additional resources, see:
+- [Wikidata Query Service User Manual](https://www.mediawiki.org/wiki/Wikidata_Query_Service/User_Manual)
+- [What is SPARQL](https://www.wikidata.org/wiki/Wikidata:SPARQL_query_service)
+
 > 💡 This image is part of Wikibase Suite (WBS). [WBS Deploy](https://github.com/wmde/wikibase-release-pipeline/deploy/README.md) provides everything you need to self-host a Wikibase instance out of the box.
 
 ## Requirements
@@ -21,11 +29,11 @@ We suggest using the [WBS Wikibase image](https://hub.docker.com/r/wikibase/wiki
 
 ### WDQS as server
 
-We suggest using the [WBS Wikibase image](https://hub.docker.com/r/wikibase/wdqs).
+We suggest using the [WBS WDQS image](https://hub.docker.com/r/wikibase/wdqs).
 
 ### WDQS as updater
 
-We suggest using the [WBS Wikibase image](https://hub.docker.com/r/wikibase/wdqs), the same as used for WDQS server. Check out the [documentation](https://wikitech.wikimedia.org/wiki/Wikidata_Query_Service) to learn how to run it in updater mode.
+We suggest using the [WBS WDQS image](https://hub.docker.com/r/wikibase/wdqs), the same as used for WDQS server. Check out the [documentation](https://wikitech.wikimedia.org/wiki/Wikidata_Query_Service) to learn how to run it in updater mode.
 
 ### Reverse proxy
 
@@ -185,7 +193,7 @@ volumes:
 
 Official releases of this image can be found on [Docker Hub wikibase/wdqs-frontend](https://hub.docker.com/r/wikibase/wdqs-frontend).
 
-## Tags and Versioning
+## Tags and versioning
 
 This WDQS Frontend image is using [semantic versioning](https://semver.org/spec/v2.0.0.html).
 
@@ -206,4 +214,4 @@ This image is built from this [Dockerfile](https://github.com/wmde/wikibase-rele
 
 This image is maintained by the Wikibase Suite Team at [Wikimedia Germany (WMDE)](https://wikimedia.de).
 
-If you have questions not listed above or need help, use this [bug report form](https://phabricator.wikimedia.org/maniphest/task/edit/form/129/) to start a conversation with the engineering team.
+If you have questions not covered above or need further help, use this [bug report form](https://phabricator.wikimedia.org/maniphest/task/edit/form/129/) to start a conversation with the engineering team.

build/wdqs-frontend/custom-config.json

@@ -12,7 +12,10 @@
 			"pageTitle": "Wikidata:SPARQL_query_service/queries/examples",
 			"pagePathElement": "wiki/"
 		},
-		"urlShortener": "tinyurl"
+		"urlShortener": "tinyurl",
+		"query-builder": {
+			"server": "https://gerrit.wikimedia.org/g/wikidata/query-builder"
+		}
 	},
 	"brand": {
 		"title": "$BRAND_TITLE",

build/wdqs-frontend/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "wdqs-frontend",
-	"version": "1.0.0",
+	"version": "1.0.1",
 	"nx": {
 		"targets": {
 			"lint": {},

build/wdqs-proxy/CHANGELOG.md

@@ -0,0 +1,5 @@
+## **[email protected]** (2024-10-09)
+
+### 📖 Documentation
+
+- Switch from `.example.com` to `.example`

build/wdqs-proxy/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "wdqs-proxy",
-	"version": "1.0.0",
+	"version": "1.0.1",
 	"nx": {
 		"targets": {
 			"lint": {},

build/wdqs/CHANGELOG.md

@@ -0,0 +1,7 @@
+## **[email protected]** (2024-10-09)
+
+### 📖 Documentation
+
+- Switch from `.example.com` to `.example`
+- Add comments in "Updater keeps restarting" FAQ entry
+- Correct typos

build/wdqs/README.md

@@ -23,6 +23,9 @@ We suggest using the [WBS Wikibase image](https://hub.docker.com/r/wikibase/wiki
 
 You'll need one instance of the image to execute the actual WDQS daemon started using `/runBlazegraph.sh`.
 
+You can send `GET` requests with your SPARQL query to the WDQS endpoint (following the example below):
+`http://wdqs:9999/bigdata/namespace/wdq/sparql?query={SPARQL}`
+
 ### WDQS as updater
 
 You'll need one instance of the image to execute the updater started using `/runUpdate.sh`. This polls changes from Wikibase.

build/wdqs/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "wdqs",
-	"version": "1.0.0",
+	"version": "1.0.1",
 	"nx": {
 		"targets": {
 			"lint": {},

build/wikibase/CHANGELOG.md

@@ -0,0 +1,11 @@
+## **[email protected]** (2024-10-09)
+
+### 📖 Documentation
+
+- Link to MediaWiki bundled extensions
+- Switch from `.example.com` to `.example`
+
+### 🏡 Chore
+
+- Removes re-installation of extensions already packaged with MediaWiki (ConfirmEdit, Nuke, Scribunto, SyntaxHighlight_Geshi, VisualEditor)
+- Updates Wikibase EDTF extension

build/wikibase/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "wikibase",
-	"version": "3.0.0",
+	"version": "3.0.1",
 	"nx": {
 		"targets": {
 			"lint": {},

deploy/CHANGELOG.md

@@ -0,0 +1,17 @@
+## **[email protected]** (2024-10-09)
+
+### 🩹 Fixes
+
+- `*_PUBLIC_HOST` URLs default to `.example` TLD from `example.com` ([T372455](https://phabricator.wikimedia.org/T372455))
+- Removes published ports for wikibase, quickstatements, and wdqs-frontend services ([T372455](https://phabricator.wikimedia.org/T372455))
+- Wikibase waits for elasticsearch service to be up ([T371162](https://phabricator.wikimedia.org/T371162))
+
+### 📖 Documentation
+
+- Do not use `--wait` in `docker compose up` in order to see logs
+- Link to WDQS updater crash FAQ entry
+- Corrected typos
+
+### 🏡 Chore
+
+- Adds default Traefik LetsEncrypt configuration for services

deploy/README.md

@@ -146,12 +146,13 @@ To back up your data, shut down the instance and dump the contents of all Docker
 docker compose down
 
 for v in \
-    wikibase-suite-wikibase_image-data \
-    wikibase-suite_mysql-data \
-    wikibase-suite_wdqs-data \
-    wikibase-suite_elasticsearch-data \
-    wikibase-suite_quickstatements-data \
-    wikibase-suite_traefik-letsencrypt-data; do
+    wbs-deploy_elasticsearch-data \
+    wbs-deploy_mysql-data \
+    wbs-deploy_quickstatements-data \
+    wbs-deploy_traefik-letsencrypt-data \
+    wbs-deploy_wdqs-data \
+    wbs-deploy_wikibase-image-data \
+    ; do
   docker run --rm --volume $v:/backup debian:12-slim tar cz backup > $v.tar.gz
 done
 ```
@@ -164,12 +165,13 @@ To restore the volume backups, ensure your instance has been shut down by runnin
 docker compose down
 
 for v in \
-    wikibase-suite-wikibase_image-data \
-    wikibase-suite_mysql-data \
-    wikibase-suite_wdqs-data \
-    wikibase-suite_elasticsearch-data \
-    wikibase-suite_quickstatements-data \
-    wikibase-suite_traefik-letsencrypt-data; do
+    wbs-deploy_elasticsearch-data \
+    wbs-deploy_mysql-data \
+    wbs-deploy_quickstatements-data \
+    wbs-deploy_traefik-letsencrypt-data \
+    wbs-deploy_wdqs-data \
+    wbs-deploy_wikibase-image-data \
+    ; do
   docker volume rm $v 2> /dev/null
   docker volume create $v
   docker run -i --rm --volume $v:/backup debian:12-slim tar xz < $v.tar.gz
@@ -303,6 +305,15 @@ Removing the `traefik-letsencrypt-data` volume will request a new certificate fr
 ```yml
       --certificatesresolvers.letsencrypt.acme.caserver=https://acme-staging-v02.api.letsencrypt.org/directory
 ```
+
+## WDQS Frontend
+
+To interact with the WDQS frontend, navigate to the URL defined as `WDQS_FRONTEND_PUBLIC_HOST` in the `.env` file. By default, this is set to `wdqs-frontend.example`.
+
+Alternatively, send `GET` requests with your SPARQL query to the WDQS frontend endpoint:
+`https://wdqs-frontend.example.com/proxy/wdqs/bigdata/namespace/wdq/sparql?query={SPARQL}`
+
+
 ## FAQ
 
 ### Can I host WBS Deploy locally?
@@ -324,7 +335,7 @@ It is possible to migrate an existing Wikibase installation to WBS Deploy. The g
 
 ### My WDQS Updater keeps crashing, what can I do?
 
-Check out the known issue in the [WDQS README](../build/WDQS/README.md#Known-issues). You may find your solution there in the form of a workaround.
+Check out the known issue in the [WDQS README](../build/wdqs/README.md#Known-issues). You may find your solution there in the form of a workaround.
 
 ### Do you recommend any VPS hosting providers?
 

deploy/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "deploy",
-	"version": "3.0.0",
+	"version": "3.0.1",
 	"nx": {
 		"targets": {
 			"lint": {},