Merge pull request #771 from wmde/wdqs-separate-concept-uri

wmde · Oct 23, 2024 · 82855b1 · 82855b1
2 parents 3dd4b6f + 3eef499
commit 82855b1
Show file tree

Hide file tree

Showing 7 changed files with 62 additions and 31 deletions.
diff --git a/build/wdqs/README.md b/build/wdqs/README.md
@@ -40,17 +40,18 @@ When running WDQS in a setup without WDQS-proxy, **please consider disabling the
 
 Variables in **bold** are required.
 
-| Variable                 | Default    | Description                                                                                                                                                                                                                                                               |
-| ------------------------ | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **`WIKIBASE_HOST`**      | "wikibase" | Hostname of the Wikibase service                                                                                                                                                                                                                                          |
-| **`WDQS_HOST`**          | "wdqs"     | WDQS hostname (this service)                                                                                                                                                                                                                                              |
-| **`WDQS_PORT`**          | "9999"     | WDQS port (this service)                                                                                                                                                                                                                                                  |
-| `WIKIBASE_SCHEME`        | "http"     | URL scheme of the Wikibase service                                                                                                                                                                                                                                        |
-| `WDQS_ENTITY_NAMESPACES` | "120,122"  | Wikibase namespaces to load data from                                                                                                                                                                                                                                     |
-| `WIKIBASE_MAX_DAYS_BACK` | "90"       | Maximum number of days updater can reach back in time from now                                                                                                                                                                                                            |
-| `MEMORY`                 | ""         | Memory limit for Blazegraph                                                                                                                                                                                                                                               |
-| `HEAP_SIZE`              | "1g"       | Heap size for Blazegraph                                                                                                                                                                                                                                                  |
-| `BLAZEGRAPH_EXTRA_OPTS`  | ""         | Extra options to be passed to Blazegraph,they must be prefixed with `-D`. Example: `-Dhttps.proxyHost=http://my.proxy.com -Dhttps.proxyPort=3128`. See [the WDQS User Manual](https://www.mediawiki.org/wiki/Wikidata_Query_Service/User_Manual#Configurable_properties). |
+| Variable                   | Default    | Description                                                                                                                                                                                                                                                               |
+| -------------------------  | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| **`WIKIBASE_HOST`**        | "wikibase" | Hostname to reach the Wikibase service, e.g. the docker network internal hostname                                                                                                                                                                                         |
+| **`WIKIBASE_CONCEPT_URI`** | ""         | Concept URI, required for `/runUpdate.sh` only, the identifying prefix to entities in this knowledge graph, e.g. the public URL of the Wikibase host.                                                                                                                     |
+| **`WDQS_HOST`**            | "wdqs"     | WDQS hostname (this service)                                                                                                                                                                                                                                              |
+| **`WDQS_PORT`**            | "9999"     | WDQS port (this service)                                                                                                                                                                                                                                                  |
+| `WIKIBASE_SCHEME`          | "http"     | URL scheme used to reach the Wikibase service, e.g. http to reach a local wikibase on the same docker network                                                                                                                                                             |
+| `WDQS_ENTITY_NAMESPACES`   | "120,122"  | Wikibase namespaces to load data from                                                                                                                                                                                                                                     |
+| `WIKIBASE_MAX_DAYS_BACK`   | "90"       | Maximum number of days updater can reach back in time from now                                                                                                                                                                                                            |
+| `MEMORY`                   | ""         | Memory limit for Blazegraph                                                                                                                                                                                                                                               |
+| `HEAP_SIZE`                | "1g"       | Heap size for Blazegraph                                                                                                                                                                                                                                                  |
+| `BLAZEGRAPH_EXTRA_OPTS`    | ""         | Extra options to be passed to Blazegraph,they must be prefixed with `-D`. Example: `-Dhttps.proxyHost=http://my.proxy.com -Dhttps.proxyPort=3128`. See [the WDQS User Manual](https://www.mediawiki.org/wiki/Wikidata_Query_Service/User_Manual#Configurable_properties). |
 
 ## Example
 
@@ -143,6 +144,8 @@ services:
       nofile:
         soft: 32768
         hard: 32768
+    environment:
+      WIKIBASE_CONCEPT_URI: https://wikibase.example
 
   wdqs-proxy:
     image: wikibase/wdqs-proxy
@@ -213,7 +216,7 @@ In the Docker Compose example provided above, you might use the commands and ins
 docker compose stop wdqs-updater
 
 # Start an updater with force sync settings
-docker compose run --rm wdqs-updater /wdqs/runUpdate.sh -h http://\$WDQS_HOST:\$WDQS_PORT -- --wikibaseUrl \$WIKIBASE_SCHEME://\$WIKIBASE_HOST --conceptUri \$WIKIBASE_SCHEME://\$WIKIBASE_HOST --entityNamespaces \$WDQS_ENTITY_NAMESPACES --init --start $(date +%Y%m%d000000)
+docker compose run --rm wdqs-updater /wdqs/runUpdate.sh -h http://\$WDQS_HOST:\$WDQS_PORT -- --wikibaseUrl \$WIKIBASE_SCHEME://\$WIKIBASE_HOST --conceptUri \$WIKIBASE_CONCEPT_URI --entityNamespaces \$WDQS_ENTITY_NAMESPACES --init --start $(date +%Y%m%d000000)
 
 # As soon as you see "Sleeping for 10 secs" in the logs, press CTRL-C to stop it again
 

diff --git a/build/wdqs/runUpdate.sh b/build/wdqs/runUpdate.sh
@@ -1,9 +1,15 @@
 #!/usr/bin/env bash
 # This file is provided by the wikibase/wdqs docker image.
 
+set +u
+if [ -z "$WIKIBASE_CONCEPT_URI" ]; then
+  echo "WIKIBASE_CONCEPT_URI is required but isn't set.";
+  exit 1;
+fi
+set -u
+
 cd /wdqs || exit
 
-# TODO env vars for entity namespaces, scheme and other settings
 /wait-for-it.sh "$WIKIBASE_HOST:80" -t 300 -- \
 /wait-for-it.sh "$WDQS_HOST:$WDQS_PORT" -t 300 -- \
-./runUpdate.sh -h http://"$WDQS_HOST":"$WDQS_PORT" -- --wikibaseUrl "$WIKIBASE_SCHEME"://"$WIKIBASE_HOST" --conceptUri "$WIKIBASE_SCHEME"://"$WIKIBASE_HOST" --entityNamespaces "$WDQS_ENTITY_NAMESPACES"
+./runUpdate.sh -h http://"$WDQS_HOST":"$WDQS_PORT" -- --wikibaseUrl "$WIKIBASE_SCHEME"://"$WIKIBASE_HOST" --conceptUri "$WIKIBASE_CONCEPT_URI" --entityNamespaces "$WDQS_ENTITY_NAMESPACES"
diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml
@@ -117,6 +117,8 @@ services:
       nofile:
         soft: 32768
         hard: 32768
+    environment:
+      WIKIBASE_CONCEPT_URI: https://${WIKIBASE_PUBLIC_HOST}
 
   wdqs-proxy:
     image: wikibase/wdqs-proxy:1

diff --git a/test/specs/repo/queryservice.ts b/test/specs/repo/queryservice.ts
@@ -69,7 +69,7 @@ describe( 'QueryService', function () {
 		const itemId = await WikibaseApi.createItem( getTestString( itemLabel ), data );
 
 		// query the item using wd: prefix
-		await QueryServiceUIPage.open( `SELECT * WHERE{ wd:${ itemId } ?p ?o }` );
+		await QueryServiceUIPage.open( `SELECT * WHERE{ <http://wikibase.example/entity/${ itemId }> ?p ?o }` );
 
 		// wait for WDQS-updater
 		// eslint-disable-next-line wdio/no-pause
@@ -114,7 +114,7 @@ describe( 'QueryService', function () {
 		).resolves.toEqual( true );
 
 		// query the property using wdt: prefix
-		await QueryServiceUIPage.open( `SELECT * WHERE{ ?s wdt:${ propertyId } ?o }` );
+		await QueryServiceUIPage.open( `SELECT * WHERE{ ?s <http://wikibase.example/prop/direct/${ propertyId }> ?o }` );
 
 		await QueryServiceUIPage.submit();
 		await QueryServiceUIPage.resultTable;
@@ -146,7 +146,7 @@ describe( 'QueryService', function () {
 		).replace( /[()]/g, '' );
 
 		// Check it shows up after creation
-		await QueryServiceUIPage.open( `SELECT * WHERE{ wd:${ itemId } ?p ?o }` );
+		await QueryServiceUIPage.open( `SELECT * WHERE{ <http://wikibase.example/entity/${ itemId }> ?p ?o }` );
 
 		// wait for WDQS-updater
 		// eslint-disable-next-line wdio/no-pause
@@ -172,7 +172,7 @@ describe( 'QueryService', function () {
 		);
 		await $( '.oo-ui-flaggedElement-destructive button' ).click();
 
-		await QueryServiceUIPage.open( `SELECT * WHERE{ wd:${ itemId } ?p ?o }` );
+		await QueryServiceUIPage.open( `SELECT * WHERE{ <http://wikibase.example/entity/${ itemId }> ?p ?o }` );
 
 		// wait for WDQS-updater
 		// eslint-disable-next-line wdio/no-pause

diff --git a/test/specs/repo/special-new-property.ts b/test/specs/repo/special-new-property.ts
@@ -17,7 +17,7 @@ describe( 'Special:NewProperty', function () {
 	// eslint-disable-next-line mocha/no-setup-in-describe
 	dataTypes.forEach( ( dataType: WikibasePropertyType ) => {
 		// eslint-disable-next-line mocha/no-setup-in-describe
-		describe( `Should be able to work with datatype ${ dataType.name }`, function () {
+		describe( `Should be able to work with datatype ${ dataType.name } `, function () {
 			before( async function () {
 				if ( dataType.extensionNeeded ) {
 					await browser.skipIfExtensionNotPresent(
@@ -51,7 +51,7 @@ describe( 'Special:NewProperty', function () {
 				await browser.waitForJobs();
 
 				await expect( browser ).toHaveUrl(
-					/http:\/\/wikibase\/wiki\/Property:P\d+/
+					/http:\/\/wikibase.example\/wiki\/Property:P\d+/
 				);
 
 				await expect( $( '.wikibase-propertyview-datatype-value' ) ).toHaveText(

diff --git a/test/suites/docker-compose.override.yml b/test/suites/docker-compose.override.yml
@@ -27,6 +27,8 @@ services:
 
   wdqs-updater:
     image: wikibase/wdqs
+    environment:
+      WIKIBASE_CONCEPT_URI: http://${WIKIBASE_PUBLIC_HOST}
 
   wdqs-proxy:
     image: wikibase/wdqs-proxy
@@ -55,5 +57,20 @@ services:
       SE_NODE_MAX_SESSIONS: ${MAX_INSTANCES}
       SE_VNC_NO_PASSWORD: 1
 
+  traefik:
+    command:
+      # Basic setup
+      - "--providers.docker=true"
+      - "--providers.docker.exposedbydefault=false"
+      # http endpoint
+      - "--entrypoints.web.address=:80"
+      - "--entrypoints.web.asdefault"
+    networks:
+      default:
+        aliases:
+          - wikibase.example
+          - wdqs-frontend.example
+          - quickstatements.example
+
 volumes:
   wikibase-config:
diff --git a/test/test-services.env b/test/test-services.env
@@ -1,21 +1,24 @@
 # Any of these can be overridden locally in ../local.env
 
-# == TEST SERVICES GLOBALS
+WIKIBASE_PUBLIC_HOST=wikibase.example
+WDQS_FRONTEND_PUBLIC_HOST=wdqs-frontend.example
+QUICKSTATEMENTS_PUBLIC_HOST=quickstatements.example
 
-# URLS
-WIKIBASE_URL=http://wikibase
+MW_ADMIN_NAME=admin
+MW_ADMIN_EMAIL=[email protected]
+MW_ADMIN_PASS=change-this-password
+
+# URLs used by tests to access services from within the
+# docker network through traefik proxy
+WIKIBASE_URL=http://wikibase.example
+WDQS_FRONTEND_URL=http://wdqs-frontend.example
+QUICKSTATEMENTS_URL=http://quickstatements.example
+
+# URLs used by tests to access services from within the docker network directly
 WIKIBASE_CLIENT_URL=http://wikibase-client
 WDQS_URL=http://wdqs:9999
-WDQS_FRONTEND_URL=http://wdqs-frontend
 WDQS_PROXY_URL=http://wdqs-proxy
-QUICKSTATEMENTS_URL=http://quickstatements
-
-# For Reverse Proxy (traefik) service routing
-WIKIBASE_PUBLIC_HOST=wikibase
-WDQS_FRONTEND_PUBLIC_HOST=wdqs-frontend
-QUICKSTATEMENTS_PUBLIC_HOST=quickstatements
 
 # CONFIG
-
 # wikibase / wikibase-client / wikibase-jobrunner
 MW_SCRIPT_PATH=/w