ci: finalize release 2.4.0 (#581)

.github/workflows/0.build-base-nifi-docker.yml

@@ -4,20 +4,20 @@ on:
   workflow_dispatch:
 
 env:
-  BASE_IMAGE_NAME: openjdk
-  BASE_IMAGE_TAG: 18-ea-bullseye
+  BASE_IMAGE_NAME: amazoncorretto
+  BASE_IMAGE_TAG: 21-alpine-jdk
   NIFI_VERSION: 1.20.0
   DISTRO_PATH: 1.20.0
   NIFI_DOCKER_IMAGE_NAME: ldes/nifi
-  NIFI_DOCKER_IMAGE_VERSION: 1.20.0-jdk18
+  NIFI_DOCKER_IMAGE_VERSION: 1.20.0-jdk21
 
 jobs:
   job-build-push-docker-nifi:
     name: Build and push VSDS NiFi docker image
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
       - name: Set up Docker Buildx
@@ -27,13 +27,13 @@ jobs:
         with:
           username: ${{ secrets.DOCKER_USER }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-      - name: Build NiFi JDK18 Docker image
+      - name: Build NiFi JDK21 Docker image
         run: |
           echo ENVIRONMENT
           echo BASE_IMAGE_NAME: ${BASE_IMAGE_NAME}
           echo BASE_IMAGE_TAG: ${BASE_IMAGE_TAG}
           echo NIFI_VERSION: ${NIFI_VERSION}
-          echo TARGET_IMAGE_TAG: ${NIFI_VERSION}-jdk18
+          echo TARGET_IMAGE_TAG: ${NIFI_VERSION}-jdk21
           echo NIFI_DOCKER_IMAGE_NAME: ${NIFI_DOCKER_IMAGE_NAME}
           echo NIFI_DOCKER_IMAGE_VERSION: ${NIFI_DOCKER_IMAGE_VERSION}
           curl -L https://github.com/apache/nifi/archive/refs/tags/rel/nifi-${NIFI_VERSION}.tar.gz | tar -zxv --strip-components=2 nifi-rel-nifi-${NIFI_VERSION}/nifi-docker/dockerhub/

.github/workflows/1.a-build-pr.yml

@@ -17,14 +17,14 @@ jobs:
     name: Build
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
-      - name: Set up JDK 18
-        uses: actions/setup-java@v3
+      - name: Set up JDK 21
+        uses: actions/setup-java@v4
         with:
-          java-version: '18'
-          distribution: 'adopt'
+          java-version: '21'
+          distribution: 'zulu'
       - name: Cache Maven packages
         uses: actions/cache@v1
         with:

.github/workflows/2.a-build-main.yml

@@ -23,12 +23,12 @@ jobs:
       contents: read
       packages: write
     steps:
-      - uses: actions/checkout@v3
-      - name: Set up JDK 18
-        uses: actions/setup-java@v3
+      - uses: actions/checkout@v4
+      - name: Set up JDK 21
+        uses: actions/setup-java@v4
         with:
           distribution: zulu
-          java-version: 18
+          java-version: 21
       - name: Cache SonarCloud packages
         uses: actions/cache@v1
         with:
@@ -42,10 +42,10 @@ jobs:
           key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
           restore-keys: ${{ runner.os }}-m2
       - name: Set up Maven Central Repository
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
-          java-version: '18'
-          distribution: 'adopt'
+          java-version: '21'
+          distribution: 'zulu'
           server-id: ossrh
           server-username: MAVEN_USERNAME
           server-password: MAVEN_PASSWORD

.github/workflows/2.b-build-ldes-discoverer-docker.yml

@@ -20,7 +20,7 @@ jobs:
       packages: write
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
       - name: Set up Docker Buildx
@@ -30,11 +30,11 @@ jobs:
           # See: https://github.com/docker/build-push-action/issues/761#issuecomment-1383822381
           driver-opts: |
             image=moby/buildkit:v0.10.6
-      - name: Set up JDK 18
-        uses: actions/setup-java@v3
+      - name: Set up JDK 21
+        uses: actions/setup-java@v4
         with:
           distribution: zulu
-          java-version: 18
+          java-version: 21
       - name: Determine version
         run: |
           export VERSION=$(mvn help:evaluate -Dexpression="project.version" -q -DforceStdout)

.github/workflows/2.b-build-ldio-docker.yml

@@ -20,7 +20,7 @@ jobs:
       packages: write
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
       - name: Set up Docker Buildx
@@ -30,11 +30,11 @@ jobs:
           # See: https://github.com/docker/build-push-action/issues/761#issuecomment-1383822381
           driver-opts: |
             image=moby/buildkit:v0.10.6
-      - name: Set up JDK 18
-        uses: actions/setup-java@v3
+      - name: Set up JDK 21
+        uses: actions/setup-java@v4
         with:
           distribution: zulu
-          java-version: 18
+          java-version: 21
       - name: Determine version
         run: |
           export VERSION=$(mvn help:evaluate -Dexpression="project.version" -q -DforceStdout)

.github/workflows/2.b-build-nifi-workbench-docker.yml

@@ -20,7 +20,7 @@ jobs:
       packages: write
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
       - name: Set up Docker Buildx
@@ -30,11 +30,11 @@ jobs:
           # See: https://github.com/docker/build-push-action/issues/761#issuecomment-1383822381
           driver-opts: |
             image=moby/buildkit:v0.10.6
-      - name: Set up JDK 18
-        uses: actions/setup-java@v3
+      - name: Set up JDK 21
+        uses: actions/setup-java@v4
         with:
           distribution: zulu
-          java-version: 18
+          java-version: 21
       - name: Determine version
         run: |
           export VERSION=$(mvn help:evaluate -Dexpression="project.version" -q -DforceStdout)

.github/workflows/3.approve-snapshot.yml

@@ -14,12 +14,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
-      - name: Set up JDK 18
-        uses: actions/setup-java@v3
+        uses: actions/checkout@v4
+      - name: Set up JDK 21
+        uses: actions/setup-java@v4
         with:
           distribution: zulu
-          java-version: 18
+          java-version: 21
       - name: Define docker variables
         run: |
           export VERSION=$(mvn help:evaluate -Dexpression="project.version" -q -DforceStdout)

CODEOWNERS

@@ -1,2 +1 @@
-*       @Yalz @rorlic @pj-cegeka @Tomvbe @jobulcke
-
+*       @Yalz @rorlic @pj-cegeka @jobulcke @ferre-vaes

docs/_ldio/examples/ex2-scrape-api.md

@@ -41,7 +41,7 @@ mappings:
       - [a, cs:GameDeal]
       - [cs:title, $(title)]
       - [cs:metacriticLink, $(metacriticLink)]
-      - [cs:thumb, $(thumb)~iri]
+      - [cs:thumb, $(thumb)]
       - p: cs:releaseDate
         o:
             function: ldi:epochToIso8601

docs/_ldio/examples/ex2/ex2-mapping.ttl

@@ -39,7 +39,7 @@ map:om_002 rml:reference "metacriticLink" ;
 
 map:om_003 rml:reference "thumb" ;
 	rdf:type rr:ObjectMap ;
-	rr:termType rr:IRI .
+	rr:termType rr:Literal .
 
 map:om_004 <http://semweb.mmlab.be/ns/fnml#functionValue> map:fn_000 ;
 	rdf:type <http://semweb.mmlab.be/ns/fnml#FunctionTermMap> ;

docs/_ldio/ldio-adapters/index.md

@@ -6,7 +6,7 @@ has_toc: true
 nav_order: 4
 ---
 
-# Linked Data Adapters
+# Linked Data Orchestrator Adapters
 
 The LDI Core module contains the components maintained by the VSDS team in order to accommodate the onboarding of LDES onboarders.
 

docs/_ldio/ldio-core/index.md

@@ -6,6 +6,6 @@ has_toc: true
 nav_order: 2
 ---
 
-# LDIO Core
+# Linked Data Orchestrator Core
 
 Contains common components used by multiple other LDIO components

docs/_ldio/ldio-inputs/index.md

@@ -6,7 +6,7 @@ has_toc: true
 nav_order: 3
 ---
 
-# LDIO Inputs
+# Linked Data Orchestrator Inputs
 
 The LDI Core module contains the components maintained by the VSDS team in order to accommodate the onboarding of LDES onboarders.
 

docs/_ldio/ldio-inputs/ldio-ldes-client-connector.md

@@ -24,7 +24,7 @@ This component exposes two endpoints:
    transfer
    with the connector and also keep the transfer alive while consuming the LDES (e.g. request a new token when it
    expires).
-3. http://<host>:<port>/<pipelines.name>/token
+2. http://<host>:<port>/<pipelines.name>/token
    This endpoint should never be called directly. This is the callback to be provided in the transfer request.
    The EDC connector will use this callback endpoint to provide the LDES Client with a token.
 
@@ -54,6 +54,12 @@ input:
       source-format: application/n-quads
 ```
 
+## INIT phase
+
+Contrary to the other ldio-input components, the connector waits in the INIT status for the edc-token and will only progress to the RUNNING status once it has received this token.
+More on the statuses in ldio can be found [here.](../pipeline-management/pipeline-status)
+
+
 ## Pausing
 
 When paused, this component will stop processing the current fragment and not make any calls to the server.

docs/_ldio/ldio-inputs/ldio-ldes-client.md

@@ -14,14 +14,14 @@ An LDIO wrapper component for the [LDI LDES Client building block](../../core/ld
 
 ### General properties
 
-| Property                    | Description                                                                             | Required | Default      | Example                                   | Supported values                                              |
-|:----------------------------|:----------------------------------------------------------------------------------------|:---------|:-------------|:------------------------------------------|:--------------------------------------------------------------|
-| urls                        | List of URLs of the LDES data sources                                                   | Yes      | N/A          | http://localhost:8080/my-ldes             | HTTP and HTTPS urls                                           |
+| Property                    | Description                                                                             | Required | Default      | Example                                   | Supported values                                                                                                        |
+|:----------------------------|:----------------------------------------------------------------------------------------|:---------|:-------------|:------------------------------------------|:------------------------------------------------------------------------------------------------------------------------|
+| urls                        | List of URLs of the LDES data sources                                                   | Yes      | N/A          | http://localhost:8080/my-ldes             | HTTP and HTTPS urls                                                                                                     |
 | source-format               | The 'Content-Type' that should be requested to the server.                              | No       | text/turtle  | application/n-quads                       | Any type supported by [Apache Jena](https://jena.apache.org/documentation/io/rdf-input.html#determining-the-rdf-syntax) |
-| state                       | 'sqlite', 'memory', 'file' or 'postgres' to indicate how the state should be persisted. | No       | memory       | sqlite                                    | 'sqlite', 'files' or 'memory'                                 |
-| keep-state                  | Indicates if the state should be persisted on shutdown (n/a for in memory states)       | No       | false        | false                                     | true or false                                                 |
-| timestamp-path              | The property-path used to determine the timestamp on which the members will be ordered  | No       | N/A          | http://www.w3.org/ns/prov#generatedAtTime | A property path                                               |
-| enable-exactly-once         | Indicates whether member must be send exactly once or at least once                     | No       | true         | true                                      | true or false                                                 |
+| state                       | 'sqlite', 'memory', 'file' or 'postgres' to indicate how the state should be persisted. | No       | memory       | sqlite                                    | 'sqlite', 'files' or 'memory'                                                                                           |
+| keep-state                  | Indicates if the state should be persisted on shutdown (n/a for in memory states)       | No       | false        | false                                     | true or false                                                                                                           |
+| timestamp-path              | The property-path used to determine the timestamp on which the members will be ordered  | No       | N/A          | http://www.w3.org/ns/prov#generatedAtTime | A property path                                                                                                         |
+| enable-exactly-once         | Indicates whether member must be send exactly once or at least once                     | No       | true         | true                                      | true or false                                                                                                           |
 
 > **_NOTE:_** The default `source-format` is `text/turtle`, as this rdf format supports relative uri's. However, if
 > relative uri's are not used, `application/n-quads` or even the binary format `application/rdf+protobuf` are better

docs/_ldio/ldio-outputs/index.md

@@ -6,7 +6,7 @@ has_toc: true
 nav_order: 6
 ---
 
-# Linked Data Adapters
+# Linked Data Orchestrator Outputs
 
 The LDI Core module contains the components maintained by the VSDS team in order to accommodate the onboarding of LDES onboarders.
 

docs/_ldio/ldio-transformers/index.md

@@ -6,8 +6,4 @@ has_toc: true
 nav_order: 5
 ---
 
-# Linked Data Adapters
-
-The LDI Core module contains the components maintained by the VSDS team in order to accommodate the onboarding of LDES onboarders.
-
-Each component can be wrapped in a desired implementation framework (LDI-orchestrator, NiFi, ...) to be used.
+# Linked Data Orchestrator Transformers

docs/_ldio/ldio-transformers/ldio-version-object-creator.md

@@ -12,13 +12,13 @@ An LDIO wrapper component for the [LDI Version Object Creator building block](..
 
 ## Config
 
-| Property               | Description                                                                                                                    | Required | Default           | Example                                                             | Supported values |
-|:-----------------------|:-------------------------------------------------------------------------------------------------------------------------------|:---------|:------------------|:--------------------------------------------------------------------|:-----------------|
-| date-observed-property | Property path (IRI format '<>') that points to a literal which should be used as timestampPath. Defaults to current timestamp. | No       | Current Timestamp | <https://uri.etsi.org/ngsi-ld/default-context/WaterQualityObserved> | String           |
-| member-type            | Defines the RDF type of the version object                                                                                     | No       | N/A               | https://uri.etsi.org/ngsi-ld/default-context/Device                 | String           |
-| delimiter              | Defines how the version object id will be constructed. (versionOf + delimiter + dateObserved)                                  | No       | /                 | /                                                                   | String           |
-| generatedAt-property   | If defined, a statement will be added to the model with the observedAt value and the given property.                           | No       | N/A               | http://www.w3.org/ns/prov#generatedAtTime                           | String           |
-| versionOf-property     | If defined, a statement will be added to the model with the versionOf value and the given property.                            | No       | N/A               | http://purl.org/dc/terms/isVersionOf                                | String           |
+| Property               | Description                                                                                                                    | Required | Default           | Example                                   | Supported values |
+|:-----------------------|:-------------------------------------------------------------------------------------------------------------------------------|:---------|:------------------|:------------------------------------------|:-----------------|
+| date-observed-property | Property path (IRI format '<>') that points to a literal which should be used as timestampPath. Defaults to current timestamp. | No       | Current Timestamp | <https://example.org/ObservedAt>          | String           |
+| member-type            | Defines the RDF type of the version object                                                                                     | No       | N/A               | https://example.org/Person                | String           |
+| delimiter              | Defines how the version object id will be constructed. (versionOf + delimiter + dateObserved)                                  | No       | /                 | /                                         | String           |
+| generatedAt-property   | If defined, a statement will be added to the model with the observedAt value and the given property.                           | No       | N/A               | http://www.w3.org/ns/prov#generatedAtTime | String           |
+| versionOf-property     | If defined, a statement will be added to the model with the versionOf value and the given property.                            | No       | N/A               | http://purl.org/dc/terms/isVersionOf      | String           |
 
 A property path can be provided for the date-observed-property. 
 For example for the following ttl:

docs/_ldio/pipeline-management/pipeline-status.md

@@ -0,0 +1,54 @@
+---
+layout: default
+parent: Pipeline Management
+title: Pipeline Status
+---
+# Status
+
+An individual ldio-pipeline can be in one of several different statuses.
+These different statuses and their behaviour are dependent on the input component of the pipeline.
+
+# Overview Of The Status Flow
+
+
+<div class="mermaid" style="width: 400px; height: 130px;">
+graph LR;
+    INIT --> RUNNING;
+    INIT --> STOPPED;
+    RUNNING --> STOPPED;
+    RUNNING --> HALTED;
+    HALTED --> RUNNING;
+    HALTED --> STOPPED;
+</div>
+
+The above diagram shows the flow between the different statuses of the pipeline.
+
+## INIT
+
+The startup step of the ldio pipeline.
+This is the preparation step before the input component can receive data and pass it on to the rest of the pipeline.
+In most components this step will take little time. The only exception for now is [the client-connector](../ldio-inputs/ldio-ldes-client-connector).
+It is not possible to pause the pipeline while in this state.
+
+## RUNNING
+
+This status indicates that the ldio pipeline is running and that the input component is ready to or currently receiving or fetching data.
+This is the only state in which the pipeline can be paused.
+
+## HALTED
+
+Every ldio pipeline can be paused, the exact behavior of which depends on the ldio input component used.
+A more in depth explanation can be found on the pages for [the individual input components.](../ldio-inputs/index)
+Currently, the HALTED status can only be reached through manually pausing the pipeline through [the pipeline-api](./pipeline-api).
+
+## STOPPED
+
+When a pipeline is deleted, it will first change to the STOPPED status, this ensures the state is correctly saved in stateful components and that the entire pipeline can be gracefully shutdown.
+This status will only last a short while before the pipeline is deleted and the status can no longer be queried.
+The pipeline can be stopped from any other status.
+
+## Status Change Source
+
+The ldio pipeline keeps track of if the last status change was triggered manually or automatic.
+Manually changing the status can be done through [the pipeline-api](./pipeline-api).
+When in the RUNNING state, this indicates if the pipeline has been started (automatic) or unpaused (manual).