From 9c5e866a8942efe895d4f0753d7bc8ac5fa748f4 Mon Sep 17 00:00:00 2001 From: tomvbe Date: Wed, 27 Mar 2024 11:25:14 +0100 Subject: [PATCH 01/20] chore: bump release version --- ldi-api/pom.xml | 2 +- ldi-core/file-archiving/pom.xml | 2 +- ldi-core/geojson-to-wkt/pom.xml | 2 +- ldi-core/json-to-ld-adapter/pom.xml | 2 +- ldi-core/ldes-client/pom.xml | 2 +- ldi-core/ldes-client/starting-node/pom.xml | 2 +- ldi-core/ldes-client/tree-node-fetcher/pom.xml | 2 +- ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml | 2 +- ldi-core/ldes-client/tree-node-supplier/pom.xml | 2 +- ldi-core/ldi-common/pom.xml | 2 +- ldi-core/ngsiv2-to-ld-adapter/pom.xml | 2 +- ldi-core/pom.xml | 2 +- ldi-core/rdf-adapter/pom.xml | 2 +- ldi-core/repository-materialiser/pom.xml | 2 +- ldi-core/request-executor/pom.xml | 2 +- ldi-core/rml-adapter/pom.xml | 2 +- ldi-core/sparql-construct/pom.xml | 2 +- ldi-core/version-materialiser/pom.xml | 2 +- ldi-core/version-object-creator/pom.xml | 2 +- ldi-extensions/ldes-discoverer/pom.xml | 2 +- ldi-extensions/pom.xml | 2 +- ldi-nifi/ldi-nifi-common/pom.xml | 2 +- ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml | 2 +- ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml | 2 +- .../create-version-object-processor/pom.xml | 2 +- ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml | 2 +- ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml | 2 +- ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml | 2 +- ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml | 2 +- ldi-nifi/ldi-nifi-processors/pom.xml | 2 +- .../rdf4j-repository-materialisation-processor/pom.xml | 2 +- .../ldi-nifi-processors/sparql-interactions-processor/pom.xml | 2 +- .../version-materialisation-processor/pom.xml | 2 +- ldi-nifi/pom.xml | 2 +- ldi-orchestrator/ldio-application/pom.xml | 2 +- ldi-orchestrator/ldio-common/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml | 2 +- .../ldio-connectors/ldio-json-to-ld-adapter/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml | 2 +- .../ldio-connectors/ldio-ldes-client-connector/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml | 2 +- .../ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml | 4 ++-- ldi-orchestrator/ldio-connectors/ldio-noop-out/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml | 2 +- .../ldio-connectors/ldio-repository-materialiser/pom.xml | 2 +- .../ldio-connectors/ldio-request-executor/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml | 2 +- .../ldio-connectors/ldio-sparql-construct/pom.xml | 2 +- .../ldio-connectors/ldio-version-materialiser/pom.xml | 2 +- .../ldio-connectors/ldio-version-object-creator/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/pom.xml | 2 +- ldi-orchestrator/pom.xml | 2 +- pom.xml | 2 +- 62 files changed, 63 insertions(+), 63 deletions(-) diff --git a/ldi-api/pom.xml b/ldi-api/pom.xml index 7990675fb..83d6474d6 100644 --- a/ldi-api/pom.xml +++ b/ldi-api/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/file-archiving/pom.xml b/ldi-core/file-archiving/pom.xml index 82fdf9517..d8461781b 100644 --- a/ldi-core/file-archiving/pom.xml +++ b/ldi-core/file-archiving/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT file-archiving diff --git a/ldi-core/geojson-to-wkt/pom.xml b/ldi-core/geojson-to-wkt/pom.xml index 4baa82451..67e492312 100644 --- a/ldi-core/geojson-to-wkt/pom.xml +++ b/ldi-core/geojson-to-wkt/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT geojson-to-wkt diff --git a/ldi-core/json-to-ld-adapter/pom.xml b/ldi-core/json-to-ld-adapter/pom.xml index 5cd11c80e..500dde2a9 100644 --- a/ldi-core/json-to-ld-adapter/pom.xml +++ b/ldi-core/json-to-ld-adapter/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT json-to-ld-adapter diff --git a/ldi-core/ldes-client/pom.xml b/ldi-core/ldes-client/pom.xml index 071f50cc2..ae20022db 100644 --- a/ldi-core/ldes-client/pom.xml +++ b/ldi-core/ldes-client/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/ldes-client/starting-node/pom.xml b/ldi-core/ldes-client/starting-node/pom.xml index 2bf57b7ae..6e699a8a3 100644 --- a/ldi-core/ldes-client/starting-node/pom.xml +++ b/ldi-core/ldes-client/starting-node/pom.xml @@ -5,7 +5,7 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 starting-node diff --git a/ldi-core/ldes-client/tree-node-fetcher/pom.xml b/ldi-core/ldes-client/tree-node-fetcher/pom.xml index 45a2529e1..9e71cfc12 100644 --- a/ldi-core/ldes-client/tree-node-fetcher/pom.xml +++ b/ldi-core/ldes-client/tree-node-fetcher/pom.xml @@ -5,7 +5,7 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT diff --git a/ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml b/ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml index 25c3d6819..64e3ec1f7 100644 --- a/ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml +++ b/ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml @@ -5,7 +5,7 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/ldes-client/tree-node-supplier/pom.xml b/ldi-core/ldes-client/tree-node-supplier/pom.xml index d0abf79ad..1f680fabc 100644 --- a/ldi-core/ldes-client/tree-node-supplier/pom.xml +++ b/ldi-core/ldes-client/tree-node-supplier/pom.xml @@ -5,7 +5,7 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 tree-node-supplier diff --git a/ldi-core/ldi-common/pom.xml b/ldi-core/ldi-common/pom.xml index e5ee8f5c0..5c97d2af2 100644 --- a/ldi-core/ldi-common/pom.xml +++ b/ldi-core/ldi-common/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT ldi-common diff --git a/ldi-core/ngsiv2-to-ld-adapter/pom.xml b/ldi-core/ngsiv2-to-ld-adapter/pom.xml index 9c201b3f8..10972eeb1 100644 --- a/ldi-core/ngsiv2-to-ld-adapter/pom.xml +++ b/ldi-core/ngsiv2-to-ld-adapter/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT ngsiv2-to-ld-adapter diff --git a/ldi-core/pom.xml b/ldi-core/pom.xml index f8200b075..6cb0e2af1 100644 --- a/ldi-core/pom.xml +++ b/ldi-core/pom.xml @@ -3,7 +3,7 @@ linked-data-interactions be.vlaanderen.informatievlaanderen.ldes - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/rdf-adapter/pom.xml b/ldi-core/rdf-adapter/pom.xml index cb72e0e49..0295d1d0c 100644 --- a/ldi-core/rdf-adapter/pom.xml +++ b/ldi-core/rdf-adapter/pom.xml @@ -5,7 +5,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/repository-materialiser/pom.xml b/ldi-core/repository-materialiser/pom.xml index eef320836..0bd227f44 100644 --- a/ldi-core/repository-materialiser/pom.xml +++ b/ldi-core/repository-materialiser/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT diff --git a/ldi-core/request-executor/pom.xml b/ldi-core/request-executor/pom.xml index d7a0b9f0f..f79dae4a4 100644 --- a/ldi-core/request-executor/pom.xml +++ b/ldi-core/request-executor/pom.xml @@ -5,7 +5,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 request-executor diff --git a/ldi-core/rml-adapter/pom.xml b/ldi-core/rml-adapter/pom.xml index 272287ae9..f843392fd 100644 --- a/ldi-core/rml-adapter/pom.xml +++ b/ldi-core/rml-adapter/pom.xml @@ -5,7 +5,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/sparql-construct/pom.xml b/ldi-core/sparql-construct/pom.xml index e445ef069..daa3e5315 100644 --- a/ldi-core/sparql-construct/pom.xml +++ b/ldi-core/sparql-construct/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/version-materialiser/pom.xml b/ldi-core/version-materialiser/pom.xml index 9235a91be..58f59287f 100644 --- a/ldi-core/version-materialiser/pom.xml +++ b/ldi-core/version-materialiser/pom.xml @@ -3,7 +3,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/version-object-creator/pom.xml b/ldi-core/version-object-creator/pom.xml index 137f3ebb8..00afe2785 100644 --- a/ldi-core/version-object-creator/pom.xml +++ b/ldi-core/version-object-creator/pom.xml @@ -3,7 +3,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-extensions/ldes-discoverer/pom.xml b/ldi-extensions/ldes-discoverer/pom.xml index 168cb466a..da524ef79 100644 --- a/ldi-extensions/ldes-discoverer/pom.xml +++ b/ldi-extensions/ldes-discoverer/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-extensions - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT ldes-discoverer diff --git a/ldi-extensions/pom.xml b/ldi-extensions/pom.xml index 181a6db33..148ffb99f 100644 --- a/ldi-extensions/pom.xml +++ b/ldi-extensions/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT be.vlaanderen.informatievlaanderen.ldes.ldi diff --git a/ldi-nifi/ldi-nifi-common/pom.xml b/ldi-nifi/ldi-nifi-common/pom.xml index 4094129aa..1d6890948 100644 --- a/ldi-nifi/ldi-nifi-common/pom.xml +++ b/ldi-nifi/ldi-nifi-common/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-nifi - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml b/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml index 445063330..667b56b7e 100644 --- a/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT archive-file-in diff --git a/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml b/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml index fb9bbc796..c7bd29879 100644 --- a/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT archive-file-out diff --git a/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml index c2ce1b1db..a6f8cfcfa 100644 --- a/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml index 3f783a002..46f7626d2 100644 --- a/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT geojson-to-wkt-processor diff --git a/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml index 3bd415d67..68fac0e64 100644 --- a/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT json-to-ld-processor diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml index 429f541d1..668b4c7d2 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml index 90c8ff1e2..37aacee46 100644 --- a/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml @@ -7,7 +7,7 @@ ldi-nifi-processors be.vlaanderen.informatievlaanderen.ldes.ldi.nifi - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT ngsiv2-to-ld-processor diff --git a/ldi-nifi/ldi-nifi-processors/pom.xml b/ldi-nifi/ldi-nifi-processors/pom.xml index 3e3ad1dbc..60da9d89e 100644 --- a/ldi-nifi/ldi-nifi-processors/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-nifi - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/pom.xml index 4bdd37a1f..61339c851 100644 --- a/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT rdf4j-repository-materialisation-processor diff --git a/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml index 9a1ca0181..f7e0651f3 100644 --- a/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml index 175958140..d33a379ca 100644 --- a/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/pom.xml b/ldi-nifi/pom.xml index b9ebad52d..9a7c496fb 100644 --- a/ldi-nifi/pom.xml +++ b/ldi-nifi/pom.xml @@ -3,7 +3,7 @@ linked-data-interactions be.vlaanderen.informatievlaanderen.ldes - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 pom diff --git a/ldi-orchestrator/ldio-application/pom.xml b/ldi-orchestrator/ldio-application/pom.xml index 838ccdd39..ae6ef7502 100644 --- a/ldi-orchestrator/ldio-application/pom.xml +++ b/ldi-orchestrator/ldio-application/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-orchestrator - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-common/pom.xml b/ldi-orchestrator/ldio-common/pom.xml index 24684abf7..c3de76ca7 100644 --- a/ldi-orchestrator/ldio-common/pom.xml +++ b/ldi-orchestrator/ldio-common/pom.xml @@ -5,7 +5,7 @@ ldi-orchestrator be.vlaanderen.informatievlaanderen.ldes.ldi - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml index cc5bbea3b..04bf8506a 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml @@ -6,7 +6,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT ldio-amqp diff --git a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml index ac80b568b..aef2490e5 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT ldio-archive-file-in diff --git a/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml index 7ce87bb6a..62c774e50 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT diff --git a/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml index 0adaac49c..8b858844a 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml index 21b8b0578..50d954b7e 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml @@ -6,7 +6,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT ldio-file-out diff --git a/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml index 797107211..f7745cf8c 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT ldio-geojson-to-wkt diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml index c38d1491c..35b8af2ed 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml index b6f4b2c66..5d7d3f716 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT ldio-http-in-poller diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml index 042ef0d61..8592214eb 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml @@ -4,7 +4,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml index 03f2e453f..31dac60e2 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml @@ -3,7 +3,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml index d9954ac86..557c3c519 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml index 96f5ab1f5..6fa14f59b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml index b4dba2581..40cfe4b93 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml index dbf5e901d..3269b8375 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml index ca4045bff..efcd37f04 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml @@ -6,7 +6,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT ldio-ngsiv2-to-ld-adapter @@ -15,7 +15,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ngsiv2-to-ld-adapter - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT compile diff --git a/ldi-orchestrator/ldio-connectors/ldio-noop-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-noop-out/pom.xml index b2f28143c..9ef3d5073 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-noop-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-noop-out/pom.xml @@ -4,7 +4,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml index af70a270b..df3ec9738 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/pom.xml index aa1074921..629338db6 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT ldio-repository-materialiser diff --git a/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml index c960299ad..8d4458479 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT ldio-request-executor diff --git a/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml index 4bd80236e..2c108101b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml index a6912d41e..ba2f75903 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml index 0b3771733..eab79311b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml @@ -3,7 +3,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml index 857cccba5..2c994abff 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml @@ -3,7 +3,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/pom.xml b/ldi-orchestrator/ldio-connectors/pom.xml index abeddf3f1..e4a658be0 100644 --- a/ldi-orchestrator/ldio-connectors/pom.xml +++ b/ldi-orchestrator/ldio-connectors/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-orchestrator - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/pom.xml b/ldi-orchestrator/pom.xml index 6f30f2899..5c74d3382 100644 --- a/ldi-orchestrator/pom.xml +++ b/ldi-orchestrator/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/pom.xml b/pom.xml index 511ee3e21..36f0edb6e 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions pom - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT ldi-api From 8e257912f9428dfae0a256f149ffbf41cda0580f Mon Sep 17 00:00:00 2001 From: Jonas Bulcke <127748878+jobulcke@users.noreply.github.com> Date: Wed, 27 Mar 2024 17:59:47 +0100 Subject: [PATCH 02/20] fix: repository materialiser deletion bugs (#557) --- .../inmemory/InMemoryMemberRepository.java | 2 +- ldi-core/repository-materialiser/pom.xml | 4 - .../ldes/ldi/Materialiser.java | 53 +++++------ .../valueobjects/MaterialiserConnection.java | 28 +++--- .../ldes/ldi/MaterialiserIT.java | 88 ++++++++++++++----- .../ldes/ldi/MaterialiserTest.java | 46 ++++++---- .../src/test/resources/movies/1.ttl | 38 ++++++++ .../src/test/resources/movies/2.ttl | 34 +++++++ .../ldes/ldio/LdioRepositoryMaterialiser.java | 27 +++--- 9 files changed, 224 insertions(+), 96 deletions(-) create mode 100644 ldi-core/repository-materialiser/src/test/resources/movies/1.ttl create mode 100644 ldi-core/repository-materialiser/src/test/resources/movies/2.ttl diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/inmemory/InMemoryMemberRepository.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/inmemory/InMemoryMemberRepository.java index 3cc5c2bf1..70611eb6a 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/inmemory/InMemoryMemberRepository.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/inmemory/InMemoryMemberRepository.java @@ -20,7 +20,7 @@ public Optional getTreeMember() { @Override public void deleteMember(MemberRecord member) { - members.remove(); + members.poll(); } @Override diff --git a/ldi-core/repository-materialiser/pom.xml b/ldi-core/repository-materialiser/pom.xml index 0bd227f44..6c03010c6 100644 --- a/ldi-core/repository-materialiser/pom.xml +++ b/ldi-core/repository-materialiser/pom.xml @@ -59,10 +59,6 @@ org.eclipse.rdf4j rdf4j-rio-trix - - org.eclipse.rdf4j - rdf4j-rio-turtle - org.eclipse.rdf4j rdf4j-rio-n3 diff --git a/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/Materialiser.java b/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/Materialiser.java index aec100cc2..1887289e8 100644 --- a/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/Materialiser.java +++ b/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/Materialiser.java @@ -6,9 +6,12 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.MaterialiserConnection; import org.apache.jena.rdf.model.Model; import org.eclipse.rdf4j.model.Resource; +import org.eclipse.rdf4j.model.Value; import org.eclipse.rdf4j.repository.manager.RemoteRepositoryManager; import org.eclipse.rdf4j.repository.manager.RepositoryManager; +import java.util.ArrayDeque; +import java.util.Deque; import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -32,8 +35,7 @@ public void process(List jenaModels) { synchronized (materialiserConnection) { try { jenaModels.stream().map(JenaToRDF4JConverter::convert).forEach(updateModel -> { - Set entityIds = ModelSubjectsExtractor.extractSubjects(updateModel); - deleteEntities(entityIds); + deleteEntity(updateModel); materialiserConnection.add(updateModel); }); materialiserConnection.commit(); @@ -52,37 +54,28 @@ public void shutdown() { materialiserConnection.shutdown(); } - protected void deleteEntities(Set subjects) { - subjects.forEach(this::deleteEntity); + protected void deleteEntity(org.eclipse.rdf4j.model.Model model) { + getAllSubjectsFromModel(model) + .forEach(subject -> materialiserConnection.remove(subject, null, null)); } - protected void deleteEntity(Resource subject) { - final String query = """ - DELETE { - ?s ?p ?o . - ?blankNode ?blankNodePredicate ?blankNodeObject . - } - WHERE { - ?s ?p ?o . - FILTER(?s = <%s>) + private Set getAllSubjectsFromModel(org.eclipse.rdf4j.model.Model model) { + final Set subjects = ModelSubjectsExtractor.extractSubjects(model); + final Deque subjectStack = new ArrayDeque<>(subjects); - OPTIONAL { - { - ?o ?blankNodePredicate ?blankNodeObject . - FILTER(isBlank(?o)) - BIND(?o AS ?blankNode) - } - UNION - { - ?o ?nestedPredicate ?nestedObject . - FILTER(isBlank(?nestedObject)) - ?nestedObject ?blankNodePredicate ?blankNodeObject . - BIND(?nestedObject AS ?blankNode) - } - } - } - """.formatted(subject.stringValue()); + while (!subjectStack.isEmpty()) { + Resource subject = subjectStack.pop(); - materialiserConnection.executeUpdate(query); + materialiserConnection.getStatements(subject, null, null).forEach(statement -> { + Value object = statement.getObject(); + if (object.isBNode()) { + Resource bnode = (Resource) object; + subjectStack.push(bnode); + subjects.add(bnode); + } + }); + + } + return subjects; } } diff --git a/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/MaterialiserConnection.java b/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/MaterialiserConnection.java index 715240d00..61acbdacb 100644 --- a/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/MaterialiserConnection.java +++ b/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/MaterialiserConnection.java @@ -6,6 +6,8 @@ import org.eclipse.rdf4j.repository.RepositoryResult; import org.eclipse.rdf4j.repository.manager.RepositoryManager; +import java.util.Optional; + public class MaterialiserConnection { private final String namedGraph; private final RepositoryConnectionHolder holder; @@ -15,14 +17,10 @@ public MaterialiserConnection(RepositoryManager repositoryManager, String reposi this.holder = new RepositoryConnectionHolder(repositoryManager, repositoryId); } - public void add(Model model) { - if (namedGraph != null && !namedGraph.isEmpty()) { - var namedGraphIRI = SimpleValueFactory.getInstance().createIRI(namedGraph); - holder.getConnection().add(model, namedGraphIRI); - } else { - holder.getConnection().add(model); - } + getNamedGraphIri().ifPresentOrElse( + namedGraphIri -> holder.getConnection().add(model, namedGraphIri), + () -> holder.getConnection().add(model)); } public RepositoryResult getStatements(Resource subject, IRI predicate, Value object) { @@ -30,7 +28,10 @@ public RepositoryResult getStatements(Resource subject, IRI predicate } public void remove(Resource subject, IRI predicate, Value object) { - holder.getConnection().remove(subject, predicate, object); + getNamedGraphIri().ifPresentOrElse( + namedGraphIri -> holder.getConnection().remove(subject, predicate, object, namedGraphIri), + () -> holder.getConnection().remove(subject, predicate, object) + ); } public void commit() { @@ -45,11 +46,14 @@ public void rollback() { connection.close(); } - public void executeUpdate(String query) { - holder.getConnection().prepareUpdate(query).execute(); - } - public void shutdown() { holder.shutdown(); } + + private Optional getNamedGraphIri() { + return Optional + .ofNullable(namedGraph) + .filter(graph -> !graph.isEmpty()) + .map(SimpleValueFactory.getInstance()::createIRI); + } } diff --git a/ldi-core/repository-materialiser/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/MaterialiserIT.java b/ldi-core/repository-materialiser/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/MaterialiserIT.java index 2410e8fb2..87c5803c4 100644 --- a/ldi-core/repository-materialiser/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/MaterialiserIT.java +++ b/ldi-core/repository-materialiser/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/MaterialiserIT.java @@ -2,9 +2,10 @@ import org.apache.jena.riot.RDFParser; import org.eclipse.rdf4j.model.Model; -import org.eclipse.rdf4j.model.Resource; import org.eclipse.rdf4j.model.Statement; +import org.eclipse.rdf4j.model.impl.LinkedHashModel; import org.eclipse.rdf4j.model.impl.SimpleValueFactory; +import org.eclipse.rdf4j.model.util.Models; import org.eclipse.rdf4j.repository.config.RepositoryConfig; import org.eclipse.rdf4j.repository.manager.LocalRepositoryManager; import org.eclipse.rdf4j.repository.manager.RepositoryManager; @@ -14,38 +15,37 @@ import org.eclipse.rdf4j.sail.memory.config.MemoryStoreConfig; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; +import org.junit.jupiter.params.provider.ArgumentsSource; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; class MaterialiserIT { - private static final String LOCAL_SERVER_URL = "http://localhost:8080/rdf4j-server"; private static final String LOCAL_REPOSITORY_ID = "test"; private static Materialiser materialiser; + private RepositoryManager subject; @TempDir File dataDir; @BeforeEach public void setUp() { - materialiser = new Materialiser(LOCAL_SERVER_URL, LOCAL_REPOSITORY_ID, ""); - final RepositoryManager subject = new LocalRepositoryManager(dataDir); + subject = new LocalRepositoryManager(dataDir); subject.init(); subject.addRepositoryConfig( new RepositoryConfig(LOCAL_REPOSITORY_ID, new SailRepositoryConfig(new MemoryStoreConfig(true)))); - - materialiser = new Materialiser(subject, LOCAL_REPOSITORY_ID, ""); } @AfterEach @@ -53,27 +53,31 @@ public void tearDown() { materialiser.shutdown(); } - @Test - void when_DeleteEntities_Then_EntitiesRemovedFromStore() throws Exception { + @ParameterizedTest + @ArgumentsSource(NamedGraphProvider.class) + void when_DeleteEntities_Then_EntitiesRemovedFromStore(String namedGraph) throws Exception { + materialiser = new Materialiser(subject, LOCAL_REPOSITORY_ID, namedGraph); + + final int statementOfModel2And3Count = 9; final List testFiles = IntStream.range(1, 4).mapToObj("src/test/resources/people/%d.nq"::formatted).toList(); - populateAndCheckRepository(testFiles); - final Model model = Rio.parse(new FileInputStream("src/test/resources/people/1.nq"), "", RDFFormat.NQUADS); - Set entityIds = Stream.of("http://somewhere/BeckySmith/", "http://somewhere/SarahJones/") - .map(iri -> SimpleValueFactory.getInstance().createIRI(iri)) - .collect(Collectors.toSet()); + populateAndCheckRepository(testFiles, namedGraph); + final Model modelToDelete = Rio.parse(new FileInputStream("src/test/resources/people/1.nq"), "", RDFFormat.NQUADS); - materialiser.deleteEntities(entityIds); + materialiser.deleteEntity(modelToDelete); List statements = materialiser.getMaterialiserConnection() .getStatements(null, null, null).stream().toList(); - assertThat(statements).hasSize(4); + assertThat(statements).hasSize(statementOfModel2And3Count); } - @Test - void when_UpdateEntities_Then_OldTriplesRemoved() throws Exception { + @ParameterizedTest + @ArgumentsSource(NamedGraphProvider.class) + void when_UpdateEntities_Then_OldTriplesRemoved(String namedGraph) throws Exception { + materialiser = new Materialiser(subject, LOCAL_REPOSITORY_ID, namedGraph); + final List testFiles = IntStream.range(1, 6).mapToObj("src/test/resources/people/%d.nq"::formatted).toList(); - populateAndCheckRepository(testFiles); + populateAndCheckRepository(testFiles, namedGraph); List models = List.of(RDFParser.source("people/5-updated.nq").toModel()); materialiser.process(models); @@ -87,7 +91,31 @@ void when_UpdateEntities_Then_OldTriplesRemoved() throws Exception { .noneMatch(statement -> statement.getObject().stringValue().equals("Taylor")); } - void populateAndCheckRepository(List files) throws IOException { + + @ParameterizedTest + @ArgumentsSource(NamedGraphProvider.class) + void given_ComplexModelToUpdate_test_Process(String namedGraph) throws IOException { + materialiser = new Materialiser(subject, LOCAL_REPOSITORY_ID, namedGraph); + + final org.apache.jena.rdf.model.Model originalModel = RDFParser.source("movies/1.ttl").toModel(); + materialiser.process(List.of(originalModel)); + + final org.apache.jena.rdf.model.Model updatedJenaModel = RDFParser.source("movies/2.ttl").toModel(); + final Model updatedRdfModel = Rio.parse(new FileInputStream("src/test/resources/movies/2.ttl"), "", RDFFormat.TURTLE); + materialiser.process(List.of(updatedJenaModel)); + + + Model updatedDbModel = new LinkedHashModel(); + materialiser.getMaterialiserConnection() + .getStatements(null, null, null) + .stream() + .map(MaterialiserIT::deleteContextFromStatement) + .forEach(updatedDbModel::add); + + assertThat(Models.isomorphic(updatedDbModel, updatedRdfModel)).isTrue(); + } + + void populateAndCheckRepository(List files, String namedGraph) throws IOException { List models = new ArrayList<>(); for (String testFile : files) { var model = Rio.parse(new FileInputStream(testFile), "", RDFFormat.NQUADS); @@ -96,8 +124,24 @@ void populateAndCheckRepository(List files) throws IOException { } List statements = materialiser.getMaterialiserConnection().getStatements(null, null, null).stream().toList(); + if (!namedGraph.isEmpty()) { + statements = statements.stream() + .map(MaterialiserIT::deleteContextFromStatement) + .toList(); + } assertThat(models).allMatch(statements::containsAll); } + private static Statement deleteContextFromStatement(Statement statement) { + return SimpleValueFactory.getInstance().createStatement(statement.getSubject(), statement.getPredicate(), statement.getObject()); + } + + static class NamedGraphProvider implements ArgumentsProvider { + @Override + public Stream provideArguments(ExtensionContext extensionContext) { + return Stream.of("", "http://example.org/my-named-graph").map(Arguments::of); + } + } + } diff --git a/ldi-core/repository-materialiser/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/MaterialiserTest.java b/ldi-core/repository-materialiser/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/MaterialiserTest.java index b2f758100..f056a4d18 100644 --- a/ldi-core/repository-materialiser/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/MaterialiserTest.java +++ b/ldi-core/repository-materialiser/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/MaterialiserTest.java @@ -11,7 +11,9 @@ import org.eclipse.rdf4j.repository.Repository; import org.eclipse.rdf4j.repository.RepositoryConnection; import org.eclipse.rdf4j.repository.RepositoryException; +import org.eclipse.rdf4j.repository.RepositoryResult; import org.eclipse.rdf4j.repository.manager.RepositoryManager; +import org.eclipse.rdf4j.repository.sparql.federation.CollectionIteration; import org.eclipse.rdf4j.rio.RDFFormat; import org.eclipse.rdf4j.rio.Rio; import org.junit.jupiter.api.AfterEach; @@ -41,8 +43,6 @@ class MaterialiserTest { private RepositoryManager repositoryManager; @Mock private Repository repository; - @Mock - private Update update; private Materialiser materialiser; @BeforeEach @@ -50,7 +50,6 @@ void setUp() { when(repositoryManager.getRepository(REPOSITORY_ID)).thenReturn(repository); when(repository.getConnection()).thenReturn(connection); materialiser = new Materialiser(repositoryManager, REPOSITORY_ID, ""); - when(connection.prepareUpdate(anyString())).thenReturn(update); } @AfterEach @@ -63,24 +62,33 @@ void when_DeleteEntities_Then_EntitiesRemovedFromStore() throws Exception { when(repositoryManager.getRepository(REPOSITORY_ID)).thenReturn(repository); when(repository.getConnection()).thenReturn(connection); Set entityIds = Stream - .of("http://somewhere/SarahJones/", "http://somewhere/MattJones/") + .of("http://somewhere/DickJones/", "http://somewhere/SarahJones/") .map(stringUri -> SimpleValueFactory.getInstance().createIRI(stringUri)) .collect(Collectors.toSet()); + when(connection.getStatements(any(), isNull(), isNull())) + .thenReturn(new RepositoryResult<>(new CollectionIteration<>(Set.of()))); - materialiser.deleteEntities(entityIds); + Model modelToDelete = Rio.parse(new FileInputStream(CHANGED_FILE), "", RDFFormat.NQUADS); - verify(update, times(2)).execute(); + materialiser.deleteEntity(modelToDelete); + entityIds.forEach(subjectIri -> { + verify(connection).getStatements(subjectIri, null, null); + verify(connection).remove(subjectIri, null, null); + }); } @Test void given_RepositoryContainingData_when_UpdateSingleModel_Then_OldTriplesRemoved() throws Exception { when(repositoryManager.getRepository(REPOSITORY_ID)).thenReturn(repository); when(repository.getConnection()).thenReturn(connection); + when(connection.getStatements(any(), isNull(), isNull())) + .thenReturn(new RepositoryResult<>(new CollectionIteration<>(Set.of()))); Model changedModel = Rio.parse(new FileInputStream(CHANGED_FILE), "", RDFFormat.NQUADS); materialiser.process(List.of(RDFParser.source(CHANGED_FILE).toModel())); - verify(update, times(2)).execute(); + verify(connection).remove(SimpleValueFactory.getInstance().createIRI("http://somewhere/DickJones/"), null, null); + verify(connection).remove(SimpleValueFactory.getInstance().createIRI("http://somewhere/SarahJones/"), null, null); verify(connection).add(changedModel); verify(connection).commit(); } @@ -89,12 +97,14 @@ void given_RepositoryContainingData_when_UpdateSingleModel_Then_OldTriplesRemove void given_ValidListOfMembers_when_ProcessList_then_CommitToRepository() { when(repositoryManager.getRepository(REPOSITORY_ID)).thenReturn(repository); when(repository.getConnection()).thenReturn(connection); + when(connection.getStatements(any(), isNull(), isNull())) + .thenReturn(new RepositoryResult<>(new CollectionIteration<>(Set.of()))); List models = readTenModelsFromFile().toList(); materialiser.process(models); - verify(update, times(10)).execute(); + verify(connection, times(10)).remove(any(Resource.class), isNull(), isNull()); verify(connection, times(10)).add(any(Model.class)); verify(connection).commit(); } @@ -103,6 +113,8 @@ void given_ValidListOfMembers_when_ProcessList_then_CommitToRepository() { void given_ValidList_when_ProcessList_and_CommitFails_then_RollbackConnection() { when(repositoryManager.getRepository(REPOSITORY_ID)).thenReturn(repository); when(repository.getConnection()).thenReturn(connection); + when(connection.getStatements(any(), isNull(), isNull())) + .thenReturn(new RepositoryResult<>(new CollectionIteration<>(Set.of()))); doThrow(RepositoryException.class).when(connection).commit(); List models = readTenModelsFromFile().toList(); @@ -111,7 +123,6 @@ void given_ValidList_when_ProcessList_and_CommitFails_then_RollbackConnection() .isInstanceOf(MaterialisationFailedException.class) .hasCauseInstanceOf(RepositoryException.class); - verify(update, times(10)).execute(); verify(connection, times(10)).add(any(Model.class)); verify(connection).commit(); verify(connection).rollback(); @@ -121,6 +132,8 @@ void given_ValidList_when_ProcessList_and_CommitFails_then_RollbackConnection() void given_ListOfMembers_when_ProcessList_and_AddModelToConnectionFails_then_StopAdditionAndRollback() { when(repositoryManager.getRepository(REPOSITORY_ID)).thenReturn(repository); when(repository.getConnection()).thenReturn(connection); + when(connection.getStatements(any(), isNull(), isNull())) + .thenReturn(new RepositoryResult<>(new CollectionIteration<>(Set.of()))); doNothing().doNothing().doThrow(RepositoryException.class).when(connection).add(any(Model.class)); List models = readTenModelsFromFile().toList(); @@ -136,11 +149,11 @@ void given_ListOfMembers_when_ProcessList_and_AddModelToConnectionFails_then_Sto @Test void given_ListOfMembers_when_ProcessList_and_GetStatementsFails_then_StopAdditionAndRollback() { - doNothing() - .doNothing() - .doNothing() - .doThrow(RepositoryException.class) - .when(update).execute(); + when(connection.getStatements(any(), isNull(), isNull())) + .thenReturn(new RepositoryResult<>(new CollectionIteration<>(Set.of()))) + .thenReturn(new RepositoryResult<>(new CollectionIteration<>(Set.of()))) + .thenReturn(new RepositoryResult<>(new CollectionIteration<>(Set.of()))) + .thenThrow(RepositoryException.class); List models = readTenModelsFromFile().toList(); @@ -148,11 +161,10 @@ void given_ListOfMembers_when_ProcessList_and_GetStatementsFails_then_StopAdditi .isInstanceOf(MaterialisationFailedException.class) .hasCauseInstanceOf(RepositoryException.class); - verify(update, times(4)).execute(); - verify(connection, times(3)).add(any(Model.class)); + verify(connection, times(4)).getStatements(any(), isNull(), isNull()); + verify(connection, times(3)).remove(any(Resource.class), isNull(), isNull()); verify(connection, times(3)).add(any(Model.class)); verify(connection, never()).commit(); verify(connection).rollback(); - } private Stream readTenModelsFromFile() { diff --git a/ldi-core/repository-materialiser/src/test/resources/movies/1.ttl b/ldi-core/repository-materialiser/src/test/resources/movies/1.ttl new file mode 100644 index 000000000..3dc99cc57 --- /dev/null +++ b/ldi-core/repository-materialiser/src/test/resources/movies/1.ttl @@ -0,0 +1,38 @@ +@prefix schema: . +@prefix rdf: . +@prefix skos: . + + a schema:Movie ; + schema:name "Dune" ; + schema:description "An epic adaptation of Frank Herbert's sci-fi novel." ; + schema:datePublished "2021-09-15" ; + schema:director [ + a schema:Person ; + schema:name "Denis Villeneuve" ; + schema:role [ + a skos:Concept ; + skos:prefLabel "Director" ; + skos:broader [ + a skos:Concept ; + skos:prefLabel "Person" + ] + ] + ] ; + schema:actor [ + a schema:Person ; + schema:name "Timothée Chalamet" ; + schema:role [ + a skos:Concept ; + skos:prefLabel "Actor" + ] + ], [ + a schema:Person ; + schema:name "Rebecca Ferguson" ; + schema:role [ + a skos:Concept ; + skos:prefLabel "Actor" + ] + ] ; + schema:genre "Science Fiction", "Adventure" ; + schema:duration "PT155M" ; + schema:language "English" . \ No newline at end of file diff --git a/ldi-core/repository-materialiser/src/test/resources/movies/2.ttl b/ldi-core/repository-materialiser/src/test/resources/movies/2.ttl new file mode 100644 index 000000000..b451f46af --- /dev/null +++ b/ldi-core/repository-materialiser/src/test/resources/movies/2.ttl @@ -0,0 +1,34 @@ +@prefix schema: . +@prefix rdf: . +@prefix skos: . + + a schema:Movie ; + schema:name "Dune: Part Two" ; + schema:description "An epic adaptation of Frank Herbert's sci-fi novel." ; + schema:datePublished "2024-02-28" ; + schema:director [ + a schema:Person ; + schema:name "Denis Villeneuve" ; + schema:role [ + a skos:Concept ; + skos:prefLabel "Director" + ] + ] ; + schema:actor [ + a schema:Person ; + schema:name "Timothée Chalamet" ; + schema:role [ + a skos:Concept ; + skos:prefLabel "Actor" + ] + ], [ + a schema:Person ; + schema:name "Rebecca Ferguson" ; + schema:role [ + a skos:Concept ; + skos:prefLabel "Actor" + ] + ] ; + schema:genre "Science Fiction", "Adventure" ; + schema:duration "PT167M" ; + schema:language "English" . \ No newline at end of file diff --git a/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioRepositoryMaterialiser.java b/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioRepositoryMaterialiser.java index 20528c6ff..896307ed1 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioRepositoryMaterialiser.java +++ b/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioRepositoryMaterialiser.java @@ -9,7 +9,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.time.LocalDateTime; +import java.io.File; +import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.List; import java.util.concurrent.ScheduledExecutorService; @@ -55,24 +56,30 @@ public void shutdown() { public synchronized void sendToMaterialiser() { if (!membersToCommit.isEmpty()) { - materialiser.processAsync(List.copyOf(membersToCommit)) + final List members = List.copyOf(membersToCommit); + materialiser.processAsync(members) .exceptionally(throwable -> { - handleException(throwable); + handleException(throwable, members); return null; }); membersToCommit.clear(); } } - private void handleException(Throwable throwable) { + private void handleException(Throwable throwable, List failedMembers) { log.atError().log(ERROR_TEMPLATE, "sendToMaterialiser", throwable.getMessage()); - final String fileName = "/tmp/materialisation/uncommitted-members-%s.ttl".formatted(LocalDateTime.now()); final Model uncommitedMembersModel = ModelFactory.createDefaultModel(); - membersToCommit.forEach(uncommitedMembersModel::add); - RDFWriter.source(uncommitedMembersModel) - .lang(Lang.TURTLE) - .output(fileName); - log.atError().log("Uncommitted members can be found in file: {}", fileName); + failedMembers.forEach(uncommitedMembersModel::add); + final File materialisationFolder = new File("materialisation"); + if (materialisationFolder.exists() || materialisationFolder.mkdir()) { + final String fileName = "uncommitted-members-%s.ttl".formatted(ZonedDateTime.now().toEpochSecond()); + final File uncommittedMembersFile = new File(materialisationFolder, fileName); + RDFWriter.source(uncommitedMembersModel).lang(Lang.TURTLE).output(uncommittedMembersFile.getPath()); + log.atError().log("Uncommitted members can be found in file: {}", uncommittedMembersFile.getAbsolutePath()); + } else { + final String uncommittedMembers = RDFWriter.source(uncommitedMembersModel).lang(Lang.TURTLE).asString(); + log.atError().log("Unable to commit the following members: \n {}", uncommittedMembers); + } } private void resetExecutor() { From 84d8d607d891fdef6112d7fc129efcb8b604bc15 Mon Sep 17 00:00:00 2001 From: Jan Robert <15772440+Yalz@users.noreply.github.com> Date: Wed, 27 Mar 2024 18:29:08 +0100 Subject: [PATCH 03/20] fix: #539 #540 : pick up ITs + fix broken Request Executor IT (#560) Co-authored-by: Yalz --- docs/docker-compose.yml | 11 ++++++++++- .../ldes/ldi/requestexecutor/RequestExecutorIT.java | 3 +-- .../src/test/resources/cucumber.properties | 0 .../src/test/resources/mappings/200-post-token.json | 8 ++++---- pom.xml | 1 + 5 files changed, 16 insertions(+), 7 deletions(-) delete mode 100644 ldi-core/request-executor/src/test/resources/cucumber.properties diff --git a/docs/docker-compose.yml b/docs/docker-compose.yml index 3c10890fa..52f9d00e6 100644 --- a/docs/docker-compose.yml +++ b/docs/docker-compose.yml @@ -4,4 +4,13 @@ services: ports: - 4000:4000 volumes: - - .:/site \ No newline at end of file + - .:/site + + postgres: + image: postgres:14-alpine + ports: + - 5432:5432 + environment: + - POSTGRES_PASSWORD=admin + - POSTGRES_USER=admin + - POSTGRES_DB=db \ No newline at end of file diff --git a/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/RequestExecutorIT.java b/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/RequestExecutorIT.java index 3c3f95a1c..82c57fcbd 100644 --- a/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/RequestExecutorIT.java +++ b/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/RequestExecutorIT.java @@ -1,13 +1,12 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor; -import org.junit.platform.suite.api.ConfigurationParameter; import org.junit.platform.suite.api.IncludeEngines; import org.junit.platform.suite.api.SelectClasspathResource; import org.junit.platform.suite.api.Suite; @Suite +@SuppressWarnings("java:S2187") @IncludeEngines("cucumber") @SelectClasspathResource("features") -@ConfigurationParameter(key = "be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor", value = "be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor") public class RequestExecutorIT { } diff --git a/ldi-core/request-executor/src/test/resources/cucumber.properties b/ldi-core/request-executor/src/test/resources/cucumber.properties deleted file mode 100644 index e69de29bb..000000000 diff --git a/ldi-core/request-executor/src/test/resources/mappings/200-post-token.json b/ldi-core/request-executor/src/test/resources/mappings/200-post-token.json index 6f4b4c3e2..36822f6fb 100644 --- a/ldi-core/request-executor/src/test/resources/mappings/200-post-token.json +++ b/ldi-core/request-executor/src/test/resources/mappings/200-post-token.json @@ -2,11 +2,11 @@ "request": { "method": "POST", "url": "/token", - "headers": { - "Authorization": { - "contains": "Basic Y2xpZW50SWQ6Y2xpZW50U2VjcmV0" + "bodyPatterns": [ + { + "contains": "client_id=clientId&client_secret=clientSecret&scope=default&grant_type=client_credentials" } - } + ] }, "response": { "status": 200, diff --git a/pom.xml b/pom.xml index 36f0edb6e..76f835123 100644 --- a/pom.xml +++ b/pom.xml @@ -165,6 +165,7 @@ integration-test + verify From 82a16c735ac7c0217bfacfd88ebf1c225724ec25 Mon Sep 17 00:00:00 2001 From: Jonas Bulcke <127748878+jobulcke@users.noreply.github.com> Date: Thu, 28 Mar 2024 09:14:27 +0100 Subject: [PATCH 04/20] fix: repo materialiser lang tag bug (#559) --- .../ldes/ldi/services/JenaToRDF4JConverter.java | 12 +++++++----- .../src/test/resources/movies/1.ttl | 8 +++++--- .../src/test/resources/movies/2.ttl | 8 +++++--- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/services/JenaToRDF4JConverter.java b/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/services/JenaToRDF4JConverter.java index e7d6fcd01..ae20817b3 100644 --- a/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/services/JenaToRDF4JConverter.java +++ b/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/services/JenaToRDF4JConverter.java @@ -1,10 +1,8 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.services; +import org.apache.jena.rdf.model.Literal; import org.apache.jena.rdf.model.RDFNode; -import org.eclipse.rdf4j.model.IRI; -import org.eclipse.rdf4j.model.Model; -import org.eclipse.rdf4j.model.Resource; -import org.eclipse.rdf4j.model.Value; +import org.eclipse.rdf4j.model.*; import org.eclipse.rdf4j.model.impl.LinkedHashModel; import org.eclipse.rdf4j.model.impl.SimpleValueFactory; @@ -43,7 +41,11 @@ private static Value convertValue(RDFNode node) { org.apache.jena.rdf.model.Resource resource = (org.apache.jena.rdf.model.Resource) node; return convertResource(resource); } else { - return SimpleValueFactory.getInstance().createLiteral(node.toString()); + Literal literal = node.asLiteral(); + if(literal.getLanguage() != null && !literal.getLanguage().isEmpty()) { + return SimpleValueFactory.getInstance().createLiteral(literal.getValue().toString(), literal.getLanguage()); + } + return SimpleValueFactory.getInstance().createLiteral(literal.getValue().toString(), SimpleValueFactory.getInstance().createIRI(literal.getDatatypeURI())); } } } diff --git a/ldi-core/repository-materialiser/src/test/resources/movies/1.ttl b/ldi-core/repository-materialiser/src/test/resources/movies/1.ttl index 3dc99cc57..3cee8d5d9 100644 --- a/ldi-core/repository-materialiser/src/test/resources/movies/1.ttl +++ b/ldi-core/repository-materialiser/src/test/resources/movies/1.ttl @@ -1,11 +1,13 @@ @prefix schema: . @prefix rdf: . @prefix skos: . +@prefix xsd: . a schema:Movie ; schema:name "Dune" ; - schema:description "An epic adaptation of Frank Herbert's sci-fi novel." ; - schema:datePublished "2021-09-15" ; + schema:description "An epic adaptation of Frank Herbert's sci-fi novel."@en, "Een epische verfilming van de sci-fi novel van Frank Herbert."@nl ; + schema:datePublished "2021-09-15"^^xsd:date ; + schema:nominated true ; schema:director [ a schema:Person ; schema:name "Denis Villeneuve" ; @@ -34,5 +36,5 @@ ] ] ; schema:genre "Science Fiction", "Adventure" ; - schema:duration "PT155M" ; + schema:duration "PT155M"^^xsd:duration ; schema:language "English" . \ No newline at end of file diff --git a/ldi-core/repository-materialiser/src/test/resources/movies/2.ttl b/ldi-core/repository-materialiser/src/test/resources/movies/2.ttl index b451f46af..1c867b99b 100644 --- a/ldi-core/repository-materialiser/src/test/resources/movies/2.ttl +++ b/ldi-core/repository-materialiser/src/test/resources/movies/2.ttl @@ -1,11 +1,13 @@ @prefix schema: . @prefix rdf: . @prefix skos: . +@prefix xsd: . a schema:Movie ; schema:name "Dune: Part Two" ; - schema:description "An epic adaptation of Frank Herbert's sci-fi novel." ; - schema:datePublished "2024-02-28" ; + schema:description "An epic adaptation of Frank Herbert's sci-fi novel."@en, "Een epische verfilming van de sci-fi novel van Frank Herbert."@nl ; + schema:datePublished "2024-02-28"^^xsd:date ; + schema:nominated false ; schema:director [ a schema:Person ; schema:name "Denis Villeneuve" ; @@ -30,5 +32,5 @@ ] ] ; schema:genre "Science Fiction", "Adventure" ; - schema:duration "PT167M" ; + schema:duration "PT167M"^^xsd:duration ; schema:language "English" . \ No newline at end of file From 610b0c71ece326d309e4f378cc92039f33b2c9c4 Mon Sep 17 00:00:00 2001 From: Tomvbe <34196062+Tomvbe@users.noreply.github.com> Date: Thu, 28 Mar 2024 10:57:57 +0100 Subject: [PATCH 05/20] fix: improve logging of HttpPoller (#564) --- .../informatievlaanderen/ldes/ldio/LdioHttpInputPoller.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPoller.java b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPoller.java index 7cea91e35..5555f8889 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPoller.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPoller.java @@ -91,6 +91,8 @@ private void executeRequest(Request request) { String content = response.getBodyAsString().orElseThrow(); processInput(content, contentType); } else { + log.error("Failed to execute request {} {} {}", request.getMethod(), request.getUrl(), response.getHttpStatus()); + response.getBodyAsString().ifPresent(log::error); throw new UnsuccesfulPollingException(response.getHttpStatus(), request.getUrl()); } } From b0f7af36a54082161203cef66009efe397de8acb Mon Sep 17 00:00:00 2001 From: Yalz Date: Thu, 28 Mar 2024 12:01:38 +0100 Subject: [PATCH 06/20] docs: #546: incorrect LDIO titles --- docs/_ldio/ldio-adapters/index.md | 2 +- docs/_ldio/ldio-core/index.md | 2 +- docs/_ldio/ldio-inputs/index.md | 2 +- docs/_ldio/ldio-outputs/index.md | 2 +- docs/_ldio/ldio-transformers/index.md | 6 +----- 5 files changed, 5 insertions(+), 9 deletions(-) diff --git a/docs/_ldio/ldio-adapters/index.md b/docs/_ldio/ldio-adapters/index.md index 2fc0c4ece..7ac9edad2 100644 --- a/docs/_ldio/ldio-adapters/index.md +++ b/docs/_ldio/ldio-adapters/index.md @@ -6,7 +6,7 @@ has_toc: true nav_order: 4 --- -# Linked Data Adapters +# Linked Data Orchestrator Adapters The LDI Core module contains the components maintained by the VSDS team in order to accommodate the onboarding of LDES onboarders. diff --git a/docs/_ldio/ldio-core/index.md b/docs/_ldio/ldio-core/index.md index 09e859890..06697a9cb 100644 --- a/docs/_ldio/ldio-core/index.md +++ b/docs/_ldio/ldio-core/index.md @@ -6,6 +6,6 @@ has_toc: true nav_order: 2 --- -# LDIO Core +# Linked Data Orchestrator Core Contains common components used by multiple other LDIO components diff --git a/docs/_ldio/ldio-inputs/index.md b/docs/_ldio/ldio-inputs/index.md index 7f1884aa4..6c6fa668b 100644 --- a/docs/_ldio/ldio-inputs/index.md +++ b/docs/_ldio/ldio-inputs/index.md @@ -6,7 +6,7 @@ has_toc: true nav_order: 3 --- -# LDIO Inputs +# Linked Data Orchestrator Inputs The LDI Core module contains the components maintained by the VSDS team in order to accommodate the onboarding of LDES onboarders. diff --git a/docs/_ldio/ldio-outputs/index.md b/docs/_ldio/ldio-outputs/index.md index d21d8dd5f..91d002f85 100644 --- a/docs/_ldio/ldio-outputs/index.md +++ b/docs/_ldio/ldio-outputs/index.md @@ -6,7 +6,7 @@ has_toc: true nav_order: 6 --- -# Linked Data Adapters +# Linked Data Orchestrator Outputs The LDI Core module contains the components maintained by the VSDS team in order to accommodate the onboarding of LDES onboarders. diff --git a/docs/_ldio/ldio-transformers/index.md b/docs/_ldio/ldio-transformers/index.md index cb1103a0d..663022c6a 100644 --- a/docs/_ldio/ldio-transformers/index.md +++ b/docs/_ldio/ldio-transformers/index.md @@ -6,8 +6,4 @@ has_toc: true nav_order: 5 --- -# Linked Data Adapters - -The LDI Core module contains the components maintained by the VSDS team in order to accommodate the onboarding of LDES onboarders. - -Each component can be wrapped in a desired implementation framework (LDI-orchestrator, NiFi, ...) to be used. +# Linked Data Orchestrator Transformers \ No newline at end of file From 3d491367d1fd2ca7c3a4e3c64bbec4d56e2380ca Mon Sep 17 00:00:00 2001 From: Tomvbe <34196062+Tomvbe@users.noreply.github.com> Date: Thu, 28 Mar 2024 14:49:27 +0100 Subject: [PATCH 07/20] feat: implement redirect logic (#563) --- .../StartingTreeNodeFinderSteps.java | 2 +- .../StartingTreeNodeRelationsFinderSteps.java | 2 +- .../treenodefetcher/TreeNodeFetcherSteps.java | 2 +- .../LdesStructureDiscovererSteps.java | 2 +- .../LdesStructureDiscovererTest.java | 2 +- .../TreeRelationsFetcherSteps.java | 2 +- .../performance/TreeNodeProcessorFactory.java | 2 +- .../treenodesupplier/MemberSupplierSteps.java | 2 +- .../ClientCredentialsConfig.java | 8 ++++++-- .../executor/noauth/DefaultConfig.java | 14 ++++++++++---- .../services/RequestExecutorFactory.java | 15 +++++++++++++-- .../ldi/requestexecutor/RequestExecutorSteps.java | 2 +- .../discoverer/common/LdesDiscovererExecutor.java | 2 +- .../ldes/ldi/processors/LdesClientProcessor.java | 3 ++- .../config/LdioLdesClientConnectorAutoConfig.java | 4 +++- .../ldio/config/LdioLdesClientAutoConfig.java | 4 +++- 16 files changed, 47 insertions(+), 21 deletions(-) diff --git a/ldi-core/ldes-client/starting-node/src/test/java/ldes/client/startingtreenode/StartingTreeNodeFinderSteps.java b/ldi-core/ldes-client/starting-node/src/test/java/ldes/client/startingtreenode/StartingTreeNodeFinderSteps.java index 4847e48f6..abf26568d 100644 --- a/ldi-core/ldes-client/starting-node/src/test/java/ldes/client/startingtreenode/StartingTreeNodeFinderSteps.java +++ b/ldi-core/ldes-client/starting-node/src/test/java/ldes/client/startingtreenode/StartingTreeNodeFinderSteps.java @@ -15,7 +15,7 @@ public class StartingTreeNodeFinderSteps { - private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(); + private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(false); private StartingTreeNodeFinder startingTreeNodeFinder; private StartingNodeRequest startingNodeRequest; diff --git a/ldi-core/ldes-client/starting-node/src/test/java/ldes/client/startingtreenode/StartingTreeNodeRelationsFinderSteps.java b/ldi-core/ldes-client/starting-node/src/test/java/ldes/client/startingtreenode/StartingTreeNodeRelationsFinderSteps.java index 30c339dc6..6b3e48958 100644 --- a/ldi-core/ldes-client/starting-node/src/test/java/ldes/client/startingtreenode/StartingTreeNodeRelationsFinderSteps.java +++ b/ldi-core/ldes-client/starting-node/src/test/java/ldes/client/startingtreenode/StartingTreeNodeRelationsFinderSteps.java @@ -22,7 +22,7 @@ public class StartingTreeNodeRelationsFinderSteps { @Given("I have a StartingTreeNodeRelationsFinder") public void iHaveAStartingTreeNodeRelationsFinder() { - final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(); + final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(false); final RequestExecutor requestExecutor = requestExecutorFactory.createNoAuthExecutor(); startingTreeNodeRelationsFinder = new StartingTreeNodeRelationsFinder(requestExecutor); } diff --git a/ldi-core/ldes-client/tree-node-fetcher/src/test/java/ldes/client/treenodefetcher/TreeNodeFetcherSteps.java b/ldi-core/ldes-client/tree-node-fetcher/src/test/java/ldes/client/treenodefetcher/TreeNodeFetcherSteps.java index 8284671e4..846fd0c5b 100644 --- a/ldi-core/ldes-client/tree-node-fetcher/src/test/java/ldes/client/treenodefetcher/TreeNodeFetcherSteps.java +++ b/ldi-core/ldes-client/tree-node-fetcher/src/test/java/ldes/client/treenodefetcher/TreeNodeFetcherSteps.java @@ -15,7 +15,7 @@ public class TreeNodeFetcherSteps { - private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(); + private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(false); private TreeNodeFetcher treeNodeFetcher; private TreeNodeRequest treeNodeRequest; private TreeNodeResponse treeNodeResponse; diff --git a/ldi-core/ldes-client/tree-node-relations-fetcher/src/test/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscovererSteps.java b/ldi-core/ldes-client/tree-node-relations-fetcher/src/test/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscovererSteps.java index da3f4f23b..893acde45 100644 --- a/ldi-core/ldes-client/tree-node-relations-fetcher/src/test/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscovererSteps.java +++ b/ldi-core/ldes-client/tree-node-relations-fetcher/src/test/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscovererSteps.java @@ -17,7 +17,7 @@ public class LdesStructureDiscovererSteps { public void iHaveATreeDiscoverer() { final String startingUrl = "http://localhost:10101/200-first-4-relations"; final Lang sourceFormat = Lang.TURTLE; - ldesStructureDiscoverer = new LdesStructureDiscoverer(startingUrl, sourceFormat, new RequestExecutorFactory().createNoAuthExecutor()); + ldesStructureDiscoverer = new LdesStructureDiscoverer(startingUrl, sourceFormat, new RequestExecutorFactory(false).createNoAuthExecutor()); } @When("I start discovering the tree node relations") diff --git a/ldi-core/ldes-client/tree-node-relations-fetcher/src/test/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscovererTest.java b/ldi-core/ldes-client/tree-node-relations-fetcher/src/test/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscovererTest.java index 9ef3dbfff..f606fb1b3 100644 --- a/ldi-core/ldes-client/tree-node-relations-fetcher/src/test/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscovererTest.java +++ b/ldi-core/ldes-client/tree-node-relations-fetcher/src/test/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscovererTest.java @@ -30,7 +30,7 @@ class LdesStructureDiscovererTest { @BeforeEach void setUp() { - final RequestExecutor requestExecutor = new RequestExecutorFactory().createNoAuthExecutor(); + final RequestExecutor requestExecutor = new RequestExecutorFactory(false).createNoAuthExecutor(); discoverer = new LdesStructureDiscoverer("http://localhost:%d%s".formatted(WIREMOCK_PORT, STARTING_ENDPOINT), Lang.TURTLE, requestExecutor); } diff --git a/ldi-core/ldes-client/tree-node-relations-fetcher/src/test/java/ldes/client/treenoderelationsfetcher/TreeRelationsFetcherSteps.java b/ldi-core/ldes-client/tree-node-relations-fetcher/src/test/java/ldes/client/treenoderelationsfetcher/TreeRelationsFetcherSteps.java index 10efb39bd..aac9ccfac 100644 --- a/ldi-core/ldes-client/tree-node-relations-fetcher/src/test/java/ldes/client/treenoderelationsfetcher/TreeRelationsFetcherSteps.java +++ b/ldi-core/ldes-client/tree-node-relations-fetcher/src/test/java/ldes/client/treenoderelationsfetcher/TreeRelationsFetcherSteps.java @@ -15,7 +15,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; public class TreeRelationsFetcherSteps { - private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(); + private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(false); private TreeNodeRequest treeNodeRequest; private TreeRelationsFetcher treeRelationsFetcher; private List treeNodeRelations; diff --git a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/performance/TreeNodeProcessorFactory.java b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/performance/TreeNodeProcessorFactory.java index da33f41a7..df0be3d76 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/performance/TreeNodeProcessorFactory.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/performance/TreeNodeProcessorFactory.java @@ -23,7 +23,7 @@ class TreeNodeProcessorFactory { - private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(); + private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(false); TreeNodeProcessor createTreeNodeProcessor(StatePersistenceStrategy statePersistenceStrategy, List url, Lang sourceFormat) { final LdesMetaData ldesMetaData = new LdesMetaData(url, sourceFormat); diff --git a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/MemberSupplierSteps.java b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/MemberSupplierSteps.java index 378fe7188..1f870faf4 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/MemberSupplierSteps.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/MemberSupplierSteps.java @@ -30,7 +30,7 @@ import static org.junit.jupiter.api.Assertions.*; public class MemberSupplierSteps { - private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(); + private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(false); private TreeNodeProcessor treeNodeProcessor; private TreeNodeRecordRepository treeNodeRecordRepository; private MemberRepository memberRepository; diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsConfig.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsConfig.java index 530b461b2..60d2b3f76 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsConfig.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsConfig.java @@ -25,17 +25,20 @@ public class ClientCredentialsConfig implements RequestExecutorSupplier { private final String secret; private final String tokenEndpoint; private final String scope; + private final boolean enableRedirectHandling; public ClientCredentialsConfig(Collection
headers, String clientId, String secret, String tokenEndpoint, - String scope) { + String scope, + boolean enableRedirectHandling) { this.headers = headers; this.clientId = notNull(clientId); this.secret = notNull(secret); this.tokenEndpoint = notNull(tokenEndpoint); this.scope = scope; + this.enableRedirectHandling = enableRedirectHandling; } public RequestExecutor createRequestExecutor() { @@ -43,7 +46,8 @@ public RequestExecutor createRequestExecutor() { } private OAuth20Service createService() { - final RequestConfig clientConfig = RequestConfig.custom().setRedirectsEnabled(false).build(); + final RequestConfig.Builder configBuilder = RequestConfig.custom(); + final RequestConfig clientConfig = configBuilder.setRedirectsEnabled(enableRedirectHandling).build(); final ApacheHttpClient apacheHttpClient = new ApacheHttpClient( HttpAsyncClientBuilder.create().setDefaultRequestConfig(clientConfig).setDefaultHeaders(headers).build()); final DefaultApi20 authorizationApi = createAuthorizationApi(tokenEndpoint); diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/DefaultConfig.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/DefaultConfig.java index d3bd09a6d..eebba062b 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/DefaultConfig.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/DefaultConfig.java @@ -10,14 +10,20 @@ public class DefaultConfig implements RequestExecutorSupplier { private final Collection
headers; + private final boolean enableRedirectHandling; - public DefaultConfig(Collection
headers) { + public DefaultConfig(Collection
headers, boolean enableRedirectHandling) { this.headers = headers; - } + this.enableRedirectHandling = enableRedirectHandling; + } public RequestExecutor createRequestExecutor() { - return new DefaultRequestExecutor( - HttpClientBuilder.create().setDefaultHeaders(headers).disableRedirectHandling().build()); + final HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); + if (!enableRedirectHandling) { + httpClientBuilder.disableRedirectHandling(); + } + + return new DefaultRequestExecutor(httpClientBuilder.setDefaultHeaders(headers).build()); } } diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/services/RequestExecutorFactory.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/services/RequestExecutorFactory.java index 15676fd50..2189c1d66 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/services/RequestExecutorFactory.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/services/RequestExecutorFactory.java @@ -13,8 +13,18 @@ public class RequestExecutorFactory { + private final boolean enableRedirectHandling; + + public RequestExecutorFactory() { + this(true); + } + + public RequestExecutorFactory(boolean enableRedirectHandling) { + this.enableRedirectHandling = enableRedirectHandling; + } + public RequestExecutor createNoAuthExecutor(Collection
headers) { - return new DefaultConfig(headers).createRequestExecutor(); + return new DefaultConfig(headers, enableRedirectHandling).createRequestExecutor(); } public RequestExecutor createNoAuthExecutor() { @@ -26,7 +36,8 @@ public RequestExecutor createClientCredentialsExecutor(Collection
header String secret, String tokenEndpoint, String scope) { - return new ClientCredentialsConfig(headers, clientId, secret, tokenEndpoint, scope).createRequestExecutor(); + var config = new ClientCredentialsConfig(headers, clientId, secret, tokenEndpoint, scope, enableRedirectHandling); + return config.createRequestExecutor(); } public RequestExecutor createClientCredentialsExecutor(String clientId, diff --git a/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/RequestExecutorSteps.java b/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/RequestExecutorSteps.java index 3957bd4c0..a727af65e 100644 --- a/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/RequestExecutorSteps.java +++ b/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/RequestExecutorSteps.java @@ -28,7 +28,7 @@ public class RequestExecutorSteps { private LocalDateTime start; - private final RequestExecutorFactory factory = new RequestExecutorFactory(); + private final RequestExecutorFactory factory = new RequestExecutorFactory(false); private RequestExecutor requestExecutor; private Response response; private Request request; diff --git a/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/common/LdesDiscovererExecutor.java b/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/common/LdesDiscovererExecutor.java index 815d0fafd..1658353e8 100644 --- a/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/common/LdesDiscovererExecutor.java +++ b/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/common/LdesDiscovererExecutor.java @@ -18,7 +18,7 @@ public class LdesDiscovererExecutor implements CommandLineRunner { public LdesDiscovererExecutor(LdesDiscovererConfig config) { this.config = config; - final RequestExecutor requestExecutor = new RequestExecutorFactory().createNoAuthExecutor(); + final RequestExecutor requestExecutor = new RequestExecutorFactory(false).createNoAuthExecutor(); ldesStructureDiscoverer = new LdesStructureDiscoverer(config.getUrl(), config.getSourceFormatAsLang(), requestExecutor); } diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java index 425477f7b..684e1d0cf 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java @@ -39,6 +39,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -55,7 +56,7 @@ public class LdesClientProcessor extends AbstractProcessor { private static final Logger LOGGER = LoggerFactory.getLogger(LdesClientProcessor.class); private MemberSupplier memberSupplier; private LdesProperties ldesProperties; - private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(); + private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(false); private final StatePersistenceFactory statePersistenceFactory = new StatePersistenceFactory(); private boolean hasLdesEnded; private boolean keepState; diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java index 247ea21e8..2112b0991 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java @@ -19,6 +19,8 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import java.util.ArrayList; + import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties.KEEP_STATE; @SuppressWarnings("java:S6830") @@ -40,7 +42,7 @@ public static class LdioClientConnectorConfigurator implements LdioInputConfigur public static final String PROXY_URL_REPLACEMENT = "proxy-url-replacement"; private final ApplicationEventPublisher eventPublisher; private final ObservationRegistry observationRegistry; - private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(); + private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(false); private final RequestExecutor baseRequestExecutor = requestExecutorFactory.createNoAuthExecutor(); public LdioClientConnectorConfigurator(ApplicationEventPublisher eventPublisher, ObservationRegistry observationRegistry) { diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfig.java index 2567d289d..52ac39679 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfig.java @@ -1,6 +1,7 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.config; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClient; @@ -37,7 +38,8 @@ public LdioInput configure(LdiAdapter adapter, ComponentExecutor componentExecut ApplicationEventPublisher applicationEventPublisher, ComponentProperties properties) { String pipelineName = properties.getPipelineName(); - RequestExecutor requestExecutor = new LdioRequestExecutorSupplier().getRequestExecutor(properties); + final var requestExecutorFactory = new RequestExecutorFactory(false); + final var requestExecutor = new LdioRequestExecutorSupplier(requestExecutorFactory).getRequestExecutor(properties); final MemberSupplier memberSupplier = new MemberSupplierFactory(properties, requestExecutor).getMemberSupplier(); final boolean keepState = properties.getOptionalBoolean(KEEP_STATE).orElse(false); var ldesClient = new LdioLdesClient(pipelineName, componentExecutor, observationRegistry, memberSupplier, From f6ae6d88bbf6e32fd6cb66d69f7d2309f99d6e81 Mon Sep 17 00:00:00 2001 From: Tomvbe <34196062+Tomvbe@users.noreply.github.com> Date: Fri, 29 Mar 2024 11:27:06 +0100 Subject: [PATCH 08/20] fix: make example less strict to deal with invalid source data (#567) --- docs/_ldio/examples/ex2-scrape-api.md | 2 +- docs/_ldio/examples/ex2/ex2-mapping.ttl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/_ldio/examples/ex2-scrape-api.md b/docs/_ldio/examples/ex2-scrape-api.md index 2f29fe30f..89e8dacb9 100644 --- a/docs/_ldio/examples/ex2-scrape-api.md +++ b/docs/_ldio/examples/ex2-scrape-api.md @@ -41,7 +41,7 @@ mappings: - [a, cs:GameDeal] - [cs:title, $(title)] - [cs:metacriticLink, $(metacriticLink)] - - [cs:thumb, $(thumb)~iri] + - [cs:thumb, $(thumb)] - p: cs:releaseDate o: function: ldi:epochToIso8601 diff --git a/docs/_ldio/examples/ex2/ex2-mapping.ttl b/docs/_ldio/examples/ex2/ex2-mapping.ttl index ffa334a33..0cf19adc2 100644 --- a/docs/_ldio/examples/ex2/ex2-mapping.ttl +++ b/docs/_ldio/examples/ex2/ex2-mapping.ttl @@ -39,7 +39,7 @@ map:om_002 rml:reference "metacriticLink" ; map:om_003 rml:reference "thumb" ; rdf:type rr:ObjectMap ; - rr:termType rr:IRI . + rr:termType rr:Literal . map:om_004 map:fn_000 ; rdf:type ; From c9dd221e1b87184eeaaf8fd04877e47eb9727824 Mon Sep 17 00:00:00 2001 From: Tomvbe <34196062+Tomvbe@users.noreply.github.com> Date: Wed, 3 Apr 2024 09:32:37 +0200 Subject: [PATCH 09/20] fix: use generic example in docs (#568) --- .../ldio-version-object-creator.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/_ldio/ldio-transformers/ldio-version-object-creator.md b/docs/_ldio/ldio-transformers/ldio-version-object-creator.md index c4031f18d..51f6ce1a0 100644 --- a/docs/_ldio/ldio-transformers/ldio-version-object-creator.md +++ b/docs/_ldio/ldio-transformers/ldio-version-object-creator.md @@ -12,13 +12,13 @@ An LDIO wrapper component for the [LDI Version Object Creator building block](.. ## Config -| Property | Description | Required | Default | Example | Supported values | -|:-----------------------|:-------------------------------------------------------------------------------------------------------------------------------|:---------|:------------------|:--------------------------------------------------------------------|:-----------------| -| date-observed-property | Property path (IRI format '<>') that points to a literal which should be used as timestampPath. Defaults to current timestamp. | No | Current Timestamp | | String | -| member-type | Defines the RDF type of the version object | No | N/A | https://uri.etsi.org/ngsi-ld/default-context/Device | String | -| delimiter | Defines how the version object id will be constructed. (versionOf + delimiter + dateObserved) | No | / | / | String | -| generatedAt-property | If defined, a statement will be added to the model with the observedAt value and the given property. | No | N/A | http://www.w3.org/ns/prov#generatedAtTime | String | -| versionOf-property | If defined, a statement will be added to the model with the versionOf value and the given property. | No | N/A | http://purl.org/dc/terms/isVersionOf | String | +| Property | Description | Required | Default | Example | Supported values | +|:-----------------------|:-------------------------------------------------------------------------------------------------------------------------------|:---------|:------------------|:------------------------------------------|:-----------------| +| date-observed-property | Property path (IRI format '<>') that points to a literal which should be used as timestampPath. Defaults to current timestamp. | No | Current Timestamp | | String | +| member-type | Defines the RDF type of the version object | No | N/A | https://example.org/Person | String | +| delimiter | Defines how the version object id will be constructed. (versionOf + delimiter + dateObserved) | No | / | / | String | +| generatedAt-property | If defined, a statement will be added to the model with the observedAt value and the given property. | No | N/A | http://www.w3.org/ns/prov#generatedAtTime | String | +| versionOf-property | If defined, a statement will be added to the model with the versionOf value and the given property. | No | N/A | http://purl.org/dc/terms/isVersionOf | String | A property path can be provided for the date-observed-property. For example for the following ttl: From 2b5b51d52bc2c73ab92984738fc517222ad9d5a7 Mon Sep 17 00:00:00 2001 From: Tomvbe <34196062+Tomvbe@users.noreply.github.com> Date: Wed, 3 Apr 2024 12:42:41 +0200 Subject: [PATCH 10/20] fix: separate ldio status from triggers (#569) --- .../edc/services/MemoryTokenService.java | 25 ++++++++- .../services/MemoryTokenServiceLifecycle.java | 51 +++++++++++++++++++ .../executor/edc/services/TokenService.java | 5 ++ .../edc/services/MemoryTokenServiceTest.java | 3 ++ .../ldio/services/PipelineStatusService.java | 31 +++-------- .../ldes/ldio/modules/DummyIn.java | 4 +- .../services/PipelineStatusServiceTest.java | 4 +- .../resources/features/management.feature | 18 +++---- .../ldes/ldio/types/LdioInput.java | 36 ++++++------- .../ldio/valueobjects/PipelineStatus.java | 2 +- .../valueobjects/PipelineStatusTrigger.java | 7 +++ .../ldes/ldio/LdioAmqpIn.java | 2 +- .../ldes/ldio/AmqpInIntegrationTestSteps.java | 12 +++-- .../ldes/ldio/LdioArchiveFileIn.java | 2 +- .../config/LdioHttpInputPollerAutoConfig.java | 2 +- .../ldio/config/LdioHttpInAutoConfig.java | 2 +- .../ldes/ldio/LdioHttpInputTest.java | 10 ++-- .../ldes/ldio/LdioKafkaIn.java | 2 +- .../config/KafkaInIntegrationTestSteps.java | 12 +++-- .../ldes/ldio/LdioLdesClientConnectorApi.java | 21 ++++++-- .../LdioLdesClientConnectorApiController.java | 19 +++++++ .../LdioLdesClientConnectorAutoConfig.java | 8 +-- .../ldes/ldio/LdioLdesClient.java | 4 +- 23 files changed, 195 insertions(+), 87 deletions(-) create mode 100644 ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenServiceLifecycle.java create mode 100644 ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineStatusTrigger.java diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenService.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenService.java index bdf52ad22..e6dd85597 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenService.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenService.java @@ -16,11 +16,17 @@ public class MemoryTokenService implements TokenService { private EdcToken token; - public MemoryTokenService(TransferService transferService) { + private final MemoryTokenServiceLifecycle lifecycle; + + public MemoryTokenService(TransferService transferService, MemoryTokenServiceLifecycle lifecycle) { this.transferService = transferService; - } + this.lifecycle = lifecycle; + } + @Override public RequestHeader waitForTokenHeader() { + lifecycle.checkPipelineState(); + if (token != null) { return token.getTokenHeader(); } @@ -45,4 +51,19 @@ public void updateToken(String token) { this.token = EdcToken.fromJsonString(token); } + @Override + public void pause() { + lifecycle.pause(); + } + + @Override + public void resume() { + lifecycle.resume(); + } + + @Override + public void shutdown() { + lifecycle.shutdown(); + } + } diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenServiceLifecycle.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenServiceLifecycle.java new file mode 100644 index 000000000..e5b100a29 --- /dev/null +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenServiceLifecycle.java @@ -0,0 +1,51 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.edc.services; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MemoryTokenServiceLifecycle { + + private final Logger log = LoggerFactory.getLogger(MemoryTokenServiceLifecycle.class); + + private boolean terminated = false; + private boolean paused = false; + + void checkPipelineState() { + checkPause(); + checkTerminated(); + } + + private synchronized void checkPause() { + while (paused) { + try { + this.wait(); + } catch (InterruptedException e) { + log.error("Thread interrupted: {}", e.getMessage()); + Thread.currentThread().interrupt(); + } + } + } + + private void checkTerminated() { + if (terminated) { + throw new TerminatedException(); + } + } + + void pause() { + this.paused = true; + } + + synchronized void resume() { + this.paused = false; + this.notifyAll(); + } + + void shutdown() { + this.terminated = true; + } + + public static class TerminatedException extends RuntimeException { + } + +} diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/TokenService.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/TokenService.java index 58db9d9b7..db7a073db 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/TokenService.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/TokenService.java @@ -10,4 +10,9 @@ public interface TokenService { void updateToken(String token); + void pause(); + + void resume(); + + void shutdown(); } diff --git a/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenServiceTest.java b/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenServiceTest.java index e2f192e9a..55c3c96e7 100644 --- a/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenServiceTest.java +++ b/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenServiceTest.java @@ -16,6 +16,9 @@ @ExtendWith(MockitoExtension.class) class MemoryTokenServiceTest { + @Mock + private MemoryTokenServiceLifecycle memoryTokenServiceLifecycle; + @Mock private TransferService transferService; diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusService.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusService.java index d887afde7..78e079aa7 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusService.java +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusService.java @@ -19,6 +19,7 @@ import java.util.stream.Collectors; import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.*; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger.*; import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.StatusChangeSource.AUTO; @Component @@ -54,30 +55,19 @@ public void handlePipelineStatusResponse(PipelineStatusEvent statusEvent) { currentSavedPipeline.updateStatus(statusEvent.status(), statusEvent.statusChangeSource()); } - @EventListener - public void handlePipelineDeleted(PipelineDeletedEvent deletedEvent) { - this.savedPipelines.remove(deletedEvent.pipelineId()); - } - @EventListener public void handlePipelineCreated(InputCreatedEvent event) { - savedPipelines.put(event.pipelineName(), new SavedPipeline(event.ldioInput(), RUNNING)); + savedPipelines.put(event.pipelineName(), new SavedPipeline(event.ldioInput(), event.ldioInput().getStatus())); } public PipelineStatus resumeHaltedPipeline(String pipelineId) { - PipelineStatus pipelineStatus = getPipelineStatus(pipelineId); + var pipelineStatus = getPipelineStatus(pipelineId); return switch (pipelineStatus) { - case HALTED -> { - savedPipelines.get(pipelineId).getLdioInput().updateStatus(RESUMING); - yield RESUMING; - } + case HALTED -> savedPipelines.get(pipelineId).getLdioInput().updateStatus(RESUME); case INIT -> INIT; - case STARTING -> STARTING; case RUNNING -> RUNNING; - case RESUMING -> RESUMING; case STOPPED -> STOPPED; - case STOPPING -> STOPPING; }; } @@ -85,23 +75,18 @@ public PipelineStatus haltRunningPipeline(String pipelineId) { PipelineStatus pipelineStatus = getPipelineStatus(pipelineId); return switch (pipelineStatus) { - case RUNNING, RESUMING -> { - savedPipelines.get(pipelineId).getLdioInput().updateStatus(HALTED); - yield HALTED; - } + case RUNNING -> savedPipelines.get(pipelineId).getLdioInput().updateStatus(HALT); case INIT -> INIT; - case STARTING -> STARTING; case HALTED -> HALTED; case STOPPED -> STOPPED; - case STOPPING -> STOPPING; }; } public PipelineStatus stopPipeline(String pipelineId) { - LdioInput input = savedPipelines.get(pipelineId).getLdioInput(); - input.updateStatus(STOPPING); + PipelineStatus newPipelineStatus = savedPipelines.get(pipelineId).getLdioInput().updateStatus(STOP); + this.savedPipelines.remove(pipelineId); eventPublisher.publishEvent(new PipelineDeletedEvent(pipelineId)); - return STOPPED; + return newPipelineStatus; } public Map getPipelineStatusOverview() { diff --git a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/modules/DummyIn.java b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/modules/DummyIn.java index 95748c607..a882fa3be 100644 --- a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/modules/DummyIn.java +++ b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/modules/DummyIn.java @@ -5,14 +5,14 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; import org.springframework.context.ApplicationEventPublisher; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.STARTING; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger.START; public class DummyIn extends LdioInput { private int counter = 0; public DummyIn(ComponentExecutor executor, LdiAdapter adapter, ApplicationEventPublisher applicationEventPublisher) { super("DummyIn", "test", executor, adapter, null, applicationEventPublisher); - this.updateStatus(STARTING); + this.updateStatus(START); } public void sendData() { diff --git a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusServiceTest.java b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusServiceTest.java index 8e86f3f19..03189b5fc 100644 --- a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusServiceTest.java +++ b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusServiceTest.java @@ -3,6 +3,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.events.InputCreatedEvent; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus; +import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.context.ApplicationEventPublisher; @@ -24,10 +25,11 @@ void setup() { @Test void when_StoppingPipeline_Then_MethodsAreCalled() { + when(input.updateStatus(any())).thenReturn(PipelineStatus.STOPPED); PipelineStatus result = pipelineStatusService.stopPipeline(pipelineName); assertEquals(PipelineStatus.STOPPED, result); - verify(input).updateStatus(PipelineStatus.STOPPING); + verify(input).updateStatus(PipelineStatusTrigger.STOP); } } diff --git a/ldi-orchestrator/ldio-application/src/test/resources/features/management.feature b/ldi-orchestrator/ldio-application/src/test/resources/features/management.feature index c8e935d67..9293c10bf 100644 --- a/ldi-orchestrator/ldio-application/src/test/resources/features/management.feature +++ b/ldi-orchestrator/ldio-application/src/test/resources/features/management.feature @@ -9,8 +9,8 @@ Feature: Management of pipelines in LDIO And I expect pipelines Examples: | file | type | statusCode | pipelineCount | - | "valid" | "json" | 200 | 1 | - | "valid" | "yml" | 200 | 1 | + | "valid" | "json" | 201 | 1 | + | "valid" | "yml" | 201 | 1 | | "invalid" | "json" | 400 | 0 | | "invalid" | "yml" | 400 | 0 | | "missing-adapter" | "yml" | 400 | 0 | @@ -23,19 +23,19 @@ Feature: Management of pipelines in LDIO And I expect pipelines Examples: | firstfile | firstStatusCode | secondFile | secondStatusCode | type | pipelineCount | - | "valid" | 200 | "valid" | 400 | "json" | 1 | - | "valid" | 200 | "valid" | 400 | "yml" | 1 | - | "valid" | 200 | "valid-2" | 200 | "json" | 2 | - | "valid" | 200 | "valid-2" | 200 | "yml" | 2 | - | "valid" | 200 | "invalid" | 400 | "json" | 1 | - | "valid" | 200 | "invalid" | 400 | "yml" | 1 | + | "valid" | 201 | "valid" | 400 | "json" | 1 | + | "valid" | 201 | "valid" | 400 | "yml" | 1 | + | "valid" | 201 | "valid-2" | 201 | "json" | 2 | + | "valid" | 201 | "valid-2" | 201 | "yml" | 2 | + | "valid" | 201 | "invalid" | 400 | "json" | 1 | + | "valid" | 201 | "invalid" | 400 | "yml" | 1 | | "invalid" | 400 | "invalid" | 400 | "json" | 0 | | "invalid" | 400 | "invalid" | 400 | "yml" | 0 | @deletion Scenario: Deleting a pipeline When I start LDIO - And I post a "valid" "yml" pipeline with a 200 response + And I post a "valid" "yml" pipeline with a 201 response And I delete the "valid-pipeline" pipeline with a 202 response And I expect 0 pipelines diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioInput.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioInput.java index c8cb49591..1bb205529 100644 --- a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioInput.java +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioInput.java @@ -6,6 +6,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.config.ObserveConfiguration; import be.vlaanderen.informatievlaanderen.ldes.ldio.events.PipelineStatusEvent; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus; +import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger; import io.micrometer.core.instrument.Metrics; import io.micrometer.observation.Observation; import io.micrometer.observation.ObservationRegistry; @@ -16,7 +17,7 @@ import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.PipelineConfig.PIPELINE_NAME; import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.*; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.StatusChangeSource.AUTO; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger.START; import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.StatusChangeSource.MANUAL; /** @@ -94,41 +95,36 @@ protected void processModel(Model model) { public abstract void shutdown(); - public void updateStatus(PipelineStatus statusEvent) { - switch (statusEvent) { - case STARTING -> { - this.pipelineStatus = RUNNING; - applicationEventPublisher.publishEvent(new PipelineStatusEvent(pipelineName, RUNNING, AUTO)); - } - case RESUMING -> { + public PipelineStatus updateStatus(PipelineStatusTrigger trigger) { + switch (trigger) { + case START -> this.pipelineStatus = RUNNING; + case RESUME -> { this.resume(); this.pipelineStatus = RUNNING; - applicationEventPublisher.publishEvent(new PipelineStatusEvent(pipelineName, RUNNING, MANUAL)); } - case HALTED -> { + case HALT -> { if (this.pipelineStatus != INIT) { this.pause(); this.pipelineStatus = HALTED; - applicationEventPublisher.publishEvent(new PipelineStatusEvent(pipelineName, HALTED, MANUAL)); } } - case STOPPING -> { - this.pipelineStatus = STOPPED; - applicationEventPublisher.publishEvent(new PipelineStatusEvent(pipelineName, STOPPED, MANUAL)); - } - default -> log.warn("Unhandled status update on pipeline: {} for status: {}", pipelineName, statusEvent); + case STOP -> this.pipelineStatus = STOPPED; + default -> log.warn("Unhandled status update on pipeline: {} for status: {}", pipelineName, pipelineStatus); } + + applicationEventPublisher.publishEvent(new PipelineStatusEvent(pipelineName, this.pipelineStatus, MANUAL)); + return this.pipelineStatus; } protected abstract void resume(); protected abstract void pause(); - public void starting() { - this.pipelineStatus = STARTING; + public void start() { + updateStatus(START); } - public boolean isHalted() { - return pipelineStatus.equals(HALTED) || pipelineStatus.equals(STOPPED); + public PipelineStatus getStatus() { + return this.pipelineStatus; } } diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineStatus.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineStatus.java index 20228b5fa..c5c1cb28a 100644 --- a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineStatus.java +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineStatus.java @@ -1,5 +1,5 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects; public enum PipelineStatus { - INIT, STARTING, RESUMING, RUNNING, HALTED, STOPPING, STOPPED + INIT, RUNNING, HALTED, STOPPED } diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineStatusTrigger.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineStatusTrigger.java new file mode 100644 index 000000000..9f792486d --- /dev/null +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineStatusTrigger.java @@ -0,0 +1,7 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects; + +public enum PipelineStatusTrigger { + + START, HALT, RESUME, STOP + +} diff --git a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioAmqpIn.java b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioAmqpIn.java index c8aa08027..1ce02e87b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioAmqpIn.java +++ b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioAmqpIn.java @@ -45,7 +45,7 @@ public LdioAmqpIn(String pipelineName, ComponentExecutor executor, LdiAdapter ad ldioAmqpInRegistrator = jmsInRegistrator; listenerId = endpoint.getId(); jmsInRegistrator.registerListener(jmsConfig, endpoint); - this.starting(); + this.start(); } @Override diff --git a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/AmqpInIntegrationTestSteps.java b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/AmqpInIntegrationTestSteps.java index 0da71b9e3..05c2e3d02 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/AmqpInIntegrationTestSteps.java +++ b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/AmqpInIntegrationTestSteps.java @@ -29,8 +29,8 @@ import java.util.stream.Stream; import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioAmqpIn.NAME; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.HALTED; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.RESUMING; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger.HALT; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger.RESUME; import static org.apache.jena.riot.RDFLanguages.contentTypeToLang; import static org.apache.jena.riot.RDFLanguages.nameToLang; import static org.awaitility.Awaitility.await; @@ -103,6 +103,7 @@ public void iCreateDefaultConfigForLdioJmsInWithContentType(String contentType) public void theListenerWillWaitForTheMessage(int i) { await().until(() -> adapterResult.size() == i); } + @Then("Wait for a grace period") public void waitForGracePeriod() throws InterruptedException { Awaitility.waitAtMost(Duration.of(500, ChronoUnit.MILLIS)); @@ -117,12 +118,14 @@ public void theResultValueWillContainTheModel() { @When("I pause the pipeline") public void pausePipeline() { - ldioInput.updateStatus(HALTED); + ldioInput.updateStatus(HALT); } + @When("I unpause the pipeline") public void unPausePipeline() { - ldioInput.updateStatus(RESUMING); + ldioInput.updateStatus(RESUME); } + @And("The result value will not contain the model") public void theResultValueWillNotContainTheModel() { assertEquals(0, adapterResult.size()); @@ -137,4 +140,5 @@ public void theComponentExecutorWillHaveBeenCalled(int interactions) { public Boolean booleanValue(String value) { return Boolean.valueOf(value); } + } diff --git a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioArchiveFileIn.java b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioArchiveFileIn.java index 03c8b0519..6ce6e259d 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioArchiveFileIn.java +++ b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioArchiveFileIn.java @@ -21,7 +21,7 @@ public LdioArchiveFileIn(String pipelineName, ComponentExecutor executor, Observ super(NAME, pipelineName, executor, null, observationRegistry, applicationEventPublisher); this.archiveFileCrawler = crawler; this.sourceFormat = source; - starting(); + start(); log.info("Starting with crawling the archive."); crawlArchive(); log.info("Finished crawling the archive."); diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInputPollerAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInputPollerAutoConfig.java index 2e9b414ca..e8836a576 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInputPollerAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInputPollerAutoConfig.java @@ -53,7 +53,7 @@ public LdioHttpInputPoller configure(LdiAdapter adapter, ComponentExecutor execu httpInputPoller.schedulePoller(getPollingInterval(properties)); - httpInputPoller.starting(); + httpInputPoller.start(); return httpInputPoller; } diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInAutoConfig.java index ce99381ff..7e77bd331 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInAutoConfig.java @@ -44,7 +44,7 @@ public LdioInput configure(LdiAdapter adapter, LdioHttpInProcess ldioHttpIn = new LdioHttpInProcess(pipelineName, executor, adapter, observationRegistry, applicationEventPublisher); eventPublisher.publishEvent(new HttpInPipelineCreatedEvent(pipelineName, ldioHttpIn)); - ldioHttpIn.starting(); + ldioHttpIn.start(); return ldioHttpIn; } diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputTest.java b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputTest.java index ce82f5cbe..81c34803b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputTest.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputTest.java @@ -17,8 +17,8 @@ import java.util.stream.Stream; import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInProcess.NAME; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.HALTED; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.RESUMING; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger.HALT; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger.RESUME; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; @@ -50,7 +50,7 @@ void setup() { void testHttpEndpoint() throws Exception { String content = "_:b0 \"Jane Doe\" ."; String contentType = "application/n-quads"; - input.updateStatus(RESUMING); + input.updateStatus(RESUME); mockMvc.perform(post("/%s".formatted(endpoint)).content(content).contentType(contentType)).andExpect(status().isAccepted()); @@ -60,13 +60,13 @@ void testHttpEndpoint() throws Exception { void when_PipelineIsHalted_Then_MessageIsNotProcessed() throws Exception { String content = "_:b0 \"Jane Doe\" ."; String contentType = "application/n-quads"; - input.updateStatus(HALTED); + input.updateStatus(HALT); mockMvc.perform(post("/%s".formatted(endpoint)).content(content).contentType(contentType)).andExpect(status().is(503)); verifyNoInteractions(adapter); - input.updateStatus(RESUMING); + input.updateStatus(RESUME); mockMvc.perform(post("/%s".formatted(endpoint)).content(content).contentType(contentType)).andExpect(status().isAccepted()); diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioKafkaIn.java b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioKafkaIn.java index 7bfae23bd..17f8c4e82 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioKafkaIn.java +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioKafkaIn.java @@ -51,7 +51,7 @@ public LdioKafkaIn(String pipelineName, ComponentExecutor executor, LdiAdapter a containerProps.setMessageListener(listener); this.container = new KafkaMessageListenerContainer<>(consumerFactory, containerProps); container.start(); - this.starting(); + this.start(); } @Override diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/KafkaInIntegrationTestSteps.java b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/KafkaInIntegrationTestSteps.java index 55583cd3c..252d662a8 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/KafkaInIntegrationTestSteps.java +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/KafkaInIntegrationTestSteps.java @@ -4,7 +4,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; -import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus; +import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger; import io.cucumber.java.en.And; import io.cucumber.java.en.Then; import io.cucumber.java.en.When; @@ -110,6 +110,7 @@ public void iSendTheModelToTheUsingTheKafkaProducer() { public void theListenerWillWaitForTheMessage() { await().until(() -> adapterResult.size() == 1); } + @Then("Wait for a grace period") public void theListenerWillWaitForPeriod() throws InterruptedException { Awaitility.waitAtMost(1500, TimeUnit.MILLISECONDS); @@ -131,14 +132,15 @@ public void theResultValueWillContainTheModel() { public void theComponentExecutorWillHaveBeenCalled(int i) { assertEquals(i, componentExecutorResult.size()); } + @When("I pause the pipeline") public void pauseInput() { - kafkaIn.updateStatus(PipelineStatus.HALTED); + kafkaIn.updateStatus(PipelineStatusTrigger.HALT); } + @When("I unpause the pipeline") public void unPauseInput() { - kafkaIn.updateStatus(PipelineStatus.RESUMING); + kafkaIn.updateStatus(PipelineStatusTrigger.RESUME); } -} - +} diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApi.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApi.java index d89ccbaeb..ba061866b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApi.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApi.java @@ -3,7 +3,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.edc.services.TokenService; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.edc.services.TransferService; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.STARTING; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger.START; public class LdioLdesClientConnectorApi { private final TransferService transferService; @@ -22,9 +22,22 @@ public void handleToken(String token) { } public String handleTransfer(String transfer) { - String transer = transferService.startTransfer(transfer).getBodyAsString() + String response = transferService.startTransfer(transfer).getBodyAsString() .orElse(""); - ldesClient.updateStatus(STARTING); - return transer; + ldesClient.updateStatus(START); + return response; } + + public void shutdown() { + tokenService.shutdown(); + } + + public void resume() { + tokenService.resume(); + } + + public void pause() { + tokenService.pause(); + } + } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApiController.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApiController.java index 39e202431..b46f3327d 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApiController.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApiController.java @@ -1,6 +1,8 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio; import be.vlaanderen.informatievlaanderen.ldes.ldio.event.LdesClientConnectorApiCreatedEvent; +import be.vlaanderen.informatievlaanderen.ldes.ldio.events.PipelineDeletedEvent; +import be.vlaanderen.informatievlaanderen.ldes.ldio.events.PipelineStatusEvent; import org.springframework.context.event.EventListener; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PathVariable; @@ -38,4 +40,21 @@ void handleNewPipelines(LdesClientConnectorApiCreatedEvent connectorApiCreatedEv clientConnectorApis.put(connectorApiCreatedEvent.pipelineName(), connectorApiCreatedEvent.ldesClientConnectorApi()); } + @EventListener + void deletePipeline(PipelineDeletedEvent deletedEvent) { + clientConnectorApis.remove(deletedEvent.pipelineId()).shutdown(); + } + + @EventListener + public void handlePipelineStatusEvent(PipelineStatusEvent event) { + LdioLdesClientConnectorApi connectorApi = clientConnectorApis.get(event.pipelineId()); + switch (event.status()) { + case RUNNING -> connectorApi.resume(); + case HALTED -> connectorApi.pause(); + default -> { + // do nothing with the other status events + } + } + } + } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java index 2112b0991..b6e57899c 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java @@ -2,6 +2,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.edc.services.MemoryTokenService; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.edc.services.MemoryTokenServiceLifecycle; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.edc.services.MemoryTransferService; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.edc.valueobjects.EdcUrlProxy; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; @@ -19,8 +20,6 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import java.util.ArrayList; - import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties.KEEP_STATE; @SuppressWarnings("java:S6830") @@ -55,7 +54,8 @@ public LdioInput configure(LdiAdapter adapter, ComponentExecutor executor, Appli final String pipelineName = properties.getPipelineName(); final var connectorTransferUrl = properties.getProperty(CONNECTOR_TRANSFER_URL); final var transferService = new MemoryTransferService(baseRequestExecutor, connectorTransferUrl); - final var tokenService = new MemoryTokenService(transferService); + final var memoryTokenServiceLifecycle = new MemoryTokenServiceLifecycle(); + final var tokenService = new MemoryTokenService(transferService, memoryTokenServiceLifecycle); final var urlProxy = getEdcUrlProxy(properties); final var edcRequestExecutor = requestExecutorFactory.createEdcExecutor(baseRequestExecutor, tokenService, @@ -64,9 +64,9 @@ public LdioInput configure(LdiAdapter adapter, ComponentExecutor executor, Appli final boolean keepState = properties.getOptionalBoolean(KEEP_STATE).orElse(false); var ldesClient = new LdioLdesClient(pipelineName, executor, observationRegistry, memberSupplier, applicationEventPublisher, keepState); - ldesClient.start(); eventPublisher.publishEvent(new LdesClientConnectorApiCreatedEvent(pipelineName, new LdioLdesClientConnectorApi(transferService, tokenService, ldesClient))); + ldesClient.start(); return ldesClient; } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java index 66ac226f3..3e84d42e5 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java @@ -35,12 +35,12 @@ public LdioLdesClient(String pipelineName, this.keepState = keepState; } - @SuppressWarnings("java:S2095") + @Override public void start() { + super.start(); final ExecutorService executorService = newSingleThreadExecutor(); executorService.submit(() -> { memberSupplier.init(); - this.starting(); this.run(); }); } From e46bd66b726063c050b7d7ce2a327033398c8c50 Mon Sep 17 00:00:00 2001 From: Jonas Bulcke <127748878+jobulcke@users.noreply.github.com> Date: Wed, 3 Apr 2024 19:03:33 +0200 Subject: [PATCH 11/20] fix: broken pipeline status event handler (#570) --- .../LdioLdesClientConnectorApiController.java | 33 ++++++++++--------- .../LdioLdesClientConnectorApiCollection.java | 13 ++++++++ ...oLdesClientConnectorApiCollectionImpl.java | 28 ++++++++++++++++ .../ldio/LdioLdesClientConnectorTest.java | 5 +-- 4 files changed, 61 insertions(+), 18 deletions(-) create mode 100644 ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/collection/LdioLdesClientConnectorApiCollection.java create mode 100644 ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/collection/LdioLdesClientConnectorApiCollectionImpl.java diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApiController.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApiController.java index b46f3327d..6f6428167 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApiController.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApiController.java @@ -1,5 +1,6 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio; +import be.vlaanderen.informatievlaanderen.ldes.ldio.collection.LdioLdesClientConnectorApiCollection; import be.vlaanderen.informatievlaanderen.ldes.ldio.event.LdesClientConnectorApiCreatedEvent; import be.vlaanderen.informatievlaanderen.ldes.ldio.events.PipelineDeletedEvent; import be.vlaanderen.informatievlaanderen.ldes.ldio.events.PipelineStatusEvent; @@ -10,18 +11,17 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RestController; -import java.util.HashMap; -import java.util.Map; - -import static java.util.Optional.ofNullable; - @RestController public class LdioLdesClientConnectorApiController { - private final Map clientConnectorApis = new HashMap<>(); + private final LdioLdesClientConnectorApiCollection clientConnectorApis; + + public LdioLdesClientConnectorApiController(LdioLdesClientConnectorApiCollection clientConnectorApis) { + this.clientConnectorApis = clientConnectorApis; + } @PostMapping(path = "/{pipeline}/token") public ResponseEntity handleToken(@PathVariable("pipeline") String pipeline, @RequestBody String token) { - ofNullable(clientConnectorApis.get(pipeline)) + clientConnectorApis.get(pipeline) .orElseThrow(() -> new IllegalArgumentException("Not a valid pipeline")) .handleToken(token); return ResponseEntity.accepted().build(); @@ -30,14 +30,14 @@ public ResponseEntity handleToken(@PathVariable("pipeline") String pipel @PostMapping(path = "/{pipeline}/transfer") public ResponseEntity handleTransfer(@PathVariable("pipeline") String pipeline, @RequestBody String transfer) { return ResponseEntity.accepted() - .body(ofNullable(clientConnectorApis.get(pipeline)) + .body(clientConnectorApis.get(pipeline) .orElseThrow(() -> new IllegalArgumentException("Not a valid pipeline")) .handleTransfer(transfer)); } @EventListener void handleNewPipelines(LdesClientConnectorApiCreatedEvent connectorApiCreatedEvent) { - clientConnectorApis.put(connectorApiCreatedEvent.pipelineName(), connectorApiCreatedEvent.ldesClientConnectorApi()); + clientConnectorApis.add(connectorApiCreatedEvent.pipelineName(), connectorApiCreatedEvent.ldesClientConnectorApi()); } @EventListener @@ -47,14 +47,15 @@ void deletePipeline(PipelineDeletedEvent deletedEvent) { @EventListener public void handlePipelineStatusEvent(PipelineStatusEvent event) { - LdioLdesClientConnectorApi connectorApi = clientConnectorApis.get(event.pipelineId()); - switch (event.status()) { - case RUNNING -> connectorApi.resume(); - case HALTED -> connectorApi.pause(); - default -> { - // do nothing with the other status events + clientConnectorApis.get(event.pipelineId()).ifPresent(connectorApi -> { + switch (event.status()) { + case RUNNING -> connectorApi.resume(); + case HALTED -> connectorApi.pause(); + default -> { + // do nothing with the other status events + } } - } + }); } } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/collection/LdioLdesClientConnectorApiCollection.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/collection/LdioLdesClientConnectorApiCollection.java new file mode 100644 index 000000000..8e3124152 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/collection/LdioLdesClientConnectorApiCollection.java @@ -0,0 +1,13 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.collection; + +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientConnectorApi; + +import java.util.Optional; + +public interface LdioLdesClientConnectorApiCollection { + Optional get(String pipeline); + + void add(String pipeline, LdioLdesClientConnectorApi ldioLdesClientConnectorApi); + + LdioLdesClientConnectorApi remove(String pipeline); +} diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/collection/LdioLdesClientConnectorApiCollectionImpl.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/collection/LdioLdesClientConnectorApiCollectionImpl.java new file mode 100644 index 000000000..130ab4a4f --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/collection/LdioLdesClientConnectorApiCollectionImpl.java @@ -0,0 +1,28 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.collection; + +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientConnectorApi; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +@Component +public class LdioLdesClientConnectorApiCollectionImpl implements LdioLdesClientConnectorApiCollection { + private final Map clientConnectorApis = new HashMap<>(); + + @Override + public Optional get(String pipeline) { + return Optional.ofNullable(clientConnectorApis.get(pipeline)); + } + + @Override + public void add(String pipeline, LdioLdesClientConnectorApi ldioLdesClientConnectorApi) { + clientConnectorApis.put(pipeline, ldioLdesClientConnectorApi); + } + + @Override + public LdioLdesClientConnectorApi remove(String pipeline) { + return clientConnectorApis.remove(pipeline); + } +} diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorTest.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorTest.java index 63742ba22..40b8bb8e2 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorTest.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorTest.java @@ -10,6 +10,7 @@ import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.context.ApplicationEventPublisher; +import org.springframework.context.annotation.ComponentScan; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.MvcResult; @@ -22,6 +23,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @SpringBootTest(classes = LdioLdesClientConnectorApiController.class) +@ComponentScan(basePackages = "be.vlaanderen.informatievlaanderen.ldes.ldio.collection") @AutoConfigureMockMvc class LdioLdesClientConnectorTest { @@ -32,13 +34,12 @@ class LdioLdesClientConnectorTest { private final String endpoint = "endpoint"; private TransferService transferService; private TokenService tokenService; - private LdioLdesClient ldesClient; @BeforeEach void setup() { transferService = mock(TransferService.class); tokenService = mock(TokenService.class); - ldesClient = mock(LdioLdesClient.class); + final LdioLdesClient ldesClient = mock(LdioLdesClient.class); eventPublisher.publishEvent(new LdesClientConnectorApiCreatedEvent(endpoint, new LdioLdesClientConnectorApi(transferService, tokenService, ldesClient))); } From 272a4c4561c7568801b8090b3fd7e6a57ee13696 Mon Sep 17 00:00:00 2001 From: Jonas Bulcke <127748878+jobulcke@users.noreply.github.com> Date: Thu, 4 Apr 2024 15:55:39 +0200 Subject: [PATCH 12/20] fix: deletion outside named graph (#571) --- .../ldes/ldi/Materialiser.java | 4 +++ .../valueobjects/MaterialiserConnection.java | 8 ++++- .../ldes/ldi/MaterialiserIT.java | 36 +++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/Materialiser.java b/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/Materialiser.java index 1887289e8..9ea379a4c 100644 --- a/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/Materialiser.java +++ b/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/Materialiser.java @@ -50,6 +50,10 @@ public CompletableFuture processAsync(List jenaModels) { return CompletableFuture.runAsync(() -> process(jenaModels)); } + public void closeConnection() { + materialiserConnection.close(); + } + public void shutdown() { materialiserConnection.shutdown(); } diff --git a/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/MaterialiserConnection.java b/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/MaterialiserConnection.java index 61acbdacb..48854ff58 100644 --- a/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/MaterialiserConnection.java +++ b/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/MaterialiserConnection.java @@ -24,7 +24,9 @@ public void add(Model model) { } public RepositoryResult getStatements(Resource subject, IRI predicate, Value object) { - return holder.getConnection().getStatements(subject, predicate, object); + return getNamedGraphIri() + .map(namedGraphIri -> holder.getConnection().getStatements(subject, predicate, object, namedGraphIri)) + .orElseGet(() -> holder.getConnection().getStatements(subject, predicate, object)); } public void remove(Resource subject, IRI predicate, Value object) { @@ -46,6 +48,10 @@ public void rollback() { connection.close(); } + public void close() { + holder.getConnection().close(); + } + public void shutdown() { holder.shutdown(); } diff --git a/ldi-core/repository-materialiser/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/MaterialiserIT.java b/ldi-core/repository-materialiser/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/MaterialiserIT.java index 87c5803c4..71be4e28f 100644 --- a/ldi-core/repository-materialiser/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/MaterialiserIT.java +++ b/ldi-core/repository-materialiser/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/MaterialiserIT.java @@ -15,6 +15,7 @@ import org.eclipse.rdf4j.sail.memory.config.MemoryStoreConfig; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; @@ -91,6 +92,41 @@ void when_UpdateEntities_Then_OldTriplesRemoved(String namedGraph) throws Except .noneMatch(statement -> statement.getObject().stringValue().equals("Taylor")); } + @Test + void given_TwoDifferentNamedGraphs_when_processModelsWithSameSubjects_then_ExpectNoChangesInOtherNamedGraph() { + materialiser = new Materialiser(subject, LOCAL_REPOSITORY_ID, "http://example.org/named-graph-1"); + final Materialiser secondMaterialiser = new Materialiser(subject, LOCAL_REPOSITORY_ID, "http://example.org/named-graph-2"); + + final org.apache.jena.rdf.model.Model firstJenaModel = RDFParser.source("movies/1.ttl").toModel(); + final org.apache.jena.rdf.model.Model secondJenaModel = RDFParser.source("movies/2.ttl").toModel(); + materialiser.process(List.of(firstJenaModel)); + secondMaterialiser.process(List.of(secondJenaModel)); + + Model firstModel = new LinkedHashModel(); + Model secondModel = new LinkedHashModel(); + + materialiser.getMaterialiserConnection() + .getStatements(null, null, null) + .stream() + .map(MaterialiserIT::deleteContextFromStatement) + .forEach(firstModel::add); + + + secondMaterialiser.getMaterialiserConnection() + .getStatements(null, null, null) + .stream() + .map(MaterialiserIT::deleteContextFromStatement) + .forEach(secondModel::add); + + assertThat(firstModel) + .hasSize((int) firstJenaModel.size()); + + assertThat(secondModel) + .hasSize((int) secondJenaModel.size()); + + materialiser.closeConnection(); + secondMaterialiser.closeConnection(); + } @ParameterizedTest @ArgumentsSource(NamedGraphProvider.class) From c426bf88a79ae68b3cacd52650bd68b98da98a90 Mon Sep 17 00:00:00 2001 From: Jonas Bulcke <127748878+jobulcke@users.noreply.github.com> Date: Fri, 5 Apr 2024 14:21:20 +0200 Subject: [PATCH 13/20] fix: change default format to support relative uris by default (#572) --- .../treenoderelationsfetcher/LdesStructureDiscoverer.java | 4 +--- .../ldes/ldi/discoverer/config/LdesDiscovererConfig.java | 7 +++---- .../ldi/discoverer/config/LdesDiscovererConfigTest.java | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscoverer.java b/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscoverer.java index 3ef50e8c7..1578ed813 100644 --- a/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscoverer.java +++ b/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscoverer.java @@ -12,7 +12,6 @@ public class LdesStructureDiscoverer { private static final Logger log = LoggerFactory.getLogger(LdesStructureDiscoverer.class); - private final TreeRelationsFetcher treeNodeFetcher; private final String startingUrl; private final Lang sourceFormat; private final RequestExecutor requestExecutor; @@ -21,7 +20,6 @@ public LdesStructureDiscoverer(String startingUrl, Lang sourceFormat, RequestExe this.startingUrl = startingUrl; this.sourceFormat = sourceFormat; this.requestExecutor = requestExecutor; - this.treeNodeFetcher = new TreeRelationsFetcher(requestExecutor); } public LdesStructure discoverLdesStructure() { @@ -47,7 +45,7 @@ private List getStartingTreeRelations() { private void fetchRelations(TreeRelation relation, boolean isChildOfRoot) { final TreeNodeRequest treeNodeRequest = new TreeNodeRequest(relation.getUri(), sourceFormat); try { - treeNodeFetcher.fetchTreeRelations(treeNodeRequest) + new TreeRelationsFetcher(requestExecutor).fetchTreeRelations(treeNodeRequest) .parallelStream() .forEach(fetchedRelation -> { if (fetchedRelation.isRequired() || isChildOfRoot) { diff --git a/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/config/LdesDiscovererConfig.java b/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/config/LdesDiscovererConfig.java index ad3c2f14d..a9aface78 100644 --- a/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/config/LdesDiscovererConfig.java +++ b/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/config/LdesDiscovererConfig.java @@ -8,8 +8,7 @@ @Configuration @ConfigurationProperties public class LdesDiscovererConfig { - public static final String DEFAULT_SOURCE_FORMAT = "application/n-quads"; - public static final String DEFAULT_OUTPUT_FORMAT = "text/turtle"; + public static final String DEFAULT_RDF_FORMAT = "text/turtle"; private String url; private String sourceFormat; private String outputFormat; @@ -26,7 +25,7 @@ public void setUrl(String url) { } public String getSourceFormat() { - return sourceFormat == null ? DEFAULT_SOURCE_FORMAT : sourceFormat; + return sourceFormat == null ? DEFAULT_RDF_FORMAT : sourceFormat; } public Lang getSourceFormatAsLang() { @@ -38,7 +37,7 @@ public void setSourceFormat(String sourceFormat) { } public String getOutputFormat() { - return outputFormat == null ? DEFAULT_OUTPUT_FORMAT : outputFormat; + return outputFormat == null ? DEFAULT_RDF_FORMAT : outputFormat; } public Lang getOutputFormatAsLang() { diff --git a/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/config/LdesDiscovererConfigTest.java b/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/config/LdesDiscovererConfigTest.java index e71945091..e94ffce32 100644 --- a/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/config/LdesDiscovererConfigTest.java +++ b/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/config/LdesDiscovererConfigTest.java @@ -18,7 +18,7 @@ void setUp() { @Test void given_EmptyConfig_when_GetSourceFormatAsLang_then_ReturnNQuads() { - final Lang defaultSourceLang = Lang.NQUADS; + final Lang defaultSourceLang = Lang.TURTLE; final Lang actualSourceLang = config.getSourceFormatAsLang(); From 412e0d7074976313169729cf81f9cf226211ecf5 Mon Sep 17 00:00:00 2001 From: Jonas Bulcke <127748878+jobulcke@users.noreply.github.com> Date: Fri, 5 Apr 2024 16:48:11 +0200 Subject: [PATCH 14/20] fea: java 21 upgrade (#573) --- .github/workflows/1.a-build-pr.yml | 10 +++---- .github/workflows/2.a-build-main.yml | 14 ++++----- .../2.b-build-ldes-discoverer-docker.yml | 8 ++--- .github/workflows/2.b-build-ldio-docker.yml | 8 ++--- .../2.b-build-nifi-workbench-docker.yml | 8 ++--- .github/workflows/3.approve-snapshot.yml | 8 ++--- ldes-discoverer.Dockerfile | 4 +-- ldi-core/pom.xml | 1 - ldi-nifi.Dockerfile | 2 +- .../ldes-client-processor/pom.xml | 2 +- ldio.Dockerfile | 4 +-- pom.xml | 30 ++++--------------- 12 files changed, 39 insertions(+), 60 deletions(-) diff --git a/.github/workflows/1.a-build-pr.yml b/.github/workflows/1.a-build-pr.yml index 6b4702ef0..6fd6f9171 100644 --- a/.github/workflows/1.a-build-pr.yml +++ b/.github/workflows/1.a-build-pr.yml @@ -17,14 +17,14 @@ jobs: name: Build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis - - name: Set up JDK 18 - uses: actions/setup-java@v3 + - name: Set up JDK 21 + uses: actions/setup-java@v4 with: - java-version: '18' - distribution: 'adopt' + java-version: '21' + distribution: 'zulu' - name: Cache Maven packages uses: actions/cache@v1 with: diff --git a/.github/workflows/2.a-build-main.yml b/.github/workflows/2.a-build-main.yml index db2c1ffbe..ce6f4354d 100644 --- a/.github/workflows/2.a-build-main.yml +++ b/.github/workflows/2.a-build-main.yml @@ -23,12 +23,12 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v3 - - name: Set up JDK 18 - uses: actions/setup-java@v3 + - uses: actions/checkout@v4 + - name: Set up JDK 21 + uses: actions/setup-java@v4 with: distribution: zulu - java-version: 18 + java-version: 21 - name: Cache SonarCloud packages uses: actions/cache@v1 with: @@ -42,10 +42,10 @@ jobs: key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ runner.os }}-m2 - name: Set up Maven Central Repository - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: - java-version: '18' - distribution: 'adopt' + java-version: '21' + distribution: 'zulu' server-id: ossrh server-username: MAVEN_USERNAME server-password: MAVEN_PASSWORD diff --git a/.github/workflows/2.b-build-ldes-discoverer-docker.yml b/.github/workflows/2.b-build-ldes-discoverer-docker.yml index 6bf687b49..e9f8fcd5d 100644 --- a/.github/workflows/2.b-build-ldes-discoverer-docker.yml +++ b/.github/workflows/2.b-build-ldes-discoverer-docker.yml @@ -20,7 +20,7 @@ jobs: packages: write steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx @@ -30,11 +30,11 @@ jobs: # See: https://github.com/docker/build-push-action/issues/761#issuecomment-1383822381 driver-opts: | image=moby/buildkit:v0.10.6 - - name: Set up JDK 18 - uses: actions/setup-java@v3 + - name: Set up JDK 21 + uses: actions/setup-java@v4 with: distribution: zulu - java-version: 18 + java-version: 21 - name: Determine version run: | export VERSION=$(mvn help:evaluate -Dexpression="project.version" -q -DforceStdout) diff --git a/.github/workflows/2.b-build-ldio-docker.yml b/.github/workflows/2.b-build-ldio-docker.yml index 92cd1e90e..2fea978e0 100644 --- a/.github/workflows/2.b-build-ldio-docker.yml +++ b/.github/workflows/2.b-build-ldio-docker.yml @@ -20,7 +20,7 @@ jobs: packages: write steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx @@ -30,11 +30,11 @@ jobs: # See: https://github.com/docker/build-push-action/issues/761#issuecomment-1383822381 driver-opts: | image=moby/buildkit:v0.10.6 - - name: Set up JDK 18 - uses: actions/setup-java@v3 + - name: Set up JDK 21 + uses: actions/setup-java@v4 with: distribution: zulu - java-version: 18 + java-version: 21 - name: Determine version run: | export VERSION=$(mvn help:evaluate -Dexpression="project.version" -q -DforceStdout) diff --git a/.github/workflows/2.b-build-nifi-workbench-docker.yml b/.github/workflows/2.b-build-nifi-workbench-docker.yml index 8c0587e12..08c620edd 100644 --- a/.github/workflows/2.b-build-nifi-workbench-docker.yml +++ b/.github/workflows/2.b-build-nifi-workbench-docker.yml @@ -20,7 +20,7 @@ jobs: packages: write steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx @@ -30,11 +30,11 @@ jobs: # See: https://github.com/docker/build-push-action/issues/761#issuecomment-1383822381 driver-opts: | image=moby/buildkit:v0.10.6 - - name: Set up JDK 18 - uses: actions/setup-java@v3 + - name: Set up JDK 21 + uses: actions/setup-java@v4 with: distribution: zulu - java-version: 18 + java-version: 21 - name: Determine version run: | export VERSION=$(mvn help:evaluate -Dexpression="project.version" -q -DforceStdout) diff --git a/.github/workflows/3.approve-snapshot.yml b/.github/workflows/3.approve-snapshot.yml index ba181ae13..21b8f2752 100644 --- a/.github/workflows/3.approve-snapshot.yml +++ b/.github/workflows/3.approve-snapshot.yml @@ -14,12 +14,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 - - name: Set up JDK 18 - uses: actions/setup-java@v3 + uses: actions/checkout@v4 + - name: Set up JDK 21 + uses: actions/setup-java@v4 with: distribution: zulu - java-version: 18 + java-version: 21 - name: Define docker variables run: | export VERSION=$(mvn help:evaluate -Dexpression="project.version" -q -DforceStdout) diff --git a/ldes-discoverer.Dockerfile b/ldes-discoverer.Dockerfile index f31bb610a..1e96b2019 100644 --- a/ldes-discoverer.Dockerfile +++ b/ldes-discoverer.Dockerfile @@ -3,7 +3,7 @@ # # INSTALL MAVEN DEPENDENCIES # -FROM maven:3.8.5-amazoncorretto-17 AS builder +FROM maven:3.9.6-amazoncorretto-21 AS builder # MAVEN: application FROM builder AS app-stage @@ -13,7 +13,7 @@ RUN mvn clean install -DskipTests # # RUN THE APPLICATION # -FROM amazoncorretto:17-alpine-jdk +FROM amazoncorretto:21-alpine-jdk RUN adduser -D -u 2000 ldes-discoverer USER ldes-discoverer diff --git a/ldi-core/pom.xml b/ldi-core/pom.xml index 6cb0e2af1..bc8cdbd86 100644 --- a/ldi-core/pom.xml +++ b/ldi-core/pom.xml @@ -30,7 +30,6 @@ 5.9.2 4.2.0 - 1.12.23 7.13.0 1.9.3 diff --git a/ldi-nifi.Dockerfile b/ldi-nifi.Dockerfile index 2536325cd..7ee77cd6b 100644 --- a/ldi-nifi.Dockerfile +++ b/ldi-nifi.Dockerfile @@ -3,7 +3,7 @@ ARG NIFI_DOCKER_IMAGE_VERSION # # INSTALL MAVEN DEPENDENCIES # -FROM maven:3.8.5-amazoncorretto-17 AS builder +FROM maven:3.9.6-amazoncorretto-21 AS builder # MAVEN: application FROM builder AS app-stage diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml index 668b4c7d2..91b3e3f23 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml @@ -34,7 +34,7 @@ net.bytebuddy byte-buddy-dep - ${byte-buddy-dep.version} + ${byte-buddy.version} org.testcontainers diff --git a/ldio.Dockerfile b/ldio.Dockerfile index 36621f7e5..c144efd2a 100644 --- a/ldio.Dockerfile +++ b/ldio.Dockerfile @@ -3,7 +3,7 @@ # # INSTALL MAVEN DEPENDENCIES # -FROM maven:3.8.5-amazoncorretto-17 AS builder +FROM maven:3.9.6-amazoncorretto-21 AS builder # MAVEN: application FROM builder AS app-stage @@ -13,7 +13,7 @@ RUN mvn clean install -DskipTests # # RUN THE APPLICATION # -FROM amazoncorretto:17-alpine-jdk +FROM amazoncorretto:21-alpine-jdk RUN adduser -D -u 2000 ldio USER ldio diff --git a/pom.xml b/pom.xml index 76f835123..1482f800d 100644 --- a/pom.xml +++ b/pom.xml @@ -37,9 +37,10 @@ - 17 - 17 - 17 + 21 + 21 + 21 + 21 3.5.0 3.2.1 @@ -54,7 +55,6 @@ false false - 1.9.0 4.10.0 1.4 @@ -71,14 +71,13 @@ 3.24.2 5.10.0 4.2.0 - 1.14.11 + 1.14.12 2.35.1 5.5.0 5.5.0 1.19.6 1.19.0 2.0.7 - 1.14.7 1.4.11 0.8.10 @@ -120,25 +119,6 @@ - - net.revelc.code - impsort-maven-plugin - ${impsort-maven-plugin.version} - - true - *, java.,javax.,com. - * - true - - - - sort-imports - - sort - - - - org.apache.maven.plugins maven-surefire-plugin From ae6a288cb4247b7ea8d19bd9d3d07f35f5710c35 Mon Sep 17 00:00:00 2001 From: pj-cegeka <119848850+pj-cegeka@users.noreply.github.com> Date: Mon, 8 Apr 2024 09:54:02 +0200 Subject: [PATCH 15/20] fix: error message (#575) --- .../ldi/processors/RDF4JRepositoryMaterialisationProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/RDF4JRepositoryMaterialisationProcessor.java b/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/RDF4JRepositoryMaterialisationProcessor.java index bed1d8c74..e2599226d 100644 --- a/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/RDF4JRepositoryMaterialisationProcessor.java +++ b/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/RDF4JRepositoryMaterialisationProcessor.java @@ -89,7 +89,7 @@ public void onTrigger(ProcessContext context, ProcessSession session) throws Pro session.transfer(flowFiles, SUCCESS); } catch (Exception e) { - getLogger().error("Error sending model to repository", e.getMessage()); + getLogger().error("Error sending model to repository: {}", e.getMessage()); session.transfer(flowFiles, FAILURE); } } From eed923d1d200d11dff478cc9dbac2e0b5d3303cc Mon Sep 17 00:00:00 2001 From: Jonas Bulcke <127748878+jobulcke@users.noreply.github.com> Date: Mon, 8 Apr 2024 11:35:33 +0200 Subject: [PATCH 16/20] fix ldes discoverer bad error logging (#576) --- .../ldi/discoverer/common/LdesDiscovererExecutor.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/common/LdesDiscovererExecutor.java b/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/common/LdesDiscovererExecutor.java index 1658353e8..1eb04cc3c 100644 --- a/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/common/LdesDiscovererExecutor.java +++ b/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/common/LdesDiscovererExecutor.java @@ -26,9 +26,13 @@ public LdesDiscovererExecutor(LdesDiscovererConfig config) { public void run(String... args) { log.info("Running LDESDiscoverer for url {}", config.getUrl()); - final LdesStructure ldesStructure = ldesStructureDiscoverer.discoverLdesStructure(); + try { + final LdesStructure ldesStructure = ldesStructureDiscoverer.discoverLdesStructure(); - log.atInfo().log(ldesStructure.toString()); + log.atInfo().log(ldesStructure.toString()); + } catch (Exception e) { + log.atError().log(e.getMessage()); + } } } \ No newline at end of file From 2e0309c375471512a57458f80a1c9afb97d0bdb1 Mon Sep 17 00:00:00 2001 From: Tomvbe <34196062+Tomvbe@users.noreply.github.com> Date: Mon, 8 Apr 2024 17:01:50 +0200 Subject: [PATCH 17/20] fix: add missing tests (#577) --- .../MemoryTokenServiceLifecycleTest.java | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenServiceLifecycleTest.java diff --git a/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenServiceLifecycleTest.java b/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenServiceLifecycleTest.java new file mode 100644 index 000000000..12fac48fa --- /dev/null +++ b/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenServiceLifecycleTest.java @@ -0,0 +1,60 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.edc.services; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.SequencedCollection; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; + +class MemoryTokenServiceLifecycleTest { + + private MemoryTokenServiceLifecycle serviceLifecycle; + + @BeforeEach + void setUp() { + serviceLifecycle = new MemoryTokenServiceLifecycle(); + } + + @Test + void shouldPauseAndResume() throws InterruptedException { + final SequencedCollection tracker = new ArrayList<>(); + + Thread testThread = new Thread(() -> { + serviceLifecycle.pause(); + tracker.addLast("The lifecycle is paused. Let's check the state!"); + serviceLifecycle.checkPipelineState(); // This should block until resumed + tracker.addLast("Finally, we have resumed!"); + }); + + testThread.start(); + await().atLeast(100, TimeUnit.MILLISECONDS); // Ensures the thread has time to pause + tracker.addLast("The thread with the lifecycle should now be waiting until someone resumes the lifecycle."); + await().atLeast(100, TimeUnit.MILLISECONDS); + tracker.addLast("After a minor nap, the lifecycle should still be waiting.."); + + serviceLifecycle.resume(); + testThread.join(); + + assertThat(tracker.getLast()).isEqualTo("Finally, we have resumed!"); + assertDoesNotThrow(() -> serviceLifecycle.checkPipelineState()); + } + + @Test + void whenLifecycleIsRunning_shouldNotThrowExceptionsOrHang() { + assertDoesNotThrow(() -> serviceLifecycle.checkPipelineState()); + } + + @Test + void whenLifecycleIsShutDown_ThenTerminatedExceptionIsThrown() { + serviceLifecycle.shutdown(); + assertThatThrownBy(() -> serviceLifecycle.checkPipelineState()) + .isInstanceOf(MemoryTokenServiceLifecycle.TerminatedException.class); + } + +} \ No newline at end of file From 25c59303ffd504101c1a2897743f2d95c5d1eb2c Mon Sep 17 00:00:00 2001 From: pj-cegeka <119848850+pj-cegeka@users.noreply.github.com> Date: Tue, 9 Apr 2024 13:53:14 +0200 Subject: [PATCH 18/20] Feat: expand status docs (#578) --- .../ldio-inputs/ldio-ldes-client-connector.md | 8 ++- docs/_ldio/ldio-inputs/ldio-ldes-client.md | 14 ++--- .../pipeline-management/pipeline-status.md | 54 +++++++++++++++++++ 3 files changed, 68 insertions(+), 8 deletions(-) create mode 100644 docs/_ldio/pipeline-management/pipeline-status.md diff --git a/docs/_ldio/ldio-inputs/ldio-ldes-client-connector.md b/docs/_ldio/ldio-inputs/ldio-ldes-client-connector.md index ec9d52f5c..43963891a 100644 --- a/docs/_ldio/ldio-inputs/ldio-ldes-client-connector.md +++ b/docs/_ldio/ldio-inputs/ldio-ldes-client-connector.md @@ -24,7 +24,7 @@ This component exposes two endpoints: transfer with the connector and also keep the transfer alive while consuming the LDES (e.g. request a new token when it expires). -3. http://://token +2. http://://token This endpoint should never be called directly. This is the callback to be provided in the transfer request. The EDC connector will use this callback endpoint to provide the LDES Client with a token. @@ -54,6 +54,12 @@ input: source-format: application/n-quads ``` +## INIT phase + +Contrary to the other ldio-input components, the connector waits in the INIT status for the edc-token and will only progress to the RUNNING status once it has received this token. +More on the statuses in ldio can be found [here.](../pipeline-management/pipeline-status) + + ## Pausing When paused, this component will stop processing the current fragment and not make any calls to the server. diff --git a/docs/_ldio/ldio-inputs/ldio-ldes-client.md b/docs/_ldio/ldio-inputs/ldio-ldes-client.md index 6a70df5fd..182f287b0 100644 --- a/docs/_ldio/ldio-inputs/ldio-ldes-client.md +++ b/docs/_ldio/ldio-inputs/ldio-ldes-client.md @@ -14,14 +14,14 @@ An LDIO wrapper component for the [LDI LDES Client building block](../../core/ld ### General properties -| Property | Description | Required | Default | Example | Supported values | -|:----------------------------|:----------------------------------------------------------------------------------------|:---------|:-------------|:------------------------------------------|:--------------------------------------------------------------| -| urls | List of URLs of the LDES data sources | Yes | N/A | http://localhost:8080/my-ldes | HTTP and HTTPS urls | +| Property | Description | Required | Default | Example | Supported values | +|:----------------------------|:----------------------------------------------------------------------------------------|:---------|:-------------|:------------------------------------------|:------------------------------------------------------------------------------------------------------------------------| +| urls | List of URLs of the LDES data sources | Yes | N/A | http://localhost:8080/my-ldes | HTTP and HTTPS urls | | source-format | The 'Content-Type' that should be requested to the server. | No | text/turtle | application/n-quads | Any type supported by [Apache Jena](https://jena.apache.org/documentation/io/rdf-input.html#determining-the-rdf-syntax) | -| state | 'sqlite', 'memory', 'file' or 'postgres' to indicate how the state should be persisted. | No | memory | sqlite | 'sqlite', 'files' or 'memory' | -| keep-state | Indicates if the state should be persisted on shutdown (n/a for in memory states) | No | false | false | true or false | -| timestamp-path | The property-path used to determine the timestamp on which the members will be ordered | No | N/A | http://www.w3.org/ns/prov#generatedAtTime | A property path | -| enable-exactly-once | Indicates whether member must be send exactly once or at least once | No | true | true | true or false | +| state | 'sqlite', 'memory', 'file' or 'postgres' to indicate how the state should be persisted. | No | memory | sqlite | 'sqlite', 'files' or 'memory' | +| keep-state | Indicates if the state should be persisted on shutdown (n/a for in memory states) | No | false | false | true or false | +| timestamp-path | The property-path used to determine the timestamp on which the members will be ordered | No | N/A | http://www.w3.org/ns/prov#generatedAtTime | A property path | +| enable-exactly-once | Indicates whether member must be send exactly once or at least once | No | true | true | true or false | > **_NOTE:_** The default `source-format` is `text/turtle`, as this rdf format supports relative uri's. However, if > relative uri's are not used, `application/n-quads` or even the binary format `application/rdf+protobuf` are better diff --git a/docs/_ldio/pipeline-management/pipeline-status.md b/docs/_ldio/pipeline-management/pipeline-status.md new file mode 100644 index 000000000..35c0a0020 --- /dev/null +++ b/docs/_ldio/pipeline-management/pipeline-status.md @@ -0,0 +1,54 @@ +--- +layout: default +parent: Pipeline Management +title: Pipeline Status +--- +# Status + +An individual ldio-pipeline can be in one of several different statuses. +These different statuses and their behaviour are dependent on the input component of the pipeline. + +# Overview Of The Status Flow + + +
+graph LR; + INIT --> RUNNING; + INIT --> STOPPED; + RUNNING --> STOPPED; + RUNNING --> HALTED; + HALTED --> RUNNING; + HALTED --> STOPPED; +
+ +The above diagram shows the flow between the different statuses of the pipeline. + +## INIT + +The startup step of the ldio pipeline. +This is the preparation step before the input component can receive data and pass it on to the rest of the pipeline. +In most components this step will take little time. The only exception for now is [the client-connector](../ldio-inputs/ldio-ldes-client-connector). +It is not possible to pause the pipeline while in this state. + +## RUNNING + +This status indicates that the ldio pipeline is running and that the input component is ready to or currently receiving or fetching data. +This is the only state in which the pipeline can be paused. + +## HALTED + +Every ldio pipeline can be paused, the exact behavior of which depends on the ldio input component used. +A more in depth explanation can be found on the pages for [the individual input components.](../ldio-inputs/index) +Currently, the HALTED status can only be reached through manually pausing the pipeline through [the pipeline-api](./pipeline-api). + +## STOPPED + +When a pipeline is deleted, it will first change to the STOPPED status, this ensures the state is correctly saved in stateful components and that the entire pipeline can be gracefully shutdown. +This status will only last a short while before the pipeline is deleted and the status can no longer be queried. +The pipeline can be stopped from any other status. + +## Status Change Source + +The ldio pipeline keeps track of if the last status change was triggered manually or automatic. +Manually changing the status can be done through [the pipeline-api](./pipeline-api). +When in the RUNNING state, this indicates if the pipeline has been started (automatic) or unpaused (manual). \ No newline at end of file From 7063691f97d357d549bfee21aca50307ccb96534 Mon Sep 17 00:00:00 2001 From: Jonas Bulcke <127748878+jobulcke@users.noreply.github.com> Date: Wed, 10 Apr 2024 08:46:30 +0200 Subject: [PATCH 19/20] chore: update codeowners (#579) --- CODEOWNERS | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CODEOWNERS b/CODEOWNERS index bbcc1dd2d..4bdfa35f0 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,2 +1 @@ -* @Yalz @rorlic @pj-cegeka @Tomvbe @jobulcke - +* @Yalz @rorlic @pj-cegeka @jobulcke @ferre-vaes From 6a9baa3b7d521369a0d20bf978c71f83488c3651 Mon Sep 17 00:00:00 2001 From: Jonas Bulcke <127748878+jobulcke@users.noreply.github.com> Date: Wed, 10 Apr 2024 13:25:06 +0200 Subject: [PATCH 20/20] feat: upgrade NiFi base image (#583) --- .github/workflows/0.build-base-nifi-docker.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/0.build-base-nifi-docker.yml b/.github/workflows/0.build-base-nifi-docker.yml index 0443cd9d1..710ec06a8 100644 --- a/.github/workflows/0.build-base-nifi-docker.yml +++ b/.github/workflows/0.build-base-nifi-docker.yml @@ -4,12 +4,12 @@ on: workflow_dispatch: env: - BASE_IMAGE_NAME: openjdk - BASE_IMAGE_TAG: 18-ea-bullseye + BASE_IMAGE_NAME: amazoncorretto + BASE_IMAGE_TAG: 21-alpine-jdk NIFI_VERSION: 1.20.0 DISTRO_PATH: 1.20.0 NIFI_DOCKER_IMAGE_NAME: ldes/nifi - NIFI_DOCKER_IMAGE_VERSION: 1.20.0-jdk18 + NIFI_DOCKER_IMAGE_VERSION: 1.20.0-jdk21 jobs: job-build-push-docker-nifi: @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx @@ -27,13 +27,13 @@ jobs: with: username: ${{ secrets.DOCKER_USER }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build NiFi JDK18 Docker image + - name: Build NiFi JDK21 Docker image run: | echo ENVIRONMENT echo BASE_IMAGE_NAME: ${BASE_IMAGE_NAME} echo BASE_IMAGE_TAG: ${BASE_IMAGE_TAG} echo NIFI_VERSION: ${NIFI_VERSION} - echo TARGET_IMAGE_TAG: ${NIFI_VERSION}-jdk18 + echo TARGET_IMAGE_TAG: ${NIFI_VERSION}-jdk21 echo NIFI_DOCKER_IMAGE_NAME: ${NIFI_DOCKER_IMAGE_NAME} echo NIFI_DOCKER_IMAGE_VERSION: ${NIFI_DOCKER_IMAGE_VERSION} curl -L https://github.com/apache/nifi/archive/refs/tags/rel/nifi-${NIFI_VERSION}.tar.gz | tar -zxv --strip-components=2 nifi-rel-nifi-${NIFI_VERSION}/nifi-docker/dockerhub/