From 68095741e42df99ddb567c528dd66b78854355dd Mon Sep 17 00:00:00 2001 From: Yalz Date: Wed, 11 Sep 2024 11:58:56 +0200 Subject: [PATCH 1/9] chore update upload-download artifacts jobs --- .github/workflows/build-project.yml | 4 ++-- .github/workflows/pr-merged.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-project.yml b/.github/workflows/build-project.yml index 703733dd2..6c7d2fcaa 100644 --- a/.github/workflows/build-project.yml +++ b/.github/workflows/build-project.yml @@ -39,7 +39,7 @@ jobs: if: ${{ github.actor == 'dependabot[bot]' || github.event.pull_request.head.repo.fork }} run: mvn -B verify - name: Upload JARs - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: artifacts path: | @@ -56,7 +56,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Download JARs - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: artifacts path: .github diff --git a/.github/workflows/pr-merged.yml b/.github/workflows/pr-merged.yml index 1c6b4697d..3a793c491 100644 --- a/.github/workflows/pr-merged.yml +++ b/.github/workflows/pr-merged.yml @@ -63,7 +63,7 @@ jobs: MAVEN_GPG_PASSPHRASE: ${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} - name: Upload JARs - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: artifacts path: | @@ -79,7 +79,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Download JARs - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: artifacts path: .github From 06aaf354e011229a623a3da95ff10c2e6e8dfc14 Mon Sep 17 00:00:00 2001 From: Jan Robert <15772440+Yalz@users.noreply.github.com> Date: Wed, 11 Sep 2024 12:17:17 +0200 Subject: [PATCH 2/9] docs: include skolemization docs (#1366) * docs: include skolemization docs * chore: broken doc link --------- Co-authored-by: Yalz --- docs/_configuration/event-stream.md | 2 +- docs/_features/skolemization.md | 56 +++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 docs/_features/skolemization.md diff --git a/docs/_configuration/event-stream.md b/docs/_configuration/event-stream.md index 29ee35467..7061cf0da 100644 --- a/docs/_configuration/event-stream.md +++ b/docs/_configuration/event-stream.md @@ -21,7 +21,7 @@ An Event Stream config needs to contain a couple of items: ingest state objects. The default value of this object is `false` and the property can be omitted. * `ldes:eventSource` object that defines which members are to be retained in the event stream. - When omitted, all members are retained. More info on this can be found [here](./event-stream#configuring-the-member-deletion-on-a-ldes-stream) + When omitted, all members are retained. More info on this can be found [here](./event-stream#configuring-member-deletion-on-an-event-stream) * For more info, visit the [Swagger API documentation.](./admin-api) diff --git a/docs/_features/skolemization.md b/docs/_features/skolemization.md new file mode 100644 index 000000000..fc3f5a397 --- /dev/null +++ b/docs/_features/skolemization.md @@ -0,0 +1,56 @@ +--- +layout: default +title: Skolemization +nav_order: 1 +--- + +# Skolemization + +## What is Skolemization + +In the context of Linked Data, Skolemization is a process used to handle blank nodes or anonymous nodes in RDF +(Resource Description Framework) graphs. +These nodes, which lack unique identifiers, are frequently employed in RDF/S knowledge bases to represent complex +attributes or resources with known properties but unknown identities. + +Skolemization in Linked Data involves the transformation of these blank nodes into Skolem Uniform Resource Identifiers (URIs). +The process enhances the clarity makes it easier to reference these nodes in future datasets. + +This process is particularly useful when dealing with substantial volumes of unstructured data distributed across +diverse sources. +By improving the accuracy and relevance of RDF summaries in relation to original datasets, +Skolemization enhances the efficiency and effectiveness of subsequent queries against these summaries. + +In summary, Skolemization in Linked Data provides a way to handle the complexity introduced by blank nodes in RDF graphs, +thereby enhancing the clarity, interoperability, and usability of the data. + +### Example + +Suppose we have the following RDF triples with a blank node represented as `_:`: + +``` +_:bnode "The Lord of the Rings" . +_:bnode "J.R.R. Tolkien" . +``` + +In this example, `_:` is a blank node that represents a resource (a book in this case) with known properties (title and creator) but an unknown identity. + +Through Skolemization, we can replace the blank node with a Skolem URI. +The Skolem URI is typically a URL that is unique to the blank node and is generated by the system handling the RDF data. +Here’s how it might look: + +``` + "The Lord of the Rings" . + "J.R.R. Tolkien" . +``` + +In this Skolemized version, the blank node has been replaced with the Skolem URI http://example.org/.well-known/genid/123456. +This URI is unique to the resource previously represented by the blank node, and can now be used to reference this +resource in other datasets. +This is a simple example, but it illustrates the basic process of Skolemization in Linked Data. + +## Skolemization in LDES Server + +To enable Skolemization, define the `` property when [setting up your Event Stream](../configuration/event-stream#configuring-a-new-event-stream) with for example with value `"http://example.org"`. + +This will transform all blank nodes to `http://example.org/.well-known/genid/{unique_id}`. From 60fcb6956886a8e69cb72ff5d0f01dbca3fbb48c Mon Sep 17 00:00:00 2001 From: Yalz Date: Wed, 11 Sep 2024 15:13:34 +0200 Subject: [PATCH 3/9] docs: optimise skolemization docs --- docs/_configuration/event-stream.md | 29 ++++++++++++++++++++++------- docs/_features/skolemization.md | 4 +--- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/docs/_configuration/event-stream.md b/docs/_configuration/event-stream.md index 7061cf0da..e7e49d421 100644 --- a/docs/_configuration/event-stream.md +++ b/docs/_configuration/event-stream.md @@ -126,7 +126,7 @@ Before introduction of the event source in the LDES Server, members were directl With the event source, it is possible to delete all views of an Event Stream without losing members so that you can create new views without having to ingest data again. If members need to be deleted directly from the Event Stream when they aren't part of any view, a timebased retention policy of a few seconds can be set on the event source of the Event Stream. -Example: +### Example: ````turtle @prefix ldes: . @@ -136,14 +136,29 @@ ldes:retentionPolicy [ ] ; ```` -### Example +## Skolemization + +To transform blank nodes into Skolem Uniform Resource Identifiers (URIs), +the [skolemization feature](../features/skolemization) can be used; + +An example configuration could look like this: ````turtle @prefix ldes: . +@prefix dcterms: . +@prefix tree: . +@prefix sh: . +@prefix server: . +@prefix xsd: . +@prefix genericES: . + +server:generic-eventstream a ldes:EventStream ; + ldes:timestampPath dcterms:created ; + ldes:versionOfPath dcterms:isVersionOf ; + ldes:skolemizationDomain "http://example.org" ; + tree:shape genericES:shape . -<> a ldes:EventSource ; - ldes:retentionPolicy [ - a ldes:DurationAgoPolicy ; - tree:value "PT5S"^^ ; - ] . +genericES:shape a sh:NodeShape . ```` + +This will transform all blank nodes to `http://example.org/.well-known/genid/{unique_id}`. diff --git a/docs/_features/skolemization.md b/docs/_features/skolemization.md index fc3f5a397..47f1109a6 100644 --- a/docs/_features/skolemization.md +++ b/docs/_features/skolemization.md @@ -51,6 +51,4 @@ This is a simple example, but it illustrates the basic process of Skolemization ## Skolemization in LDES Server -To enable Skolemization, define the `` property when [setting up your Event Stream](../configuration/event-stream#configuring-a-new-event-stream) with for example with value `"http://example.org"`. - -This will transform all blank nodes to `http://example.org/.well-known/genid/{unique_id}`. +Skolemization can be configured on [Event Stream level](../configuration/event-stream#skolemization). From 683eb16e858827b6066ad8bfc513eed419645d48 Mon Sep 17 00:00:00 2001 From: Yalz Date: Wed, 11 Sep 2024 15:29:03 +0200 Subject: [PATCH 4/9] docs: optimise skolemization docs --- docs/_configuration/event-stream.md | 31 +++-------------------------- 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/docs/_configuration/event-stream.md b/docs/_configuration/event-stream.md index e7e49d421..60918dbfd 100644 --- a/docs/_configuration/event-stream.md +++ b/docs/_configuration/event-stream.md @@ -22,6 +22,8 @@ An Event Stream config needs to contain a couple of items: The default value of this object is `false` and the property can be omitted. * `ldes:eventSource` object that defines which members are to be retained in the event stream. When omitted, all members are retained. More info on this can be found [here](./event-stream#configuring-member-deletion-on-an-event-stream) + * `ldes:skolemizationDomain` object that defines the [skolemization](../features/skolemization) domain. + Using `"http://example.com"` as domain will result in all blank nodes being transformed to `http://example.org/.well-known/genid/{unique_id}`. * For more info, visit the [Swagger API documentation.](./admin-api) @@ -134,31 +136,4 @@ ldes:retentionPolicy [ a ldes:LatestVersionSubset ; ldes:amount 0 ; ] ; -```` - -## Skolemization - -To transform blank nodes into Skolem Uniform Resource Identifiers (URIs), -the [skolemization feature](../features/skolemization) can be used; - -An example configuration could look like this: - -````turtle -@prefix ldes: . -@prefix dcterms: . -@prefix tree: . -@prefix sh: . -@prefix server: . -@prefix xsd: . -@prefix genericES: . - -server:generic-eventstream a ldes:EventStream ; - ldes:timestampPath dcterms:created ; - ldes:versionOfPath dcterms:isVersionOf ; - ldes:skolemizationDomain "http://example.org" ; - tree:shape genericES:shape . - -genericES:shape a sh:NodeShape . -```` - -This will transform all blank nodes to `http://example.org/.well-known/genid/{unique_id}`. +```` \ No newline at end of file From 87c9ac9fa8a6782e1603c8858a6559eccd194a32 Mon Sep 17 00:00:00 2001 From: Yalz Date: Wed, 11 Sep 2024 15:58:03 +0200 Subject: [PATCH 5/9] docs: mention fragmentation job cron --- docs/how-to-run.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/how-to-run.md b/docs/how-to-run.md index 914e998a7..8bb59ef91 100644 --- a/docs/how-to-run.md +++ b/docs/how-to-run.md @@ -120,10 +120,10 @@ Here is an explanation provided for all the possibilities on how to tweak and co Bucketisation & pagination batching - spring.batch.jdbc.initialize-schema - Indicates whether the database should be initialized with the tables required for the batching process. This can be either `always`, `embedded` (embedded databases) or `never` - Yes (to let the bucketisation and pagination process run) - + ldes-server.fragmentation-cron + Defines how often Fragmentation Service will check for unprocessed members (when present, trigger fragmentation job). + No + */30 * * * * * Fragment Compaction From 0c347a924e2c05f7abe0c9407ae0ce1d20f3c0c5 Mon Sep 17 00:00:00 2001 From: Yalz Date: Thu, 12 Sep 2024 09:50:19 +0200 Subject: [PATCH 6/9] ci: update approval pipeline job --- .github/workflows/3.approve-snapshot.yml | 5 +++-- .github/workflows/pr-merged.yml | 2 -- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/3.approve-snapshot.yml b/.github/workflows/3.approve-snapshot.yml index 73364bdfa..1040281b7 100644 --- a/.github/workflows/3.approve-snapshot.yml +++ b/.github/workflows/3.approve-snapshot.yml @@ -1,6 +1,8 @@ name: 3. Approve Snapshot on: + release: + types: [ published ] workflow_dispatch: env: @@ -47,9 +49,8 @@ jobs: branch: ci/promote-snapshot delete-branch: true title: '[ci] Promote snapshot ${{ env.NEW_VERSION }}' - reviewers: xdxxxdx body: | As part of QA process, the snapshot version is being promoted to a release version. labels: | - report + promotion automated pr diff --git a/.github/workflows/pr-merged.yml b/.github/workflows/pr-merged.yml index 09be95df0..936fcd9d9 100644 --- a/.github/workflows/pr-merged.yml +++ b/.github/workflows/pr-merged.yml @@ -1,8 +1,6 @@ name: 2. Build & Deploy Project on: - release: - types: [ published ] push: branches: - main From 1f5996183627a5b0a181930268b677613d700368 Mon Sep 17 00:00:00 2001 From: Yalz Date: Thu, 12 Sep 2024 14:11:47 +0200 Subject: [PATCH 7/9] ci: prepare version 3.4.0 --- ldes-fragmentisers/ldes-fragmentisers-common/pom.xml | 2 +- ldes-fragmentisers/ldes-fragmentisers-geospatial/pom.xml | 2 +- ldes-fragmentisers/ldes-fragmentisers-reference/pom.xml | 2 +- .../ldes-fragmentisers-timebased-hierarchical/pom.xml | 2 +- ldes-fragmentisers/pom.xml | 2 +- ldes-server-admin/pom.xml | 2 +- ldes-server-application/pom.xml | 2 +- ldes-server-compaction/pom.xml | 2 +- ldes-server-domain/pom.xml | 2 +- ldes-server-infra-postgres/pom.xml | 2 +- ldes-server-infra-postgres/postgres-admin-repository/pom.xml | 2 +- .../postgres-fragmentation-repository/pom.xml | 2 +- ldes-server-infra-postgres/postgres-ingest-repository/pom.xml | 2 +- ldes-server-infra-postgres/postgres-liquibase/pom.xml | 2 +- .../postgres-pagination-repository/pom.xml | 2 +- .../postgres-retention-repository/pom.xml | 2 +- ldes-server-instrumentation/pom.xml | 2 +- ldes-server-integration-test/pom.xml | 2 +- ldes-server-pagination/pom.xml | 2 +- ldes-server-port-fetch-rest/pom.xml | 2 +- ldes-server-port-fetch/pom.xml | 2 +- ldes-server-port-ingest-rest/pom.xml | 2 +- ldes-server-port-ingest/pom.xml | 2 +- ldes-server-retention/pom.xml | 2 +- pom.xml | 2 +- 25 files changed, 25 insertions(+), 25 deletions(-) diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/pom.xml b/ldes-fragmentisers/ldes-fragmentisers-common/pom.xml index b49a82aa5..4c81c8bf2 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/pom.xml +++ b/ldes-fragmentisers/ldes-fragmentisers-common/pom.xml @@ -5,7 +5,7 @@ ldes-fragmentisers be.vlaanderen.informatievlaanderen.vsds - 3.3.0-SNAPSHOT + 3.4.0-SNAPSHOT 4.0.0 diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/pom.xml b/ldes-fragmentisers/ldes-fragmentisers-geospatial/pom.xml index 6659dc54e..b09b62ffa 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/pom.xml +++ b/ldes-fragmentisers/ldes-fragmentisers-geospatial/pom.xml @@ -3,7 +3,7 @@ ldes-fragmentisers be.vlaanderen.informatievlaanderen.vsds - 3.3.0-SNAPSHOT + 3.4.0-SNAPSHOT 4.0.0 diff --git a/ldes-fragmentisers/ldes-fragmentisers-reference/pom.xml b/ldes-fragmentisers/ldes-fragmentisers-reference/pom.xml index f83a0759e..1cb9db965 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-reference/pom.xml +++ b/ldes-fragmentisers/ldes-fragmentisers-reference/pom.xml @@ -5,7 +5,7 @@ ldes-fragmentisers be.vlaanderen.informatievlaanderen.vsds - 3.3.0-SNAPSHOT + 3.4.0-SNAPSHOT 4.0.0 jar diff --git a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/pom.xml b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/pom.xml index ae78fbe60..837dc6fa6 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/pom.xml +++ b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-fragmentisers - 3.3.0-SNAPSHOT + 3.4.0-SNAPSHOT ldes-fragmentisers-timebased-hierarchical diff --git a/ldes-fragmentisers/pom.xml b/ldes-fragmentisers/pom.xml index a1c8dbb8f..e60483037 100644 --- a/ldes-fragmentisers/pom.xml +++ b/ldes-fragmentisers/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0-SNAPSHOT + 3.4.0-SNAPSHOT ldes-fragmentisers diff --git a/ldes-server-admin/pom.xml b/ldes-server-admin/pom.xml index 2f6b8589f..47fecf78a 100644 --- a/ldes-server-admin/pom.xml +++ b/ldes-server-admin/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0-SNAPSHOT + 3.4.0-SNAPSHOT diff --git a/ldes-server-application/pom.xml b/ldes-server-application/pom.xml index 4f4557f3c..0bce3852d 100644 --- a/ldes-server-application/pom.xml +++ b/ldes-server-application/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0-SNAPSHOT + 3.4.0-SNAPSHOT ldes-server-application diff --git a/ldes-server-compaction/pom.xml b/ldes-server-compaction/pom.xml index cb4b11a1b..ed5971a3a 100644 --- a/ldes-server-compaction/pom.xml +++ b/ldes-server-compaction/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0-SNAPSHOT + 3.4.0-SNAPSHOT ldes-server-compaction diff --git a/ldes-server-domain/pom.xml b/ldes-server-domain/pom.xml index 9f7bbf663..90cc3253e 100644 --- a/ldes-server-domain/pom.xml +++ b/ldes-server-domain/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0-SNAPSHOT + 3.4.0-SNAPSHOT ldes-server-domain diff --git a/ldes-server-infra-postgres/pom.xml b/ldes-server-infra-postgres/pom.xml index 7dcd6de81..1bd4c3efc 100644 --- a/ldes-server-infra-postgres/pom.xml +++ b/ldes-server-infra-postgres/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0-SNAPSHOT + 3.4.0-SNAPSHOT pom diff --git a/ldes-server-infra-postgres/postgres-admin-repository/pom.xml b/ldes-server-infra-postgres/postgres-admin-repository/pom.xml index d2e70aa4e..af0cdc88e 100644 --- a/ldes-server-infra-postgres/postgres-admin-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-admin-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.3.0-SNAPSHOT + 3.4.0-SNAPSHOT postgres-admin-repository diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/pom.xml b/ldes-server-infra-postgres/postgres-fragmentation-repository/pom.xml index c3aaa2736..10572b603 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.3.0-SNAPSHOT + 3.4.0-SNAPSHOT postgres-fragmentation-repository diff --git a/ldes-server-infra-postgres/postgres-ingest-repository/pom.xml b/ldes-server-infra-postgres/postgres-ingest-repository/pom.xml index c41e1806b..234333ebc 100644 --- a/ldes-server-infra-postgres/postgres-ingest-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-ingest-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.3.0-SNAPSHOT + 3.4.0-SNAPSHOT postgres-ingest-repository diff --git a/ldes-server-infra-postgres/postgres-liquibase/pom.xml b/ldes-server-infra-postgres/postgres-liquibase/pom.xml index 420f26ca7..f2c0848f6 100644 --- a/ldes-server-infra-postgres/postgres-liquibase/pom.xml +++ b/ldes-server-infra-postgres/postgres-liquibase/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.3.0-SNAPSHOT + 3.4.0-SNAPSHOT postgres-liquibase diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/pom.xml b/ldes-server-infra-postgres/postgres-pagination-repository/pom.xml index ada03668a..14a364019 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-pagination-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.3.0-SNAPSHOT + 3.4.0-SNAPSHOT postgres-pagination-repository diff --git a/ldes-server-infra-postgres/postgres-retention-repository/pom.xml b/ldes-server-infra-postgres/postgres-retention-repository/pom.xml index 8e6e487bb..403e150fd 100644 --- a/ldes-server-infra-postgres/postgres-retention-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-retention-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.3.0-SNAPSHOT + 3.4.0-SNAPSHOT postgres-retention-repository diff --git a/ldes-server-instrumentation/pom.xml b/ldes-server-instrumentation/pom.xml index d4d2eb9d2..b2b1f472a 100644 --- a/ldes-server-instrumentation/pom.xml +++ b/ldes-server-instrumentation/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0-SNAPSHOT + 3.4.0-SNAPSHOT ldes-server-instrumentation diff --git a/ldes-server-integration-test/pom.xml b/ldes-server-integration-test/pom.xml index be5ff9bb5..2ed29db81 100644 --- a/ldes-server-integration-test/pom.xml +++ b/ldes-server-integration-test/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0-SNAPSHOT + 3.4.0-SNAPSHOT ldes-server-integration-test diff --git a/ldes-server-pagination/pom.xml b/ldes-server-pagination/pom.xml index 191006ba0..b92029b8d 100644 --- a/ldes-server-pagination/pom.xml +++ b/ldes-server-pagination/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0-SNAPSHOT + 3.4.0-SNAPSHOT ldes-server-pagination diff --git a/ldes-server-port-fetch-rest/pom.xml b/ldes-server-port-fetch-rest/pom.xml index dbf904ac0..109eec0d7 100644 --- a/ldes-server-port-fetch-rest/pom.xml +++ b/ldes-server-port-fetch-rest/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0-SNAPSHOT + 3.4.0-SNAPSHOT ldes-server-port-fetch-rest diff --git a/ldes-server-port-fetch/pom.xml b/ldes-server-port-fetch/pom.xml index 460465a3b..732383753 100644 --- a/ldes-server-port-fetch/pom.xml +++ b/ldes-server-port-fetch/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0-SNAPSHOT + 3.4.0-SNAPSHOT ldes-server-port-fetch diff --git a/ldes-server-port-ingest-rest/pom.xml b/ldes-server-port-ingest-rest/pom.xml index bde4eafbd..3b2d318eb 100644 --- a/ldes-server-port-ingest-rest/pom.xml +++ b/ldes-server-port-ingest-rest/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0-SNAPSHOT + 3.4.0-SNAPSHOT ldes-server-port-ingest-rest diff --git a/ldes-server-port-ingest/pom.xml b/ldes-server-port-ingest/pom.xml index d3485eb91..55ef8001b 100644 --- a/ldes-server-port-ingest/pom.xml +++ b/ldes-server-port-ingest/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0-SNAPSHOT + 3.4.0-SNAPSHOT ldes-server-port-ingest diff --git a/ldes-server-retention/pom.xml b/ldes-server-retention/pom.xml index 9f088edce..f2dddcdbb 100644 --- a/ldes-server-retention/pom.xml +++ b/ldes-server-retention/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0-SNAPSHOT + 3.4.0-SNAPSHOT ldes-server-retention diff --git a/pom.xml b/pom.xml index 7eefc18a3..e079614b1 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.3.0-SNAPSHOT + 3.4.0-SNAPSHOT pom From 8728e4fc339174d486e2f81b151826c398967849 Mon Sep 17 00:00:00 2001 From: Jan Robert <15772440+Yalz@users.noreply.github.com> Date: Wed, 18 Sep 2024 12:13:24 +0200 Subject: [PATCH 8/9] feat: pagination rework (#1369) * wip * add extra tests * add extra tests * add extra tests * sonar * sonar * sonar * pr feedback * broken test * cleanup * cleanup --------- Co-authored-by: Yalz --- .../postgres/entity/BucketEntity.java | 3 + .../ingest/postgres/entity/MemberEntity.java | 4 + .../postgres-pagination-repository/pom.xml | 6 + .../PageMemberPostgresRepository.java | 18 ++- .../postgres/PagePostgresRepository.java | 27 ++-- .../PageRelationPostgresRepository.java | 14 +- .../postgres/TreeNodePostgresRepository.java | 12 +- .../postgres/batch/PageAssignmentsWriter.java | 59 ------- .../postgres/batch/UnpagedReader.java | 80 ---------- .../postgres/entity/PageEntity.java | 19 ++- .../postgres/entity/PageMemberEntity.java | 20 +++ .../postgres/entity/PageMemberId.java | 8 + ...ionEntity.java => PageRelationEntity.java} | 26 +-- .../postgres/mapper/TreeRelationMapper.java | 14 +- .../repository/PageEntityRepository.java | 2 +- .../PageMemberEntityRepository.java | 51 +++--- ...java => PageRelationEntityRepository.java} | 6 +- .../pagination/postgres/PaginationSteps.java | 115 ++++++++++++++ .../PostgresPaginationIntegrationTest.java | 69 ++++++++ .../batch/PageAssignmentsWriterTest.java | 65 -------- .../batch/PageRelationProcessorTest.java | 115 -------------- .../resources/application-postgres-test.yml | 3 + .../pagination/postgres/init-paged-test.sql | 20 +++ .../resources/features/pagination.feature | 19 +++ .../ldes/server/CompactionServiceSteps.java | 4 +- .../ldes/server/FragmentationSteps.java | 6 +- .../server/LdesServerIntegrationTest.java | 8 +- .../ldes/server/LdesServerSteps.java | 8 +- .../resources/application-postgres-test.yml | 1 + .../batch/PageRelationProcessor.java | 82 ---------- .../batch/PaginationJobDefinitions.java | 25 +-- .../server/pagination/batch/Paginator.java | 65 ++++++++ .../repositories/PageMemberRepository.java | 23 +++ .../repositories/PageRepository.java | 3 +- .../pagination/valueobjects/Bucket.java | 4 - .../valueobjects/PageAssignment.java | 4 - .../pagination/batch/PaginatorTest.java | 148 ++++++++++++++++++ 37 files changed, 619 insertions(+), 537 deletions(-) delete mode 100644 ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/PageAssignmentsWriter.java delete mode 100644 ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/UnpagedReader.java rename ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/{RelationEntity.java => PageRelationEntity.java} (65%) rename ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/{RelationEntityRepository.java => PageRelationEntityRepository.java} (85%) create mode 100644 ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PaginationSteps.java create mode 100644 ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PostgresPaginationIntegrationTest.java delete mode 100644 ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/PageAssignmentsWriterTest.java delete mode 100644 ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/PageRelationProcessorTest.java create mode 100644 ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/application-postgres-test.yml create mode 100644 ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/init-paged-test.sql create mode 100644 ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/features/pagination.feature delete mode 100644 ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PageRelationProcessor.java create mode 100644 ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/Paginator.java create mode 100644 ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageMemberRepository.java delete mode 100644 ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/Bucket.java delete mode 100644 ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/PageAssignment.java create mode 100644 ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PaginatorTest.java diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/entity/BucketEntity.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/entity/BucketEntity.java index 2c5e51597..4a7343d83 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/entity/BucketEntity.java +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/entity/BucketEntity.java @@ -23,6 +23,9 @@ public class BucketEntity { public BucketEntity() {} + public BucketEntity(Long bucketId) { + this.bucketId = bucketId; + } public BucketEntity(ViewEntity view, String bucketDescriptor) { this.view = view; diff --git a/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/entity/MemberEntity.java b/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/entity/MemberEntity.java index 86a78274e..038962c80 100644 --- a/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/entity/MemberEntity.java +++ b/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/entity/MemberEntity.java @@ -51,6 +51,10 @@ public MemberEntity(String subject, EventStreamEntity collection, String version this.model = model; } + public MemberEntity(long id) { + this.id = id; + } + protected MemberEntity() { } diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/pom.xml b/ldes-server-infra-postgres/postgres-pagination-repository/pom.xml index 14a364019..4774ad589 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-pagination-repository/pom.xml @@ -44,6 +44,12 @@ spring-batch-test test + + be.vlaanderen.informatievlaanderen.vsds + postgres-liquibase + ${project.version} + test + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PageMemberPostgresRepository.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PageMemberPostgresRepository.java index fcbe111d7..9037e48cf 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PageMemberPostgresRepository.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PageMemberPostgresRepository.java @@ -1,6 +1,8 @@ package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.entities.Page; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.PageEntity; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.PageMemberEntityRepository; import be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.PageMemberRepository; import org.springframework.data.jpa.repository.Modifying; @@ -10,13 +12,13 @@ import java.util.List; @Repository -public class PageMemberPostgresRepository implements PageMemberRepository { +public class PageMemberPostgresRepository implements PageMemberRepository, be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageMemberRepository { private final PageMemberEntityRepository entityRepository; - public PageMemberPostgresRepository(PageMemberEntityRepository entityRepository) { + public PageMemberPostgresRepository(PageMemberEntityRepository entityRepository) { this.entityRepository = entityRepository; - } + } @Override @Transactional @@ -30,4 +32,14 @@ public void setPageMembersToNewPage(long newPageId, List pageIds) { public void deleteByViewNameAndMembersIds(ViewName viewName, List memberIds) { entityRepository.deleteAllByBucket_View_EventStream_NameAndBucket_View_NameAndMember_IdIn(viewName.getCollectionName(), viewName.getViewName(), memberIds); } + + @Override + public List getUnpaginatedMembersForBucket(long bucketId) { + return entityRepository.findByBucketIdAndPageIdIsNullOrderByMemberId(bucketId); + } + + @Override + public void assignMembersToPage(Page openPage, List pageMembers) { + entityRepository.updatePageForMembers(new PageEntity(openPage.getId()), openPage.getBucketId(), pageMembers); + } } diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PagePostgresRepository.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PagePostgresRepository.java index d625537e2..4c61ef33b 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PagePostgresRepository.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PagePostgresRepository.java @@ -4,7 +4,9 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.CompactionCandidate; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.entities.Page; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.batch.PaginationRowMapper; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.PageEntity; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.PageEntityRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRelationRepository; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRepository; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Repository; @@ -12,17 +14,18 @@ import java.time.LocalDateTime; import java.util.List; -import java.util.Objects; import java.util.stream.Stream; @Repository public class PagePostgresRepository implements PageRepository { private final JdbcTemplate jdbcTemplate; private final PageEntityRepository pageEntityRepository; + private final PageRelationRepository pageRelationRepository; - public PagePostgresRepository(JdbcTemplate jdbcTemplate, PageEntityRepository pageEntityRepository) { + public PagePostgresRepository(JdbcTemplate jdbcTemplate, PageEntityRepository pageEntityRepository, PageRelationRepository pageRelationRepository) { this.jdbcTemplate = jdbcTemplate; this.pageEntityRepository = pageEntityRepository; + this.pageRelationRepository = pageRelationRepository; } @Override @@ -46,21 +49,15 @@ select DISTINCT p.page_id, p.bucket_id, p.partial_url, v.page_size, COUNT(member @Override @Transactional - public int createPage(Long bucketId, String partialUrl) { - String sql = """ - INSERT INTO pages (bucket_id, expiration, partial_url) - VALUES (?, NULL, ?) - ON CONFLICT (partial_url) DO UPDATE SET bucket_id = pages.bucket_id - RETURNING page_id; - """; + public Page createNextPage(Page parentPage) { + String partialUrl = parentPage.createChildPartialUrl().asString(); + PageEntity newPage = new PageEntity(parentPage.getBucketId(), partialUrl); - return Objects.requireNonNull(jdbcTemplate.queryForObject(sql, Long.class, bucketId, partialUrl)).intValue(); - } + pageEntityRepository.save(newPage); + pageRelationRepository.insertGenericBucketRelation(parentPage.getId(), newPage.getId()); + pageEntityRepository.setPageImmutable(parentPage.getId()); - @Override - @Transactional - public void setPageImmutable(long pageId) { - pageEntityRepository.setPageImmutable(pageId); + return new Page(newPage.getId(), parentPage.getBucketId(), partialUrl, parentPage.getPageSize()); } @Override diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PageRelationPostgresRepository.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PageRelationPostgresRepository.java index 7129565ed..e58ed1ec3 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PageRelationPostgresRepository.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PageRelationPostgresRepository.java @@ -2,7 +2,7 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.RdfConstants; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.RelationEntityRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.PageRelationEntityRepository; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRelationRepository; import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; @@ -11,22 +11,22 @@ @Repository public class PageRelationPostgresRepository implements PageRelationRepository { - private final RelationEntityRepository relationEntityRepository; + private final PageRelationEntityRepository pageRelationEntityRepository; - public PageRelationPostgresRepository(RelationEntityRepository relationEntityRepository) { - this.relationEntityRepository = relationEntityRepository; + public PageRelationPostgresRepository(PageRelationEntityRepository pageRelationEntityRepository) { + this.pageRelationEntityRepository = pageRelationEntityRepository; } @Override @Transactional public void insertGenericBucketRelation(long fromPageId, long toPageId) { - relationEntityRepository.insertRelation(fromPageId, toPageId, RdfConstants.GENERIC_TREE_RELATION); + pageRelationEntityRepository.insertRelation(fromPageId, toPageId, RdfConstants.GENERIC_TREE_RELATION); } @Override @Transactional public void insertBucketRelation(BucketRelation bucketRelation) { - relationEntityRepository.insertRelation( + pageRelationEntityRepository.insertRelation( bucketRelation.fromBucket().createPartialUrl(), bucketRelation.toBucket().createPartialUrl(), bucketRelation.treeRelationType(), @@ -39,6 +39,6 @@ public void insertBucketRelation(BucketRelation bucketRelation) { @Override @Transactional public void updateCompactionBucketRelations(List compactedPageIds, long targetId) { - relationEntityRepository.updateToPageRelations(compactedPageIds, targetId); + pageRelationEntityRepository.updateToPageRelations(compactedPageIds, targetId); } } diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/TreeNodePostgresRepository.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/TreeNodePostgresRepository.java index 89e41ac97..8e7e71fc6 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/TreeNodePostgresRepository.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/TreeNodePostgresRepository.java @@ -13,7 +13,7 @@ import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.projection.TreeRelationProjection; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.PageEntityRepository; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.PageMemberEntityRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.RelationEntityRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.PageRelationEntityRepository; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Repository; @@ -25,13 +25,13 @@ @Repository public class TreeNodePostgresRepository implements TreeNodeRepository { private final PageEntityRepository pageEntityRepository; - private final RelationEntityRepository relationEntityRepository; + private final PageRelationEntityRepository pageRelationEntityRepository; private final PageMemberEntityRepository pageMemberEntityRepository; private final Map versionObjectCreatorMap = new HashMap<>(); - public TreeNodePostgresRepository(PageEntityRepository pageEntityRepository, RelationEntityRepository relationEntityRepository, PageMemberEntityRepository pageMemberEntityRepository) { + public TreeNodePostgresRepository(PageEntityRepository pageEntityRepository, PageRelationEntityRepository pageRelationEntityRepository, PageMemberEntityRepository pageMemberEntityRepository) { this.pageEntityRepository = pageEntityRepository; - this.relationEntityRepository = relationEntityRepository; + this.pageRelationEntityRepository = pageRelationEntityRepository; this.pageMemberEntityRepository = pageMemberEntityRepository; } @@ -40,7 +40,7 @@ public Optional findByFragmentIdentifier(LdesFragmentIdentifier fragme return pageEntityRepository .findTreeNodeByPartialUrl(fragmentIdentifier.asDecodedFragmentId()) .map(page -> { - final List relations = relationEntityRepository.findDistinctByFromPageId(page.getId()); + final List relations = pageRelationEntityRepository.findDistinctByFromPageId(page.getId()); var versionObjectCreator = versionObjectCreatorMap.get(page.getCollectionName()); @@ -60,7 +60,7 @@ public Optional findTreeNodeWithoutMembers(LdesFragmentIdentifier frag return pageEntityRepository .findTreeNodeByPartialUrl(fragmentIdentifier.asDecodedFragmentId()) .map(page -> { - final List relations = relationEntityRepository.findDistinctByFromPageId(page.getId()); + final List relations = pageRelationEntityRepository.findDistinctByFromPageId(page.getId()); return TreeNodeMapper.fromProjection(page, relations, List.of()); }); } diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/PageAssignmentsWriter.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/PageAssignmentsWriter.java deleted file mode 100644 index ee91c6c07..000000000 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/PageAssignmentsWriter.java +++ /dev/null @@ -1,59 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.batch; - -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.PageAssignment; -import org.springframework.batch.item.Chunk; -import org.springframework.batch.item.ItemWriter; -import org.springframework.batch.item.database.JdbcBatchItemWriter; -import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.stereotype.Component; - -import javax.sql.DataSource; -import java.util.Collection; -import java.util.List; - -@Component -public class PageAssignmentsWriter implements ItemWriter> { - private final JdbcBatchItemWriter delegateWriter; - - public PageAssignmentsWriter(JdbcBatchItemWriter batchItemWriter) { - this.delegateWriter = batchItemWriter; - } - - @Override - - public void write(Chunk> chunk) throws Exception { - var items = chunk.getItems() - .stream() - .flatMap(Collection::stream) - .toList(); - - if (!items.isEmpty()) { - delegateWriter.write(new Chunk<>(items)); - } - } - - @Configuration - public static class BatchPageAssignmentWriterConfig { - private static final String SQL = """ - UPDATE page_members - SET page_id = ? - WHERE bucket_id = ? AND member_id = ? - """; - - @Bean - JdbcBatchItemWriter batchPageAssignmentWriter(DataSource dataSource) { - return new JdbcBatchItemWriterBuilder() - .dataSource(dataSource) - .sql(SQL) - .itemPreparedStatementSetter((item, ps) -> { - ps.setLong(1, item.pageId()); - ps.setLong(2, item.bucketId()); - ps.setLong(3, item.memberId()); - }) - .build(); - } - } - -} diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/UnpagedReader.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/UnpagedReader.java deleted file mode 100644 index 46b0f23d8..000000000 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/UnpagedReader.java +++ /dev/null @@ -1,80 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.batch; - -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.entities.UnpagedMember; -import org.springframework.batch.core.configuration.annotation.StepScope; -import org.springframework.batch.item.ExecutionContext; -import org.springframework.batch.item.ItemStreamException; -import org.springframework.batch.item.ItemStreamReader; -import org.springframework.batch.item.database.JdbcCursorItemReader; -import org.springframework.batch.item.database.builder.JdbcCursorItemReaderBuilder; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.stereotype.Component; - -import javax.sql.DataSource; -import java.util.ArrayList; -import java.util.List; - -import static be.vlaanderen.informatievlaanderen.ldes.server.pagination.batch.PaginationJobDefinitions.CHUNK_SIZE; - - -@Component -@StepScope -public class UnpagedReader implements ItemStreamReader> { - private static final String SQL = """ - SELECT member_id, bucket_id - FROM page_members - WHERE bucket_id = ? - AND page_id IS NULL - ORDER BY member_id - """; - private final ItemStreamReader delegate; - - public UnpagedReader(ItemStreamReader delegate) { - this.delegate = delegate; - } - - @Override - public List read() throws Exception { - List items = new ArrayList<>(CHUNK_SIZE); - - for (int i = 0; i < CHUNK_SIZE; i++) { - UnpagedMember item = delegate.read(); - if (item == null) { - // If there are no more items, return what we have (even if it's less than the bundle size) - return items.isEmpty() ? null : items; - } - items.add(item); - } - - return items; - } - - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException { - delegate.open(executionContext); - } - - @Override - public void update(ExecutionContext executionContext) throws ItemStreamException { - delegate.update(executionContext); - } - - @Override - public void close() throws ItemStreamException { - delegate.close(); - } - - @Bean - @StepScope - JdbcCursorItemReader delegateReader(DataSource dataSource, - @Value("#{stepExecutionContext['bucketId']}") Long bucketId) { - return new JdbcCursorItemReaderBuilder() - .name("unpagedReader") - .dataSource(dataSource) - .sql(SQL) - .preparedStatementSetter(ps -> ps.setLong(1, bucketId)) - .rowMapper((rs, rowNum) -> new UnpagedMember(rs.getLong(1), rs.getLong(2))) - .build(); - } -} \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageEntity.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageEntity.java index 104c77a57..7bd32d38c 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageEntity.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageEntity.java @@ -31,16 +31,19 @@ public class PageEntity { private String partialUrl; @OneToMany(mappedBy = "fromPage") - private List relations; + private List relations; public PageEntity() { } - public PageEntity(BucketEntity bucket, String partialUrl, List relations) { - this.bucket = bucket; - this.immutable = true; + public PageEntity(Long pageId) { + this.id = pageId; + } + + public PageEntity(Long bucketId, String partialUrl) { + this.bucket = new BucketEntity(bucketId); + this.immutable = false; this.partialUrl = partialUrl; - this.relations = relations; } public Long getId() { @@ -67,11 +70,15 @@ public String getPartialUrl() { return partialUrl; } - public List getRelations() { + public List getRelations() { return relations; } public boolean isView() { return bucket.getView().getComposedViewName().equals("/%s".formatted(partialUrl)); } + + public void setId(Long id) { + this.id = id; + } } diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageMemberEntity.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageMemberEntity.java index fd5fdf72f..6612d9a23 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageMemberEntity.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageMemberEntity.java @@ -28,4 +28,24 @@ public class PageMemberEntity { @OnDelete(action = OnDeleteAction.CASCADE) @JoinColumn(name = "page_id", columnDefinition = "BIGINT") private PageEntity page; + + public void setPage(PageEntity page) { + this.page = page; + } + + public void setMember(Long memberId) { + this.member = new MemberEntity(memberId); + } + + public MemberEntity getMember() { + return member; + } + + public BucketEntity getBucket() { + return bucket; + } + + public PageEntity getPage() { + return page; + } } diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageMemberId.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageMemberId.java index 6f1e84351..b20c6c4bf 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageMemberId.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageMemberId.java @@ -13,6 +13,14 @@ public class PageMemberId implements Serializable { @Column(name = "bucket_id", nullable = false, columnDefinition = "BIGINT") private Long bucketId; + public PageMemberId() { + } + + public PageMemberId(Long memberId, Long bucketId) { + this.memberId = memberId; + this.bucketId = bucketId; + } + @Override public final boolean equals(Object o) { if (this == o) return true; diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/RelationEntity.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageRelationEntity.java similarity index 65% rename from ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/RelationEntity.java rename to ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageRelationEntity.java index 2e569cf01..28053b7e7 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/RelationEntity.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageRelationEntity.java @@ -6,7 +6,7 @@ @Entity @Table(name = "page_relations") -public class RelationEntity { +public class PageRelationEntity { @EmbeddedId private RelationId relationId; @@ -34,35 +34,13 @@ public class RelationEntity { @Column(name = "path", columnDefinition = "VARCHAR(255)") private String treePath; - - public RelationEntity() { - } - - public RelationEntity(PageEntity fromPage, PageEntity toPage, String treeRelationType, String treeValue, String treeValueType, String treePath) { - this.fromPage = fromPage; - this.toPage = toPage; - this.treeRelationType = treeRelationType; - this.treeValue = treeValue; - this.treeValueType = treeValueType; - this.treePath = treePath; - } - - public RelationEntity(RelationId relationId, String treeRelationType, String treeValue, String treeValueType, String treePath) { - this.relationId = relationId; - this.treeRelationType = treeRelationType; - this.treeValue = treeValue; - this.treeValueType = treeValueType; - this.treePath = treePath; + protected PageRelationEntity() { } public RelationId getRelationId() { return relationId; } - public PageEntity getFromPage() { - return fromPage; - } - public PageEntity getToPage() { return toPage; } diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/mapper/TreeRelationMapper.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/mapper/TreeRelationMapper.java index a6ca3a68a..d069f4dd9 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/mapper/TreeRelationMapper.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/mapper/TreeRelationMapper.java @@ -2,19 +2,19 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.TreeRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.RelationEntity; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.PageRelationEntity; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.projection.TreeRelationProjection; public class TreeRelationMapper { private TreeRelationMapper() {} - public static TreeRelation fromRelationEntity(RelationEntity relationEntity) { + public static TreeRelation fromRelationEntity(PageRelationEntity pageRelationEntity) { return new TreeRelation( - relationEntity.getTreePath(), - LdesFragmentIdentifier.fromFragmentId(relationEntity.getToPage().getPartialUrl()), - relationEntity.getTreeValue(), - relationEntity.getTreeValueType(), - relationEntity.getTreeRelationType() + pageRelationEntity.getTreePath(), + LdesFragmentIdentifier.fromFragmentId(pageRelationEntity.getToPage().getPartialUrl()), + pageRelationEntity.getTreeValue(), + pageRelationEntity.getTreeValueType(), + pageRelationEntity.getTreeRelationType() ); } diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageEntityRepository.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageEntityRepository.java index 54558b555..4b3a974b8 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageEntityRepository.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageEntityRepository.java @@ -47,7 +47,7 @@ JOIN views USING (view_id) @Query(value = "SELECT p.id as fragmentId, COUNT(*) AS size, r.toPage.id AS toPage, p.immutable AS immutable, " + "p.expiration AS expiration, " + "p.bucket.bucketId AS bucketId, p.partialUrl AS partialUrl " + - "FROM PageEntity p JOIN BucketEntity b ON p.bucket = b JOIN ViewEntity v ON b.view = v JOIN RelationEntity r ON p = r.fromPage " + + "FROM PageEntity p JOIN BucketEntity b ON p.bucket = b JOIN ViewEntity v ON b.view = v JOIN PageRelationEntity r ON p = r.fromPage " + "WHERE v.eventStream.name = :collectionName AND v.name = :viewName " + "GROUP BY p.id, r.toPage.id " + "HAVING COUNT(*) < :capacityPerPage") diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageMemberEntityRepository.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageMemberEntityRepository.java index 860a3ea64..d47f23e71 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageMemberEntityRepository.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageMemberEntityRepository.java @@ -1,12 +1,14 @@ package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository; import be.vlaanderen.informatievlaanderen.ldes.server.ingest.postgres.projection.TreeMemberProjection; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.PageEntity; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.PageMemberEntity; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.PageMemberId; import jakarta.persistence.Tuple; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import java.util.List; @@ -14,31 +16,42 @@ public interface PageMemberEntityRepository extends JpaRepository findAllMembersByPageId(long pageId); - @Modifying - @Query("UPDATE PageMemberEntity m SET m.page = (SELECT p FROM PageEntity p WHERE p.id = :newPageId) WHERE m.page.id IN (:pageIds)") - void setPageMembersToNewPage(long newPageId, List pageIds); + @Modifying + @Query("UPDATE PageMemberEntity m SET m.page = (SELECT p FROM PageEntity p WHERE p.id = :newPageId) WHERE m.page.id IN (:pageIds)") + void setPageMembersToNewPage(long newPageId, List pageIds); @Query(value = """ - select v.name, count(*) - from page_members - JOIN buckets b on b.bucket_id = page_members.bucket_id - JOIN views v on v.view_id = b.view_id - JOIN collections c on c.collection_id = v.collection_id - WHERE c.name = :collectionName - group by v.name - """, nativeQuery = true) + select v.name, count(*) + from page_members + JOIN buckets b on b.bucket_id = page_members.bucket_id + JOIN views v on v.view_id = b.view_id + JOIN collections c on c.collection_id = v.collection_id + WHERE c.name = :collectionName + group by v.name + """, nativeQuery = true) List getBucketisedMemberCounts(String collectionName); @Query(value = """ - select v.name, count(*) - from page_members - JOIN buckets b on b.bucket_id = page_members.bucket_id - JOIN views v on v.view_id = b.view_id - JOIN collections c on c.collection_id = v.collection_id - WHERE page_id IS NOT NULL AND c.name = :collectionName - group by v.name - """, nativeQuery = true) + select v.name, count(*) + from page_members + JOIN buckets b on b.bucket_id = page_members.bucket_id + JOIN views v on v.view_id = b.view_id + JOIN collections c on c.collection_id = v.collection_id + WHERE page_id IS NOT NULL AND c.name = :collectionName + group by v.name + """, nativeQuery = true) List getPaginatedMemberCounts(String collectionName); + @Modifying + @Query(""" + UPDATE PageMemberEntity p SET p.page = :page WHERE p.pageMemberId.bucketId = :bucketId + AND p.pageMemberId.memberId IN :memberIds + """) + void updatePageForMembers(@Param("page") PageEntity page, @Param("bucketId") Long bucketId, @Param("memberIds") List memberIds); + + @Query("SELECT pm.pageMemberId.memberId FROM PageMemberEntity pm WHERE pm.pageMemberId.bucketId = :bucketId AND " + + "pm.page IS NULL ORDER BY pm.pageMemberId.memberId") + List findByBucketIdAndPageIdIsNullOrderByMemberId(long bucketId); + void deleteAllByBucket_View_EventStream_NameAndBucket_View_NameAndMember_IdIn(String collectionName, String viewName, List memberIds); } \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/RelationEntityRepository.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageRelationEntityRepository.java similarity index 85% rename from ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/RelationEntityRepository.java rename to ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageRelationEntityRepository.java index 02f3293fd..7f4bf5f80 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/RelationEntityRepository.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageRelationEntityRepository.java @@ -1,6 +1,6 @@ package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.RelationEntity; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.PageRelationEntity; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.RelationId; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.projection.TreeRelationProjection; import org.springframework.data.jpa.repository.JpaRepository; @@ -9,7 +9,7 @@ import java.util.List; -public interface RelationEntityRepository extends JpaRepository { +public interface PageRelationEntityRepository extends JpaRepository { @Modifying @Query(value = """ @@ -29,7 +29,7 @@ INSERT INTO page_relations (from_page_id, to_page_id, relation_type, value, valu List findDistinctByFromPageId(long pageId); @Modifying - @Query("UPDATE RelationEntity r SET r.toPage = ( SELECT p FROM PageEntity p WHERE p.id = :targetId ) " + + @Query("UPDATE PageRelationEntity r SET r.toPage = ( SELECT p FROM PageEntity p WHERE p.id = :targetId ) " + "WHERE r.toPage.id IN :ids OR r.fromPage.id IN : ids") void updateToPageRelations(List ids, long targetId); } diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PaginationSteps.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PaginationSteps.java new file mode 100644 index 000000000..a70e039fb --- /dev/null +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PaginationSteps.java @@ -0,0 +1,115 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres; + +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.entities.Page; +import io.cucumber.java.Before; +import io.cucumber.java.en.And; +import io.cucumber.java.en.Given; +import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.stream.LongStream; + +import static org.assertj.core.api.Assertions.assertThat; + +public class PaginationSteps extends PostgresPaginationIntegrationTest { + private List memberIds; + private int pageCount; + private Long bucketId; + private Page openPage; + + @Before + public void setUp() { + pageCount = pageEntityRepository.findAll().size(); + bucketId = bucketEntityRepository.findAll().getFirst().getBucketId(); + openPage = pageRepository.getOpenPage(bucketId); + } + + @Given("I have {int} unpaged members in one bucket") + public void iHaveUnpagedMembersForBucket(int memberCount) { + memberIds = LongStream.rangeClosed(1, memberCount).boxed().toList(); + + saveMembers(memberIds); + saveMemberBuckets(memberIds); + } + + @When("I assign the members to the page") + public void iAssignTheMembersToThePage() { + pageMemberRepository.assignMembersToPage(openPage, memberIds); + } + + @Then("I expect a new page is created") + public void iExpectANewPageIsCreated() { + var currentPageCount = pageEntityRepository.count(); + assertThat(currentPageCount).isGreaterThan(pageCount); + } + + @Then("the open page has space for {int} more members") + public void theOpenPageContainsMembers(int memberCount) { + var page = pageRepository.getOpenPage(bucketId); + assertThat(page.getAvailableMemberSpace()).isEqualTo(memberCount); + } + + @Then("I expect no more unpaged members") + public void iExpectNoMoreUnpagedMembers() { + var unpagedMembersCount = pageMemberEntityRepository.findAll() + .stream() + .filter(entity -> entity.getBucket() == null) + .count(); + assertThat(unpagedMembersCount).isZero(); + } + + @When("I create a page given the mutable page") + public void iCreateAPageGivenTheMutablePage() { + openPage = pageRepository.getOpenPage(bucketId); + pageRepository.createNextPage(openPage); + } + + @And("The old page has a generic relation to the new page") + public void theOldPageHasAGenericRelationToTheNewPage() { + var oldOpenPageId = openPage.getId(); + openPage = pageRepository.getOpenPage(bucketId); + var newOpenPageId = openPage.getId(); + + var relationPresent = pageRelationEntityRepository.findAll() + .stream() + .anyMatch(pageRelationEntity -> + pageRelationEntity.getRelationId().getFromPageId().equals(oldOpenPageId) && + pageRelationEntity.getRelationId().getToPageId().equals(newOpenPageId) && + pageRelationEntity.getTreeRelationType().equals("https://w3id.org/tree#Relation")); + + assertThat(relationPresent).isTrue(); + } + + private void saveMembers(List memberIds) { + var collectionId = eventStreamEntityRepository.findAll().getFirst().getId(); + String eventStream = "http://example.com/es"; + String sql = "INSERT INTO members (subject, collection_id, version_of, timestamp, transaction_id, member_model, old_id) VALUES (?,?,?,?,?,?,?)"; + + + final List batchArgs = memberIds.stream() + .map(member -> new Object[]{ + "http://example.com/es/%d".formatted(member), collectionId, eventStream, + LocalDateTime.now(), 0, new byte[]{}, + "%s/%s/%d".formatted(eventStream, eventStream, member) + }) + .toList(); + + jdbcTemplate.batchUpdate(sql, batchArgs); + } + + private void saveMemberBuckets(List memberIds) { + + String sql = "insert into page_members (bucket_id, member_id) VALUES (?, ?)"; + + final List batchArgs = memberIds.stream() + .map(member -> new Object[]{ + bucketId, member + }) + .toList(); + + jdbcTemplate.batchUpdate(sql, batchArgs); + } + +} diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PostgresPaginationIntegrationTest.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PostgresPaginationIntegrationTest.java new file mode 100644 index 000000000..47323daaa --- /dev/null +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PostgresPaginationIntegrationTest.java @@ -0,0 +1,69 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres; + +import be.vlaanderen.informatievlaanderen.ldes.server.admin.postgres.eventstream.repository.EventStreamEntityRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.admin.postgres.view.repository.ViewEntityRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.repository.BucketEntityRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.ingest.postgres.repository.MemberEntityRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.PageEntityRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.PageMemberEntityRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.PageRelationEntityRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageMemberRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRepository; +import io.cucumber.spring.CucumberContextConfiguration; +import io.zonky.test.db.AutoConfigureEmbeddedDatabase; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.jdbc.Sql; + +@CucumberContextConfiguration +@EnableAutoConfiguration +@DataJpaTest +@AutoConfigureEmbeddedDatabase(refresh = AutoConfigureEmbeddedDatabase.RefreshMode.AFTER_EACH_TEST_METHOD) +@ActiveProfiles("postgres-test") +@EntityScan(basePackages = {"be.vlaanderen.informatievlaanderen.ldes.server"}) +@ComponentScan(basePackages = {"be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres", + "be.vlaanderen.informatievlaanderen.ldes.server.ingest.postgres", + "be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres", + "be.vlaanderen.informatievlaanderen.ldes.server.admin.postgres.eventstream" +}) +@ContextConfiguration(classes = {PageEntityRepository.class}) +@EnableJpaRepositories(basePackageClasses = {PageEntityRepository.class, PageMemberEntityRepository.class, PageRelationEntityRepository.class, +MemberEntityRepository.class, BucketEntityRepository.class, ViewEntityRepository.class, EventStreamEntityRepository.class}) +@Sql(value = {"init-paged-test.sql"}, executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD) + +@SuppressWarnings("java:S2187") +public class PostgresPaginationIntegrationTest { + + @Autowired + PageRepository pageRepository; + @Autowired + PageMemberRepository pageMemberRepository; + + @Autowired + PageEntityRepository pageEntityRepository; + @Autowired + PageMemberEntityRepository pageMemberEntityRepository; + @Autowired + PageRelationEntityRepository pageRelationEntityRepository; + + @Autowired + JdbcTemplate jdbcTemplate; + + @Autowired + MemberEntityRepository memberRepository; + @Autowired + BucketEntityRepository bucketEntityRepository; + + @Autowired + EventStreamEntityRepository eventStreamEntityRepository; + @Autowired + ViewEntityRepository viewEntityRepository; + +} diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/PageAssignmentsWriterTest.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/PageAssignmentsWriterTest.java deleted file mode 100644 index 2563eb98b..000000000 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/PageAssignmentsWriterTest.java +++ /dev/null @@ -1,65 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.batch; - -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.PageAssignment; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.ArgumentCaptor; -import org.mockito.Captor; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.batch.item.Chunk; -import org.springframework.batch.item.database.JdbcBatchItemWriter; - -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; - -@ExtendWith(MockitoExtension.class) -class PageAssignmentsWriterTest { - private static final long BUCKET_ID = 123; - private static final long PAGE_ID = 1234; - private static final long CHILD_PAGE_ID = 12345; - - @Mock - private JdbcBatchItemWriter delegateWriter; - private PageAssignmentsWriter pageAssignmentsWriter; - @Captor - private ArgumentCaptor> captor; - - @BeforeEach - public void setup() { - pageAssignmentsWriter = new PageAssignmentsWriter(delegateWriter); - } - - @Test - void test_write() throws Exception { - Chunk> chunk = Chunk.of( - List.of(new PageAssignment(PAGE_ID, BUCKET_ID, 1), - new PageAssignment(PAGE_ID, BUCKET_ID, 2)), - List.of(new PageAssignment(CHILD_PAGE_ID, BUCKET_ID, 3), - new PageAssignment(CHILD_PAGE_ID, BUCKET_ID, 4)) - ); - - pageAssignmentsWriter.write(chunk); - - verify(delegateWriter).write(captor.capture()); - assertThat(captor.getValue().size()).isEqualTo(4); - } - - @Test - void given_emptyChunk_test_write() throws Exception { - pageAssignmentsWriter.write(new Chunk<>()); - - verifyNoInteractions(delegateWriter); - } - - @Test - void given_chunkWithEmptyLists_test_write() throws Exception { - pageAssignmentsWriter.write(Chunk.of(List.of())); - - verifyNoInteractions(delegateWriter); - } -} \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/PageRelationProcessorTest.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/PageRelationProcessorTest.java deleted file mode 100644 index db7dd6b82..000000000 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/PageRelationProcessorTest.java +++ /dev/null @@ -1,115 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.batch; - -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.batch.PageRelationProcessor; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.entities.Page; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.entities.UnpagedMember; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRelationRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.PageAssignment; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import java.util.Collections; -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.*; - -@ExtendWith(SpringExtension.class) -class PageRelationProcessorTest { - private static final int PAGE_SIZE = 150; - private static final long BUCKET_ID = 12; - private static final int UNPROCESSED_MEMBER_COUNT = 32; - private static final long PARENT_PAGE_ID = 2; - private static final long CHILD_PAGE_ID = PARENT_PAGE_ID + 1; - - @Mock - private PageRepository pageRepository; - @Mock - private PageRelationRepository pageRelationRepository; - @InjectMocks - private PageRelationProcessor pageRelationProcessor; - - @Test - void given_FullPage_when_Process_then_CreateNewPage() { - final Page parentPage = new Page(PARENT_PAGE_ID, BUCKET_ID, "/mobility-hindrances/by-page?pageNumber=2", PAGE_SIZE, PAGE_SIZE); - when(pageRepository.getOpenPage(BUCKET_ID)).thenReturn(parentPage); - when(pageRepository.createPage(BUCKET_ID, parentPage.createChildPartialUrl().asString())).thenReturn((int) CHILD_PAGE_ID); - - final List result = pageRelationProcessor.process( - Collections.nCopies(UNPROCESSED_MEMBER_COUNT, new UnpagedMember(0, BUCKET_ID))); - - verify(pageRepository).createPage(parentPage.getBucketId(), parentPage.createChildPartialUrl().asString()); - verify(pageRelationRepository).insertGenericBucketRelation(eq(parentPage.getId()), anyLong()); - assertThat(result) - .isNotNull() - .filteredOn(PageAssignment::pageId, CHILD_PAGE_ID) - .filteredOn(PageAssignment::bucketId, BUCKET_ID) - .hasSize(UNPROCESSED_MEMBER_COUNT); - } - - @Test - void given_RootPage_when_Process_then_CreateNewPage() { - final Page rootPage = new Page(PARENT_PAGE_ID, BUCKET_ID, "/mobility-hindrances/by-page", PAGE_SIZE, 0); - when(pageRepository.getOpenPage(BUCKET_ID)).thenReturn(rootPage); - when(pageRepository.createPage(BUCKET_ID, rootPage.createChildPartialUrl().asString())).thenReturn((int) CHILD_PAGE_ID); - - final List result = pageRelationProcessor.process( - Collections.nCopies(UNPROCESSED_MEMBER_COUNT, new UnpagedMember(0, BUCKET_ID))); - - verify(pageRepository).createPage(rootPage.getBucketId(), rootPage.createChildPartialUrl().asString()); - verify(pageRelationRepository).insertGenericBucketRelation(eq(rootPage.getId()), anyLong()); - assertThat(result) - .isNotNull() - .filteredOn(PageAssignment::pageId, CHILD_PAGE_ID) - .filteredOn(PageAssignment::bucketId, BUCKET_ID) - .hasSize(UNPROCESSED_MEMBER_COUNT); - } - - @Test - void given_PageWithSpace_when_Process_then_UseExistingPage() { - final Page rootPage = new Page(PARENT_PAGE_ID, BUCKET_ID, "/mobility-hindrances/by-page?pageNumber=3", PAGE_SIZE, 100); - when(pageRepository.getOpenPage(BUCKET_ID)).thenReturn(rootPage); - - final List result = pageRelationProcessor.process( - Collections.nCopies(UNPROCESSED_MEMBER_COUNT, new UnpagedMember(0, BUCKET_ID))); - - assertThat(result) - .isNotNull() - .filteredOn(PageAssignment::pageId, PARENT_PAGE_ID) - .filteredOn(PageAssignment::bucketId, BUCKET_ID) - .hasSize(UNPROCESSED_MEMBER_COUNT); - } - - @Test - void given_PageWithInsufficientSpace_when_Process_then_UseBothExistingAndNewPage() { - final int alreadyAssignedMemberCount = 132; - final Page page = new Page(PARENT_PAGE_ID, BUCKET_ID, "/mobility-hindrances/by-page?pageNumber=3", PAGE_SIZE, alreadyAssignedMemberCount); - when(pageRepository.getOpenPage(BUCKET_ID)).thenReturn(page); - when(pageRepository.createPage(BUCKET_ID, page.createChildPartialUrl().asString())).thenReturn((int) CHILD_PAGE_ID); - - final List result = pageRelationProcessor.process( - Collections.nCopies(UNPROCESSED_MEMBER_COUNT, new UnpagedMember(0, BUCKET_ID))); - - assertThat(result) - .isNotNull() - .filteredOn(PageAssignment::pageId, PARENT_PAGE_ID) - .filteredOn(PageAssignment::bucketId, BUCKET_ID) - .hasSize(PAGE_SIZE - alreadyAssignedMemberCount); - assertThat(result) - .isNotNull() - .filteredOn(PageAssignment::pageId, CHILD_PAGE_ID) - .filteredOn(PageAssignment::bucketId, BUCKET_ID) - .hasSize(UNPROCESSED_MEMBER_COUNT - (PAGE_SIZE - alreadyAssignedMemberCount)); - } - - @Test - void given_ZeroAssignedMembers_when_Process_then_ReturnEmptyList() { - final List result = pageRelationProcessor.process(List.of()); - - assertThat(result).isEmpty(); - } -} \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/application-postgres-test.yml b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/application-postgres-test.yml new file mode 100644 index 000000000..b8c8bf725 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/application-postgres-test.yml @@ -0,0 +1,3 @@ +spring: + liquibase: + change-log: classpath:/db/changelog/master.xml \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/init-paged-test.sql b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/init-paged-test.sql new file mode 100644 index 000000000..b75701c22 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/init-paged-test.sql @@ -0,0 +1,20 @@ +--TODO remove after VSDSPUB-1433 is done +ALTER TABLE buckets DISABLE TRIGGER ALL; + +WITH new_collection AS ( + INSERT INTO collections (name, timestamp_path, version_of_path, create_versions, is_closed, skolemization_domain) + VALUES ('es', '', '', false, false, null) + RETURNING collection_id +), + new_view AS ( + INSERT INTO views (collection_id, name, fragmentations, retention_policies, page_size) + SELECT collection_id, 'es/view', '[]', '[]', 10 FROM new_collection + RETURNING view_id + ), + new_bucket AS ( + INSERT INTO buckets (bucket, view_id) + SELECT '', view_id FROM new_view + RETURNING bucket_id + ) +INSERT INTO pages (bucket_id, partial_url, immutable) +SELECT bucket_id, '/es/view?pageNumber=1', false FROM new_bucket; \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/features/pagination.feature b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/features/pagination.feature new file mode 100644 index 000000000..c99c34f6a --- /dev/null +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/features/pagination.feature @@ -0,0 +1,19 @@ +Feature: Pagination Interactions + + Scenario: Assign members to page (5/10 members) + Given I have 5 unpaged members in one bucket + When I assign the members to the page + Then I expect no more unpaged members + And the open page has space for 5 more members + + Scenario: Assign members to page (10/10 members) + Given I have 10 unpaged members in one bucket + When I assign the members to the page + And I expect no more unpaged members + + Scenario: Create new page + When I create a page given the mutable page + Then I expect a new page is created + And The old page has a generic relation to the new page + + diff --git a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/CompactionServiceSteps.java b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/CompactionServiceSteps.java index 52ae3456d..46daf65a3 100644 --- a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/CompactionServiceSteps.java +++ b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/CompactionServiceSteps.java @@ -62,7 +62,7 @@ public void verifyCreationOfTheFollowingFragments(int i) { @And("verify the following pages have no relation pointing to them") public void verifyUpdateOfPredecessorRelations(List ids) { await().untilAsserted(() -> { - var count = relationEntityRepository.findAll() + var count = pageRelationEntityRepository.findAll() .stream() .filter(relationEntity -> ids.contains(relationEntity.getToPage().getId())) .count(); @@ -86,7 +86,7 @@ public void verifyRemovalOfPages(List ids) { @And("verify {long} pages have a relation pointing to the new page {long}") public void verifyUpdateOfPredecessorRelations(long pointingCount, long id) { await().untilAsserted(() -> { - var countNewPage = relationEntityRepository.findAll() + var countNewPage = pageRelationEntityRepository.findAll() .stream() .filter(relationEntity -> relationEntity.getToPage().getId().equals(id)) .count(); diff --git a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/FragmentationSteps.java b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/FragmentationSteps.java index efe97e1c9..1b067aa8e 100644 --- a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/FragmentationSteps.java +++ b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/FragmentationSteps.java @@ -82,7 +82,7 @@ public void iFetchTheNextFragmentThroughTheFirst(String relation) { @Then("this fragment only has {int} {string} relation") public void thisFragmentOnlyHasOne(int expectedRelationCount, String relation) { - await().atMost(Duration.of(FRAGMENTATION_POLLING_RATE, ChronoUnit.SECONDS)).until(() -> { + await().atMost(Duration.of(60, ChronoUnit.SECONDS)).until(() -> { fetchFragment(currentPath); int relationCount = currentFragment.listStatements(null, RDF.type, createResource(TREE + relation)) .toList().size(); @@ -106,7 +106,7 @@ public void thisFragmentIsImmutable() { @And("this fragment contains {int} members") public void thisFragmentContainsMembers(int expectedMemberCount) { - await().atMost(Duration.of(FRAGMENTATION_POLLING_RATE, ChronoUnit.SECONDS)).until(() -> { + await().atMost(Duration.of(60, ChronoUnit.SECONDS)).until(() -> { fetchFragment(currentPath); return MemberCounter.countMembers(expectedMemberCount).matches(currentFragment); }); @@ -119,7 +119,7 @@ public void thisFragmentIsNotImmutable() { @And("this fragment has no relations") public void thisFragmentHasNoRelations() { - await().atMost(Duration.of(FRAGMENTATION_POLLING_RATE, ChronoUnit.SECONDS)).until(() -> { + await().atMost(Duration.of(60, ChronoUnit.SECONDS)).until(() -> { fetchFragment(currentPath); return !currentFragment.listObjectsOfProperty(createProperty(TREE + "relation")).hasNext(); }); diff --git a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerIntegrationTest.java b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerIntegrationTest.java index f31f99127..dc53e37f5 100644 --- a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerIntegrationTest.java +++ b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerIntegrationTest.java @@ -5,7 +5,7 @@ import be.vlaanderen.informatievlaanderen.ldes.server.ingest.repositories.MemberRepository; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.PageRelationPostgresRepository; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.PageEntityRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.RelationEntityRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.PageRelationEntityRepository; import io.cucumber.spring.CucumberContextConfiguration; import io.zonky.test.db.AutoConfigureEmbeddedDatabase; import jakarta.persistence.EntityManager; @@ -33,14 +33,12 @@ refresh = AutoConfigureEmbeddedDatabase.RefreshMode.AFTER_EACH_TEST_METHOD, replace = AutoConfigureEmbeddedDatabase.Replace.ANY) @ActiveProfiles("postgres-test") -@ContextConfiguration(classes = { MemberEntityRepository.class, PageRelationPostgresRepository.class, RelationEntityRepository.class }) +@ContextConfiguration(classes = { MemberEntityRepository.class, PageRelationPostgresRepository.class, PageRelationEntityRepository.class }) @ComponentScan(value = { "be.vlaanderen.informatievlaanderen.ldes.server" }) @TestPropertySource(properties = { "ldes-server.fragmentation-cron=*/1 * * * * *", "ldes-server.compaction-cron=*/10 * * * * *", "ldes-server.deletion-cron=*/20 * * * * *", "ldes-server.compaction-duration=PT1S" }) @SuppressWarnings("java:S2187") public class LdesServerIntegrationTest { - static final int FRAGMENTATION_POLLING_RATE = 1000; - @Autowired MockMvc mockMvc; @@ -56,7 +54,7 @@ public class LdesServerIntegrationTest { @Autowired MemberMetricsRepository memberMetricsRepository; @Autowired - RelationEntityRepository relationEntityRepository; + PageRelationEntityRepository pageRelationEntityRepository; @Autowired PageEntityRepository pageEntityRepository; diff --git a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerSteps.java b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerSteps.java index 71b55d027..bdc8c3969 100644 --- a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerSteps.java +++ b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerSteps.java @@ -135,7 +135,7 @@ private Model readModelFromFile(String fileName) throws URISyntaxException { @Then("I can fetch the TreeNode {string} and it contains {int} members") public void iCanFetchTheTreeNodeAndItContainsMembers(String url, int expectedNumberOfMembers) { await() - .atMost(FRAGMENTATION_POLLING_RATE, SECONDS) + .atMost(60, SECONDS) .untilAsserted(() -> { responseModel = fetchFragment(url); assertNotNull(responseModel); @@ -255,7 +255,7 @@ public void firstFragmentOfViewContainsMembers(String view, String collection, l .listObjectsOfProperty(createProperty("https://w3id.org/tree#node")).next().toString(); - await().atMost(FRAGMENTATION_POLLING_RATE, SECONDS) + await().atMost(60, SECONDS) .until(() -> { Model fragmentPage = fetchFragment(fragmentUrl); @@ -266,7 +266,7 @@ public void firstFragmentOfViewContainsMembers(String view, String collection, l @And("the LDES {string} contains {int} members") public void theLDESContainsMembers(String collection, int expectedMemberCount) { - await().atMost(FRAGMENTATION_POLLING_RATE, SECONDS) + await().atMost(60, SECONDS) .until(() -> jdbcTemplate .queryForObject("SELECT COUNT(*) FROM members m JOIN collections c ON m.collection_id = c.collection_id WHERE c.name = ?", Integer.class, collection) @@ -313,7 +313,7 @@ public void iIngestFilesOfStateObjectsFromFolderToTheCollection(int numberOfStat @When("I fetch a fragment from url {string} in a streaming way and is equal to the model of {string}") public void iFetchAStreamingFragment(String url, String compareUrl) { - await().atMost(FRAGMENTATION_POLLING_RATE, SECONDS) + await().atMost(60, SECONDS) .untilAsserted(() -> { FluxExchangeResult response = client.get() .uri(url) diff --git a/ldes-server-integration-test/src/test/resources/application-postgres-test.yml b/ldes-server-integration-test/src/test/resources/application-postgres-test.yml index 65d8727bd..b19995f7f 100644 --- a/ldes-server-integration-test/src/test/resources/application-postgres-test.yml +++ b/ldes-server-integration-test/src/test/resources/application-postgres-test.yml @@ -3,6 +3,7 @@ ldes-server: compaction-duration: "*/10 * * * * *" retention-cron: "*/10 * * * * *" deletion-cron: "*/5 * * * * *" + fragmentation-cron: "*/10 * * * * *" springdoc.swaggerui.path: "/swagger" management: diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PageRelationProcessor.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PageRelationProcessor.java deleted file mode 100644 index df0620701..000000000 --- a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PageRelationProcessor.java +++ /dev/null @@ -1,82 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.pagination.batch; - -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.entities.Page; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.entities.UnpagedMember; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRelationRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.PageAssignment; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.PartialUrl; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.stereotype.Component; - -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.List; -import java.util.Queue; - -@Component -public class PageRelationProcessor implements ItemProcessor, List> { - private final PageRepository pageRepository; - private final PageRelationRepository pageRelationRepository; - - public PageRelationProcessor(PageRepository pageRepository, PageRelationRepository pageRelationRepository) { - this.pageRepository = pageRepository; - this.pageRelationRepository = pageRelationRepository; - } - - @Override - public List process(List items) { - if (items.isEmpty()) { - return List.of(); - } - - Page pageToFill = pageRepository.getOpenPage(items.getFirst().bucketId()); - - Queue unpagedMembers = new ArrayDeque<>(items); - List pageAssignments = new ArrayList<>(); - - while (!unpagedMembers.isEmpty()) { - pageToFill = getPageWithSpace(pageToFill); - int membersAdded = fillPage(pageToFill, unpagedMembers.size()); - for (int i = 0; i < membersAdded; i++) { - UnpagedMember member = unpagedMembers.poll(); - assert member != null; - pageAssignments.add(new PageAssignment(pageToFill.getId(), pageToFill.getBucketId(), member.memberId())); - } - } - - return pageAssignments; - } - - private int fillPage(Page pageToFile, int numOfMembersToAdd) { - int membersAdded = Math.min(numOfMembersToAdd, pageToFile.getAvailableMemberSpace()); - pageToFile.incrementAssignedMemberCount(membersAdded); - return membersAdded; - } - - private Page getPageWithSpace(Page page) { - if (page.isNumberLess() || page.isFull()) { - return createNewPage(page); - } - return page; - } - - - private Page createNewPage(Page page) { - final PartialUrl childPagePartialUrl = page.createChildPartialUrl(); - final long childPageId = pageRepository.createPage(page.getBucketId(), childPagePartialUrl.asString()); - closeParentIfNecessary(page); - addRelation(page.getId(), childPageId); - return new Page(childPageId, page.getBucketId(), childPagePartialUrl, page.getPageSize()); - } - - private void closeParentIfNecessary(Page parentPage) { - if (!parentPage.isNumberLess()) { - pageRepository.setPageImmutable(parentPage.getId()); - } - } - - private void addRelation(long parentPageId, long childPageId) { - pageRelationRepository.insertGenericBucketRelation(parentPageId, childPageId); - } -} diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PaginationJobDefinitions.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PaginationJobDefinitions.java index b7691afaa..0e76009b7 100644 --- a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PaginationJobDefinitions.java +++ b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PaginationJobDefinitions.java @@ -1,24 +1,15 @@ package be.vlaanderen.informatievlaanderen.ldes.server.pagination.batch; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.entities.UnpagedMember; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.PageAssignment; import org.springframework.batch.core.Step; import org.springframework.batch.core.partition.support.Partitioner; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.batch.item.ItemReader; -import org.springframework.batch.item.ItemWriter; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.core.task.SimpleAsyncTaskExecutor; import org.springframework.core.task.TaskExecutor; import org.springframework.transaction.PlatformTransactionManager; -import org.springframework.transaction.TransactionException; - -import java.sql.SQLException; -import java.util.List; @Configuration public class PaginationJobDefinitions { @@ -27,22 +18,14 @@ public class PaginationJobDefinitions { @Bean public Step paginationStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, - Partitioner bucketPartitioner, ItemReader> pageItemReader, - ItemProcessor, List> pageRelationsProcessor, - ItemWriter> memberAssigner, + Partitioner bucketPartitioner, + Paginator paginator, PaginationMetricUpdater paginationMetricUpdater, @Qualifier("paginationTaskExecutor") TaskExecutor taskExecutor) { return new StepBuilder(PAGINATION_STEP, jobRepository) .partitioner("memberBucketPartitionStep", bucketPartitioner) .step(new StepBuilder("paginationStep", jobRepository) - ., List>chunk(1, transactionManager) - .reader(pageItemReader) - .processor(pageRelationsProcessor) - .writer(memberAssigner) - .faultTolerant() - .retryLimit(3) - .retry(SQLException.class) - .retry(TransactionException.class) + .tasklet(paginator, transactionManager) .build() ) .allowStartIfComplete(true) @@ -54,7 +37,7 @@ public Step paginationStep(JobRepository jobRepository, PlatformTransactionManag @Bean("paginationTaskExecutor") public TaskExecutor paginationTaskExecutor() { var taskExecutor = new SimpleAsyncTaskExecutor("spring_batch"); - taskExecutor.setConcurrencyLimit(5); + taskExecutor.setConcurrencyLimit(50); return taskExecutor; } } diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/Paginator.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/Paginator.java new file mode 100644 index 000000000..69a6664a0 --- /dev/null +++ b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/Paginator.java @@ -0,0 +1,65 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.batch; + +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.entities.Page; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageMemberRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRepository; +import org.springframework.batch.core.StepContribution; +import org.springframework.batch.core.scope.context.ChunkContext; +import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.item.ExecutionContext; +import org.springframework.batch.repeat.RepeatStatus; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Component +public class Paginator implements Tasklet { + private final PageMemberRepository pageMemberRepository; + private final PageRepository pageRepository; + + public Paginator(PageMemberRepository pageMemberRepository, PageRepository pageRepository) { + this.pageMemberRepository = pageMemberRepository; + this.pageRepository = pageRepository; + } + + @Override + public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) { + ExecutionContext context = chunkContext.getStepContext().getStepExecution().getExecutionContext(); + long bucketId = context.getLong("bucketId"); + + List members = pageMemberRepository.getUnpaginatedMembersForBucket(bucketId); + + if (members.isEmpty()) { + return RepeatStatus.FINISHED; + } + + Page openPage = pageRepository.getOpenPage(bucketId); + + if (openPage.isNumberLess()) { + openPage = pageRepository.createNextPage(openPage); + } + + int membersInPage; + + for (int i = 0; i < members.size(); i += membersInPage) { + List pageMembers = members.subList(i, Math.min(i + openPage.getAvailableMemberSpace(), members.size())); + membersInPage = pageMembers.size(); + + openPage = fillPageWithMembers(openPage, pageMembers); + } + + return RepeatStatus.FINISHED; + } + + private Page fillPageWithMembers(Page openPage, List pageMembers) { + openPage.incrementAssignedMemberCount(pageMembers.size()); + pageMemberRepository.assignMembersToPage(openPage, pageMembers); + + if (openPage.isFull()) { + return pageRepository.createNextPage(openPage); + } + else { + return openPage; + } + } +} diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageMemberRepository.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageMemberRepository.java new file mode 100644 index 000000000..37ab7695b --- /dev/null +++ b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageMemberRepository.java @@ -0,0 +1,23 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories; + +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.entities.Page; + +import java.util.List; + +public interface PageMemberRepository { + /** + * Retrieves a list of unpaginated member ids for a bucket + * + * @param bucketId Id of bucket + * @return List of unpaginated member ids for a bucket + */ + List getUnpaginatedMembersForBucket(long bucketId); + + /** + * Assigns Page Member to a Page + * + * @param openPage Open page that will be used to assign members to + * @param pageMembers A list of member ids that will be assigned to a page + */ + void assignMembersToPage(Page openPage, List pageMembers); +} diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageRepository.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageRepository.java index c18155afa..7f0ab6728 100644 --- a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageRepository.java +++ b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageRepository.java @@ -10,8 +10,7 @@ public interface PageRepository { Page getOpenPage(long bucketId); - int createPage(Long bucketId, String partialUrl); - void setPageImmutable(long pageId); + Page createNextPage(Page parentPage); void setChildrenImmutableByBucketId(long bucketId); void markAllPagesImmutableByCollectionName(String collectionName); Stream getPossibleCompactionCandidates(ViewName viewName, int capacityPerPage); diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/Bucket.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/Bucket.java deleted file mode 100644 index c6650fe8e..000000000 --- a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/Bucket.java +++ /dev/null @@ -1,4 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects; - -public record Bucket(long id, String descriptor) { -} diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/PageAssignment.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/PageAssignment.java deleted file mode 100644 index df69d3ad7..000000000 --- a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/PageAssignment.java +++ /dev/null @@ -1,4 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects; - -public record PageAssignment(long pageId, long bucketId, long memberId) { -} diff --git a/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PaginatorTest.java b/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PaginatorTest.java new file mode 100644 index 000000000..e07e51f0a --- /dev/null +++ b/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PaginatorTest.java @@ -0,0 +1,148 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.batch; + +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.entities.Page; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageMemberRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.PageNumber; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.PartialUrl; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.*; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.batch.core.StepExecution; +import org.springframework.batch.core.scope.context.ChunkContext; +import org.springframework.batch.core.scope.context.StepContext; +import org.springframework.batch.item.ExecutionContext; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class PaginatorTest { + private static final long BUCKET_ID = 1; + private static final String VIEW_NAME = "event-stream/paged"; + private static final int PAGE_SIZE = 2; + @Mock + private ChunkContext chunkContext; + @Mock + private PageMemberRepository pageMemberRepository; + @Mock + private PageRepository pageRepository; + @InjectMocks + Paginator paginator; + @Captor + ArgumentCaptor pageCaptor; + @Captor + ArgumentCaptor> listCaptor; + + @Test + void when_Paginating_withNoUnexpectedMembers_doNothing() { + mockBucketId(); + + when(pageMemberRepository.getUnpaginatedMembersForBucket(BUCKET_ID)).thenReturn(List.of()); + + paginator.execute(null, chunkContext); + + InOrder inOrder = inOrder(pageMemberRepository, pageRepository); + inOrder.verify(pageMemberRepository).getUnpaginatedMembersForBucket(BUCKET_ID); + inOrder.verifyNoMoreInteractions(); + } + + @Test + void when_Paginating_withNumberedOpenPage_expectCorrectPages() { + mockBucketId(); + Page numberedPage = new Page(1, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new PageNumber(1)), PAGE_SIZE); + + when(pageMemberRepository.getUnpaginatedMembersForBucket(BUCKET_ID)).thenReturn(List.of(1L, 2L, 3L, 4L, 5L)); + when(pageRepository.getOpenPage(BUCKET_ID)) + .thenReturn(numberedPage); + + when(pageRepository.createNextPage(any())) + .thenReturn(new Page(2, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new PageNumber(2)), PAGE_SIZE)) + .thenReturn(new Page(3, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new PageNumber(3)), PAGE_SIZE)); + + paginator.execute(null, chunkContext); + + InOrder inOrder = inOrder(pageMemberRepository, pageRepository); + inOrder.verify(pageMemberRepository).getUnpaginatedMembersForBucket(BUCKET_ID); + inOrder.verify(pageMemberRepository, times(3)).assignMembersToPage(pageCaptor.capture(), listCaptor.capture()); + inOrder.verifyNoMoreInteractions(); + + assertThat(pageCaptor.getAllValues().stream().map(Page::getId).toList()) + .containsExactlyInAnyOrder(1L, 2L, 3L); + assertThat(listCaptor.getAllValues().stream().map(List::size).toList()) + .containsExactlyInAnyOrder(2,2,1); + } + + @Test + void when_Paginating_withUnNumberedOpenPage_expectCorrectPages() { + mockBucketId(); + Page unNumberedPage = new Page(0, BUCKET_ID, new PartialUrl(VIEW_NAME, "", null), PAGE_SIZE); + Page numberedPage = new Page(1, BUCKET_ID, new PartialUrl(VIEW_NAME, "", null), PAGE_SIZE); + + when(pageMemberRepository.getUnpaginatedMembersForBucket(BUCKET_ID)).thenReturn(List.of(1L, 2L, 3L, 4L, 5L)); + when(pageRepository.getOpenPage(BUCKET_ID)) + .thenReturn(unNumberedPage); + + when(pageRepository.createNextPage(any())) + .thenReturn(numberedPage) + .thenReturn(new Page(2, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new PageNumber(2)), PAGE_SIZE)) + .thenReturn(new Page(3, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new PageNumber(3)), PAGE_SIZE)); + + paginator.execute(null, chunkContext); + + InOrder inOrder = inOrder(pageMemberRepository, pageRepository); + inOrder.verify(pageMemberRepository).getUnpaginatedMembersForBucket(BUCKET_ID); + inOrder.verify(pageMemberRepository, times(3)).assignMembersToPage(pageCaptor.capture(), listCaptor.capture()); + inOrder.verifyNoMoreInteractions(); + + assertThat(pageCaptor.getAllValues().stream().map(Page::getId).toList()) + .containsExactlyInAnyOrder(1L, 2L, 3L); + assertThat(listCaptor.getAllValues().stream().map(List::size).toList()) + .containsExactlyInAnyOrder(2,2,1); + } + + @Test + void when_Paginating_withSemiFilledNumberedOpenPage_expectCorrectPages() { + mockBucketId(); + Page numberedPage = new Page(1, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new PageNumber(1)), PAGE_SIZE,1); + + when(pageMemberRepository.getUnpaginatedMembersForBucket(BUCKET_ID)).thenReturn(List.of(1L, 2L, 3L, 4L, 5L)); + when(pageRepository.getOpenPage(BUCKET_ID)) + .thenReturn(numberedPage); + when(pageRepository.createNextPage(any())) + .thenReturn(new Page(2, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new PageNumber(2)), PAGE_SIZE)) + .thenReturn(new Page(3, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new PageNumber(3)), PAGE_SIZE)); + + paginator.execute(null, chunkContext); + + InOrder inOrder = inOrder(pageMemberRepository, pageRepository); + inOrder.verify(pageMemberRepository).getUnpaginatedMembersForBucket(BUCKET_ID); + inOrder.verify(pageMemberRepository).assignMembersToPage(pageCaptor.capture(), listCaptor.capture()); + inOrder.verify(pageRepository).createNextPage(any()); + inOrder.verify(pageMemberRepository).assignMembersToPage(pageCaptor.capture(), listCaptor.capture()); + inOrder.verify(pageRepository).createNextPage(any()); + inOrder.verify(pageMemberRepository).assignMembersToPage(pageCaptor.capture(), listCaptor.capture()); + inOrder.verify(pageRepository).createNextPage(any()); + inOrder.verifyNoMoreInteractions(); + + assertThat(pageCaptor.getAllValues().stream().map(Page::getId).toList()) + .containsExactlyInAnyOrder(1L, 2L, 3L); + assertThat(listCaptor.getAllValues().stream().map(List::size).toList()) + .containsExactlyInAnyOrder(1,2,2); + } + + private void mockBucketId() { + StepContext stepContext = mock(StepContext.class); + StepExecution stepExecution = mock(StepExecution.class); + ExecutionContext executionContext = mock(ExecutionContext.class); + + // Set up the mock behavior + when(chunkContext.getStepContext()).thenReturn(stepContext); + when(stepContext.getStepExecution()).thenReturn(stepExecution); + when(stepExecution.getExecutionContext()).thenReturn(executionContext); + when(executionContext.getLong("bucketId")).thenReturn(BUCKET_ID); + } +} From 00ad636d3ab6d88913459fa8ea968608f5aa17bf Mon Sep 17 00:00:00 2001 From: Jonas Bulcke <127748878+jobulcke@users.noreply.github.com> Date: Mon, 23 Sep 2024 15:36:54 +0200 Subject: [PATCH 9/9] feat: move all bucket db queries to ItemWriter (#1370) * chore: add view-id to page-members chore: fix views and queries to use view-id instead of bucket-id * chore: add and use last page view * chore: optimize member reader query chore: fix get open page query * chore: improve member query * chore: improve statistics queries * chore: improve unprocessed_views perfo * chore: add view-id to page-members chore: fix views and queries to use view-id instead of bucket-id * chore: add and use last page view * chore: optimize member reader query chore: fix get open page query * chore: improve member query * chore: improve statistics queries * chore: improve unprocessed_views perfo * fix: statistics on_page_member_inserted & on_page_member_updating * chore: add collection stats * chore: revert add collection stats * feat: modified bucket java entity and some tweaks in some strategies to use this modified entity * chore: add todos * feat: strategies updated to use new bucket and method related to this and tests added for GeospatialStrategy * fix: duplicate test * feat: strategies updated and tested * chore: cleanup (that should have been happened a few months ago) * chore: cleanup (that should have been happened a few months ago) * feat: ItemProcessor and ItemWriters modified * feat: bucket creators updated * feat: cleaner bucket writer * fix: missing cascade operation * feat: all writers finished * feat: better root bucket looping and some small cleanup * feat: tests fixed * fix: merge conflicts * fix: invalid parallel actions * fix: broken tests * fix: broken tests * feat: sonar - part 1 * feat: pr remarks - part 1 * feat: PR remarks - part 2 * feat: coverage increase * fix: broken test * fix: broken test * feat: unit tests for the writers * fix: broken test --------- Co-authored-by: Ranko Orlic --- .../ldes-fragmentisers-common/pom.xml | 6 - .../fragmentation/FragmentationStrategy.java | 5 +- .../FragmentationStrategyBatchCollection.java | 7 +- .../FragmentationStrategyBatchExecutor.java | 17 +- .../FragmentationStrategyDecorator.java | 22 +-- .../FragmentationStrategyImpl.java | 7 +- .../fragmentation/RootBucketRetriever.java | 36 ---- .../batch/BucketJobDefinitions.java | 26 +-- .../fragmentation/batch/BucketProcessors.java | 6 +- .../server/fragmentation/entities/Bucket.java | 80 +++++++-- .../fragmentation/entities/ChildBucket.java | 26 +++ .../fragmentation/entities/Fragment.java | 164 ----------------- .../factory/RootFragmentCreator.java | 10 -- .../repository/BucketRepository.java | 4 - .../BucketisedMemberRepository.java | 8 - .../services/MemberRetriever.java | 10 -- .../valueobjects/BucketRelation.java | 17 +- .../BucketRelationCreatedEvent.java | 4 - .../valueobjects/TreeRelation.java | 11 ++ .../src/main/java/module-info.java | 3 - .../FragmentationServiceTest.java | 7 +- ...gmentationStrategyBatchCollectionTest.java | 7 +- ...ragmentationStrategyBatchExecutorTest.java | 19 +- .../FragmentationStrategyCreatorImplTest.java | 6 +- .../FragmentationStrategyDecoratorTest.java | 20 +-- .../FragmentationStrategyImplTest.java | 14 +- .../RootBucketRetrieverTest.java | 69 ------- .../fragmentation/entities/BucketTest.java | 50 ++++++ .../entities/FragmentSequenceTest.java | 19 -- .../fragmentation/entities/FragmentTest.java | 170 ------------------ .../GeospatialFragmentationStrategy.java | 44 ++--- ...eospatialFragmentationStrategyWrapper.java | 8 +- .../bucketising/GeospatialBucketiser.java | 4 +- .../GeospatialRelationsAttributer.java | 53 ------ .../TileBucketRelationsAttributer.java | 41 +++-- .../exceptions/RdfGeometryException.java | 19 -- .../fragments/GeospatialBucketCreator.java | 30 +--- .../GeospatialFragmentationStrategyTest.java | 67 +++---- ...atialFragmentationStrategyWrapperTest.java | 6 +- .../bucketising/GeospatialBucketiserTest.java | 10 +- .../GeospatialRelationsAttributerTest.java | 35 ---- .../TileBucketRelationsAttributerTest.java | 31 ++-- .../GeospatialBucketCreatorTest.java | 115 ++++-------- .../ReferenceFragmentationStrategy.java | 93 +++++----- ...ReferenceFragmentationStrategyWrapper.java | 10 +- .../bucketising/ReferenceBucketiser.java | 2 +- .../fragmentation/ReferenceBucketCreator.java | 36 +--- .../ReferenceFragmentRelationsAttributer.java | 23 +-- .../ReferenceFragmentationStrategyTest.java | 67 ++----- .../bucketising/ReferenceBucketiserTest.java | 8 +- .../ReferenceBucketCreatorTest.java | 75 +------- ...erenceFragmentRelationsAttributerTest.java | 39 ++-- ...rchicalTimeBasedFragmentationStrategy.java | 15 +- ...TimeBasedFragmentationStrategyWrapper.java | 13 +- .../config/TimeBasedConfig.java | 10 +- .../services/TimeBasedBucketCreator.java | 34 ++-- .../services/TimeBasedBucketFinder.java | 7 +- .../TimeBasedRelationsAttributer.java | 77 +++----- ...calTimeBasedFragmentationStrategyTest.java | 61 ++----- .../services/TimeBasedBucketCreatorTest.java | 69 ++----- .../services/TimeBasedBucketFinderTest.java | 10 +- .../TimeBasedRelationsAttributerTest.java | 69 ++----- .../FragmentationConfigCompaction.java | 3 +- .../compaction/FragmentsCompactedEvent.java | 8 - .../BulkFragmentDeletedEvent.java | 8 - .../BulkMemberAllocatedEvent.java | 6 - .../fragmentation/MemberAllocatedEvent.java | 5 - .../TimeBasedLinearCachingTriggered.java | 6 - .../ViewNeedsRebucketisationEvent.java | 12 -- .../events/ingest/MembersIngestedEvent.java | 9 - .../exceptions/DataConversionException.java | 32 ---- .../server/domain/model/FragmentPair.java | 4 - .../server/domain/model/FragmentSequence.java | 13 -- .../src/main/java/module-info.java | 3 - .../postgres/BucketPostgresRepository.java | 28 +-- .../batch/BucketisationItemWriter.java | 56 ++++++ .../batch/BucketisedMemberWriter.java | 54 ------ .../postgres/batch/chunk/ChunkCollector.java | 40 +++++ .../batch/delegates/BucketItemWriter.java | 49 +++++ .../delegates/BucketPageItemWriterConfig.java | 34 ++++ .../BucketisedMemberItemWriterConfig.java | 32 ++++ .../PageRelationItemWriterConfig.java | 37 ++++ .../postgres/entity/MemberBucketEntity.java | 54 ------ .../postgres/entity/TreeRelationEntity.java | 77 -------- .../postgres/mapper/BucketMapper.java | 10 -- .../BucketPostgresRepositoryTest.java | 11 +- .../PostgresBucketisationIntegrationTest.java | 12 +- .../batch/BucketisationItemWriterTest.java | 124 +++++++++++++ .../batch/delegates/BucketItemWriterTest.java | 36 ++++ .../delegates/BucketPageItemWriterTest.java | 38 ++++ .../BucketisedMemberWriterTest.java | 25 ++- .../delegates/PageRelationItemWriterTest.java | 41 +++++ .../batch/delegates/TestBucketSupplier.java | 54 ++++++ .../delegates/init-collection-and-view.sql | 14 ++ .../{ => delegates}/init-writer-test.sql | 19 +- .../postgres/batch/delegates/insert-pages.sql | 7 + .../postgres/init-bucketReader.sql | 3 +- .../postgres/batch/MemberItemReader.java | 28 +-- .../3_4_0/add-view-id-to-page-members.xml | 23 +++ .../db/changelog/3_4_0/add_view_stats.sql | 65 +++++++ .../3_4_0/define-bucket-last-page-view.sql | 5 + .../3_4_0/drop-on-bucket-insert-trigger.sql | 2 + .../db/changelog/3_4_0/initialize-view-id.sql | 3 + .../resources/db/changelog/3_4_0/master.xml | 14 ++ .../changelog/3_4_0/redefine-bucket-stats.sql | 6 + .../3_4_0/redefine-needs-bucketization.sql | 5 + .../3_4_0/redefine-needs-pagination.sql | 6 + .../main/resources/db/changelog/master.xml | 1 + .../postgres/PagePostgresRepository.java | 23 +-- .../PageRelationPostgresRepository.java | 14 -- .../postgres/TreeNodePostgresRepository.java | 2 +- .../postgres/batch/BucketPartitioner.java | 3 +- .../postgres/mapper/TreeRelationMapper.java | 15 +- .../repository/PageEntityRepository.java | 12 -- .../PageMemberEntityRepository.java | 26 ++- .../pagination/postgres/PaginationSteps.java | 10 +- .../pagination/postgres/init-paged-test.sql | 3 - .../ldes/server/CompactionServiceSteps.java | 22 ++- .../MemberCounter.java | 9 +- .../features/compaction/compaction.feature | 2 +- .../batch/PaginationJobDefinitions.java | 4 +- .../repositories/PageRelationRepository.java | 3 - .../repositories/PageRepository.java | 1 - .../PaginationSequenceRepository.java | 16 -- .../BucketRelationsEventListener.java | 20 --- .../services/LinearCachingEventsListener.java | 20 --- .../rest/caching/EtagCachingStrategy.java | 4 +- .../rest/treenode/TreeNodeController.java | 6 +- .../services/TreeRelationResponse.java | 6 - .../rest/caching/EtagCachingStrategyTest.java | 4 +- .../rest/treenode/TreeNodeControllerTest.java | 3 + .../services/TreeNodeConverterImplTest.java | 2 + .../server/fetching/entities/TreeNode.java | 2 +- .../LdesFragmentIdentifierParseException.java | 2 +- .../repository/TreeNodeRepository.java | 2 +- .../services/StreamingTreeNodeFactory.java | 2 +- .../StreamingTreeNodeFactoryImpl.java | 2 +- .../fetching/services/TreeNodeFetcher.java | 2 +- .../services/TreeNodeFetcherImpl.java | 4 +- .../fetching/valueobjects/FragmentPair.java | 4 + .../valueobjects}/LdesFragmentIdentifier.java | 37 +--- .../valueobjects}/LdesFragmentRequest.java | 8 +- .../fetching/valueobjects}/TreeRelation.java | 2 +- .../src/main/java/module-info.java | 3 +- .../StreamingTreeNodeFactoryImplTest.java | 8 +- .../services/TreeNodeFetcherImplTest.java | 24 ++- .../LdesFragmentIdentifierTest.java | 20 +-- .../valueobjects/LdesFragmentRequestTest.java | 17 +- .../valueobjects/TreeRelationTest.java | 9 +- .../server/ingest/MemberIngesterImpl.java | 17 +- .../server/ingest/MemberIngesterImplTest.java | 12 +- 151 files changed, 1442 insertions(+), 2118 deletions(-) delete mode 100644 ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/RootBucketRetriever.java create mode 100644 ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/ChildBucket.java delete mode 100644 ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/Fragment.java delete mode 100644 ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/factory/RootFragmentCreator.java delete mode 100644 ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/BucketisedMemberRepository.java delete mode 100644 ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/services/MemberRetriever.java delete mode 100644 ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/valueobjects/BucketRelationCreatedEvent.java create mode 100644 ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/valueobjects/TreeRelation.java delete mode 100644 ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/RootBucketRetrieverTest.java delete mode 100644 ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/FragmentSequenceTest.java delete mode 100644 ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/FragmentTest.java delete mode 100644 ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/GeospatialRelationsAttributer.java delete mode 100644 ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/exceptions/RdfGeometryException.java delete mode 100644 ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/GeospatialRelationsAttributerTest.java delete mode 100644 ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/compaction/FragmentsCompactedEvent.java delete mode 100644 ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/BulkFragmentDeletedEvent.java delete mode 100644 ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/BulkMemberAllocatedEvent.java delete mode 100644 ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/MemberAllocatedEvent.java delete mode 100644 ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/TimeBasedLinearCachingTriggered.java delete mode 100644 ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/ViewNeedsRebucketisationEvent.java delete mode 100644 ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/ingest/MembersIngestedEvent.java delete mode 100644 ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/exceptions/DataConversionException.java delete mode 100644 ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/FragmentPair.java delete mode 100644 ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/FragmentSequence.java create mode 100644 ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/BucketisationItemWriter.java delete mode 100644 ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/BucketisedMemberWriter.java create mode 100644 ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/chunk/ChunkCollector.java create mode 100644 ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketItemWriter.java create mode 100644 ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketPageItemWriterConfig.java create mode 100644 ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketisedMemberItemWriterConfig.java create mode 100644 ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/PageRelationItemWriterConfig.java delete mode 100644 ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/entity/MemberBucketEntity.java delete mode 100644 ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/entity/TreeRelationEntity.java create mode 100644 ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/BucketisationItemWriterTest.java create mode 100644 ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketItemWriterTest.java create mode 100644 ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketPageItemWriterTest.java rename ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/{ => delegates}/BucketisedMemberWriterTest.java (65%) create mode 100644 ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/PageRelationItemWriterTest.java create mode 100644 ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/TestBucketSupplier.java create mode 100644 ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/init-collection-and-view.sql rename ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/{ => delegates}/init-writer-test.sql (99%) create mode 100644 ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/insert-pages.sql create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/add-view-id-to-page-members.xml create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/add_view_stats.sql create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/define-bucket-last-page-view.sql create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/drop-on-bucket-insert-trigger.sql create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/initialize-view-id.sql create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/master.xml create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/redefine-bucket-stats.sql create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/redefine-needs-bucketization.sql create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/redefine-needs-pagination.sql delete mode 100644 ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PaginationSequenceRepository.java delete mode 100644 ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/BucketRelationsEventListener.java delete mode 100644 ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/LinearCachingEventsListener.java rename {ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain => ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching}/exceptions/LdesFragmentIdentifierParseException.java (80%) create mode 100644 ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/FragmentPair.java rename {ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model => ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects}/LdesFragmentIdentifier.java (73%) rename {ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model => ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects}/LdesFragmentRequest.java (73%) rename {ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model => ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects}/TreeRelation.java (90%) rename {ldes-server-domain/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/ldesfragment => ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching}/valueobjects/LdesFragmentIdentifierTest.java (81%) rename {ldes-server-domain/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/ldesfragmentrequest => ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching}/valueobjects/LdesFragmentRequestTest.java (75%) rename {ldes-server-domain/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/ldesfragment => ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching}/valueobjects/TreeRelationTest.java (86%) diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/pom.xml b/ldes-fragmentisers/ldes-fragmentisers-common/pom.xml index 4c81c8bf2..496386d18 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/pom.xml +++ b/ldes-fragmentisers/ldes-fragmentisers-common/pom.xml @@ -19,12 +19,6 @@ ${project.version} - - be.vlaanderen.informatievlaanderen.vsds - ldes-server-retention - ${project.version} - - org.springframework.boot diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategy.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategy.java index e1c5199d3..9793c7de7 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategy.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategy.java @@ -1,12 +1,9 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.FragmentationMember; import io.micrometer.observation.Observation; -import java.util.List; - public interface FragmentationStrategy { - List addMemberToBucket(Bucket rootFragmentOfView, FragmentationMember member, Observation parentObservation); + void addMemberToBucket(Bucket rootBucketOfView, FragmentationMember member, Observation parentObservation); } \ No newline at end of file diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchCollection.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchCollection.java index cbc68266a..d589f75a4 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchCollection.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchCollection.java @@ -4,7 +4,6 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewSpecification; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.factory.FragmentationStrategyCreator; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketRepository; import io.micrometer.observation.ObservationRegistry; import org.springframework.context.event.EventListener; import org.springframework.core.annotation.Order; @@ -16,16 +15,13 @@ @Component public class FragmentationStrategyBatchCollection implements FragmentationStrategyCollection { - private final BucketRepository bucketRepository; private final Set fragmentationStrategySet; private final FragmentationStrategyCreator fragmentationStrategyCreator; private final ObservationRegistry observationRegistry; public FragmentationStrategyBatchCollection( - BucketRepository bucketRepository, FragmentationStrategyCreator fragmentationStrategyCreator, ObservationRegistry observationRegistry) { - this.bucketRepository = bucketRepository; this.fragmentationStrategyCreator = fragmentationStrategyCreator; this.observationRegistry = observationRegistry; this.fragmentationStrategySet = new HashSet<>(); @@ -78,7 +74,6 @@ private void removeFromStrategySet(Predicate private FragmentationStrategyBatchExecutor createExecutor(ViewName viewName, ViewSpecification viewSpecification) { final FragmentationStrategy fragmentationStrategy = fragmentationStrategyCreator .createFragmentationStrategyForView(viewSpecification); - final var rootBucketRetriever = new RootBucketRetriever(viewName, bucketRepository, observationRegistry); - return new FragmentationStrategyBatchExecutor(viewName, fragmentationStrategy, rootBucketRetriever, observationRegistry); + return new FragmentationStrategyBatchExecutor(viewName, fragmentationStrategy, observationRegistry); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchExecutor.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchExecutor.java index b1bdc4c39..73c64efe0 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchExecutor.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchExecutor.java @@ -1,11 +1,11 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.FragmentationMember; +import io.micrometer.observation.Observation; import io.micrometer.observation.ObservationRegistry; -import java.util.List; import java.util.Objects; import static io.micrometer.observation.Observation.createNotStarted; @@ -14,26 +14,23 @@ public class FragmentationStrategyBatchExecutor { private final FragmentationStrategy fragmentationStrategy; private final ViewName viewName; - private final RootBucketRetriever rootBucketRetriever; private final ObservationRegistry observationRegistry; @SuppressWarnings("java:S107") public FragmentationStrategyBatchExecutor(ViewName viewName, FragmentationStrategy fragmentationStrategy, - RootBucketRetriever rootBucketRetriever, ObservationRegistry observationRegistry) { - this.rootBucketRetriever = rootBucketRetriever; this.observationRegistry = observationRegistry; this.fragmentationStrategy = fragmentationStrategy; this.viewName = viewName; } - public List bucketise(FragmentationMember member) { - var parentObservation = createNotStarted("execute fragmentation", observationRegistry).start(); - final var rootBucket = rootBucketRetriever.retrieveRootBucket(parentObservation); - List bucketisedMembers = fragmentationStrategy.addMemberToBucket(rootBucket, member, parentObservation); + public Bucket bucketise(FragmentationMember member) { + final Observation parentObservation = createNotStarted("execute fragmentation", observationRegistry).start(); + final Bucket rootBucket = Bucket.createRootBucketForView(viewName); + fragmentationStrategy.addMemberToBucket(rootBucket, member, parentObservation); parentObservation.stop(); - return bucketisedMembers; + return rootBucket; } public boolean isPartOfCollection(String collectionName) { diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyDecorator.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyDecorator.java index ce58eb633..1c2d7a56e 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyDecorator.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyDecorator.java @@ -1,35 +1,19 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.FragmentationMember; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelationCreatedEvent; import io.micrometer.observation.Observation; -import org.springframework.context.ApplicationEventPublisher; - -import java.util.List; public abstract class FragmentationStrategyDecorator implements FragmentationStrategy { - private final FragmentationStrategy fragmentationStrategy; - private final ApplicationEventPublisher applicationEventPublisher; - - protected FragmentationStrategyDecorator(FragmentationStrategy fragmentationStrategy, - ApplicationEventPublisher applicationEventPublisher) { + protected FragmentationStrategyDecorator(FragmentationStrategy fragmentationStrategy) { this.fragmentationStrategy = fragmentationStrategy; - this.applicationEventPublisher = applicationEventPublisher; } @Override - public List addMemberToBucket(Bucket rootFragmentOfView, FragmentationMember member, Observation parentObservation) { - return fragmentationStrategy.addMemberToBucket(rootFragmentOfView, member, parentObservation); - } - - protected void addRelationFromParentToChild(Bucket parentBucket, Bucket childBucket) { - BucketRelation bucketRelation = BucketRelation.createGenericRelation(parentBucket, childBucket); - applicationEventPublisher.publishEvent(new BucketRelationCreatedEvent(bucketRelation)); + public void addMemberToBucket(Bucket parentBucket, FragmentationMember member, Observation parentObservation) { + fragmentationStrategy.addMemberToBucket(parentBucket, member, parentObservation); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyImpl.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyImpl.java index e854d7533..d3be5d5dc 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyImpl.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyImpl.java @@ -1,15 +1,12 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.FragmentationMember; import io.micrometer.observation.Observation; -import java.util.List; - public class FragmentationStrategyImpl implements FragmentationStrategy { @Override - public List addMemberToBucket(Bucket bucket, FragmentationMember member, Observation parentObservation) { - return List.of(new BucketisedMember(bucket.getBucketId(), member.getMemberId())); + public void addMemberToBucket(Bucket rootBucketOfView, FragmentationMember member, Observation parentObservation) { + rootBucketOfView.assignMember(member.getMemberId()); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/RootBucketRetriever.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/RootBucketRetriever.java deleted file mode 100644 index f0d738662..000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/RootBucketRetriever.java +++ /dev/null @@ -1,36 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.exceptions.MissingRootFragmentException; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketRepository; -import io.micrometer.observation.Observation; -import io.micrometer.observation.ObservationRegistry; - -public class RootBucketRetriever { - private final ViewName viewName; - private final BucketRepository bucketRepository; - private final ObservationRegistry observationRegistry; - private Bucket rootBucket; - - public RootBucketRetriever(ViewName viewName, BucketRepository bucketRepository, ObservationRegistry observationRegistry) { - this.viewName = viewName; - this.bucketRepository = bucketRepository; - this.observationRegistry = observationRegistry; - } - - public Bucket retrieveRootBucket(Observation parentObservation) { - final Observation rootRetrievalObservation = Observation - .createNotStarted("retrieve root of view %s".formatted(viewName.asString()), observationRegistry) - .parentObservation(parentObservation) - .start(); - - if(rootBucket == null) { - rootBucket = bucketRepository.retrieveRootBucket(viewName) - .orElseThrow(() -> new MissingRootFragmentException(viewName.asString())); - } - - rootRetrievalObservation.stop(); - return rootBucket; - } -} diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/batch/BucketJobDefinitions.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/batch/BucketJobDefinitions.java index 6e90ae92e..d3f706f92 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/batch/BucketJobDefinitions.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/batch/BucketJobDefinitions.java @@ -1,6 +1,6 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.batch; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.FragmentationMember; import org.springframework.batch.core.Step; import org.springframework.batch.core.repository.JobRepository; @@ -8,15 +8,10 @@ import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemReader; import org.springframework.batch.item.ItemWriter; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.core.task.SimpleAsyncTaskExecutor; -import org.springframework.core.task.TaskExecutor; import org.springframework.transaction.PlatformTransactionManager; -import java.util.List; - @Configuration public class BucketJobDefinitions { public static final String BUCKETISATION_STEP = "bucketisation"; @@ -26,23 +21,16 @@ public class BucketJobDefinitions { public Step bucketiseMembersStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, ItemReader memberReader, - ItemProcessor> viewBucketProcessor, - ItemWriter> writer, - BucketMetricUpdater bucketMetricUpdater, - @Qualifier("bucketTaskExecutor") TaskExecutor taskExecutor) { + ItemProcessor bucketProcessor, + ItemWriter bucketisationItemWriter, + BucketMetricUpdater bucketMetricUpdater) { return new StepBuilder(BUCKETISATION_STEP, jobRepository) - .>chunk(CHUNK_SIZE, transactionManager) + .chunk(CHUNK_SIZE, transactionManager) .reader(memberReader) - .processor(viewBucketProcessor) - .writer(writer) + .processor(bucketProcessor) + .writer(bucketisationItemWriter) .listener(bucketMetricUpdater) .build(); } - @Bean("bucketTaskExecutor") - public TaskExecutor paginationTaskExecutor() { - var taskExecutor = new SimpleAsyncTaskExecutor("spring_batch"); - taskExecutor.setConcurrencyLimit(1); - return taskExecutor; - } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/batch/BucketProcessors.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/batch/BucketProcessors.java index 4b8601272..ee7a5b982 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/batch/BucketProcessors.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/batch/BucketProcessors.java @@ -2,7 +2,7 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategyCollection; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.FragmentationMember; import org.springframework.batch.core.configuration.annotation.StepScope; import org.springframework.batch.item.ItemProcessor; @@ -10,13 +10,11 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import java.util.List; - @Configuration public class BucketProcessors { @Bean @StepScope - public ItemProcessor> viewBucketProcessor( + public ItemProcessor bucketProcessor( FragmentationStrategyCollection fragmentationStrategyCollection, @Value("#{jobParameters['collectionName']}") String collectionName, @Value("#{jobParameters['viewName']}") String viewName diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/Bucket.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/Bucket.java index 9b0aafdc8..9bdabe089 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/Bucket.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/Bucket.java @@ -5,25 +5,29 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.exceptions.DuplicateFragmentPairException; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.TreeRelation; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.Optional; +import java.util.*; +import java.util.stream.Stream; public class Bucket { - private final long bucketId; + private long bucketId; private final BucketDescriptor bucketDescriptor; private final ViewName viewName; + private final List children; + private long assignedMemberId; - public Bucket(long bucketId, BucketDescriptor bucketDescriptor, ViewName viewName) { + public Bucket(long bucketId, BucketDescriptor bucketDescriptor, ViewName viewName, List children, long assignedMemberId) { this.bucketId = bucketId; this.bucketDescriptor = bucketDescriptor; this.viewName = viewName; + this.children = new ArrayList<>(children); + this.assignedMemberId = assignedMemberId; } public Bucket(BucketDescriptor bucketDescriptor, ViewName viewName) { - this(0, bucketDescriptor, viewName); + this(0, bucketDescriptor, viewName, new ArrayList<>(), 0); } public static Bucket createRootBucketForView(ViewName viewName) { @@ -34,12 +38,12 @@ public long getBucketId() { return bucketId; } - public ViewName getViewName() { - return viewName; + public void setBucketId(long bucketId) { + this.bucketId = bucketId; } - public List getBucketDescriptorPairs() { - return bucketDescriptor.getDescriptorPairs(); + public ViewName getViewName() { + return viewName; } public BucketDescriptor getBucketDescriptor() { @@ -50,6 +54,26 @@ public String getBucketDescriptorAsString() { return bucketDescriptor.asDecodedString(); } + public ChildBucket addChildBucket(ChildBucket childBucket) { + final int index = children.indexOf(childBucket); + if (index == -1) { + children.add(childBucket); + return childBucket; + } + ChildBucket existingChildBucket = children.get(index); + existingChildBucket.addRelations(childBucket.getRelations()); + return existingChildBucket; + } + + + public ChildBucket withRelations(TreeRelation... relationDefinition) { + return new ChildBucket(bucketId, bucketDescriptor, viewName, children, assignedMemberId, Set.of(relationDefinition)); + } + + public ChildBucket withGenericRelation() { + return withRelations(TreeRelation.generic()); + } + public Bucket createChild(BucketDescriptorPair descriptorPair) { return new Bucket(createChildDescriptor(descriptorPair), viewName); } @@ -63,6 +87,40 @@ public BucketDescriptor createChildDescriptor(BucketDescriptorPair descriptorPai return new BucketDescriptor(childFragmentPairs); } + public void assignMember(long memberId) { + assignedMemberId = memberId; + } + + public Optional getMember() { + return Optional.of(assignedMemberId) + .filter(member -> member != 0) + .map(member -> new BucketisedMember(bucketId, member)); + } + + public List getChildren() { + return List.copyOf(children); + } + + public List getAllDescendants() { + return children.stream() + .flatMap(child -> Stream.concat(Stream.of(child), child.getAllDescendants().stream())) + .toList(); + } + + public List getBucketTree() { + final List bucketTree = new ArrayList<>(List.of(this)); + getAllDescendants().stream().distinct().forEach(bucketTree::add); + return bucketTree; + } + + public List getChildRelations() { + return children.stream() + .flatMap(child -> child.getRelations().stream() + .map(relation -> new BucketRelation(createPartialUrl(), child.createPartialUrl(), relation)) + ) + .toList(); + } + public String createPartialUrl() { return "/" + viewName.asString() + (bucketDescriptor.isEmpty() ? "" : "?" + bucketDescriptor.asDecodedString()); } diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/ChildBucket.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/ChildBucket.java new file mode 100644 index 000000000..6b75a6016 --- /dev/null +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/ChildBucket.java @@ -0,0 +1,26 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.TreeRelation; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class ChildBucket extends Bucket { + private final Set relations; + + public ChildBucket(long bucketId, BucketDescriptor bucketDescriptor, ViewName viewName, List children, long assignedMemberId, Set relations) { + super(bucketId, bucketDescriptor, viewName, children, assignedMemberId); + this.relations = new HashSet<>(relations); + } + + public Set getRelations() { + return relations; + } + + public void addRelations(Set relations) { + this.relations.addAll(relations); + } +} diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/Fragment.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/Fragment.java deleted file mode 100644 index 9aa76a992..000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/Fragment.java +++ /dev/null @@ -1,164 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentPair; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.TreeRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.exceptions.DuplicateFragmentPairException; -import org.jetbrains.annotations.Nullable; - -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -public class Fragment { - - public static final String ROOT = "root"; - private final LdesFragmentIdentifier identifier; - private Boolean immutable; - private int nrOfMembersAdded; - private final List relations; - private LocalDateTime deleteTime; - private LocalDateTime nextUpdateTs = null; - - public Fragment(LdesFragmentIdentifier identifier) { - this(identifier, false, 0, new ArrayList<>(), null); - } - - public Fragment(LdesFragmentIdentifier identifier, Boolean immutable, int nrOfMembersAdded, - List relations, LocalDateTime deleteTime) { - this.identifier = identifier; - this.immutable = immutable; - this.nrOfMembersAdded = nrOfMembersAdded; - this.relations = relations; - this.deleteTime = deleteTime; - } - - public LdesFragmentIdentifier getFragmentId() { - return identifier; - } - - public String getFragmentIdString() { - return identifier.asDecodedFragmentId(); - } - - public List getFragmentPairs() { - return this.identifier.getFragmentPairs(); - } - - public void makeImmutable() { - this.immutable = true; - } - - public boolean isImmutable() { - return this.immutable; - } - - public Fragment createChild(FragmentPair fragmentPair) { - List childFragmentPairs = new ArrayList<>(this.identifier.getFragmentPairs().stream().toList()); - if (hasChildWithSameFragmentKey(fragmentPair, childFragmentPairs)) { - throw new DuplicateFragmentPairException(identifier.asDecodedFragmentId(), fragmentPair.fragmentKey()); - } - childFragmentPairs.add(fragmentPair); - return new Fragment(new LdesFragmentIdentifier(getViewName(), childFragmentPairs)); - } - - private static boolean hasChildWithSameFragmentKey(FragmentPair fragmentPair, List childFragmentPairs) { - return childFragmentPairs - .stream() - .map(FragmentPair::fragmentKey) - .anyMatch(key -> key.equals(fragmentPair.fragmentKey())); - } - - public Optional getValueOfKey(String key) { - return this.identifier.getValueOfFragmentPairKey(key); - } - - public ViewName getViewName() { - return this.identifier.getViewName(); - } - - public int getNrOfMembersAdded() { - return this.nrOfMembersAdded; - } - - public int incrementNrOfMembersAdded() { - return this.nrOfMembersAdded++; - } - - public Optional getParentId() { - return identifier.getParentId(); - } - - public String getParentIdAsString() { - return identifier.getParentId().map(LdesFragmentIdentifier::asDecodedFragmentId).orElse(ROOT); - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - Fragment that = (Fragment) o; - return Objects.equals(getFragmentId(), that.getFragmentId()); - } - - @Override - public int hashCode() { - return Objects.hash(getFragmentId()); - } - - public void addRelation(TreeRelation relation) { - relations.add(relation); - } - - public boolean containsRelation(TreeRelation parentChildRelation) { - return relations.contains(parentChildRelation); - } - - public List getRelations() { - return relations; - } - - public boolean isRoot() { - return this.identifier.getFragmentPairs().isEmpty(); - } - - public LocalDateTime getDeleteTime() { - return deleteTime; - } - - public void removeRelationToIdentifier(LdesFragmentIdentifier fragmentIdentifier) { - relations.removeIf(treeRelation -> treeRelation.treeNode().equals(fragmentIdentifier)); - } - - public boolean isConnectedTo(Fragment otherFragment) { - return getRelations() - .stream() - .anyMatch(treeRelation -> treeRelation.treeNode() - .equals(otherFragment.getFragmentId())); - } - - @Nullable - public LocalDateTime getNextUpdateTs() { - return nextUpdateTs; - } - - public void setNextUpdateTs(@Nullable LocalDateTime nextUpdateTs) { - this.nextUpdateTs = nextUpdateTs; - } - - @Override - public String toString() { - return "Fragment{" + - "identifier=" + identifier + - ", immutable=" + immutable + - ", nrOfMembersAdded=" + nrOfMembersAdded + - ", relations=" + relations + - ", deleteTime=" + deleteTime + - '}'; - } -} diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/factory/RootFragmentCreator.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/factory/RootFragmentCreator.java deleted file mode 100644 index f9f394780..000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/factory/RootFragmentCreator.java +++ /dev/null @@ -1,10 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.factory; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Fragment; - -public interface RootFragmentCreator { - - Fragment createRootFragmentForView(ViewName viewName); - -} diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/BucketRepository.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/BucketRepository.java index 8fa397800..b2adaec89 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/BucketRepository.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/BucketRepository.java @@ -2,14 +2,10 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; import java.util.Optional; public interface BucketRepository { - Optional retrieveBucket(ViewName viewName, BucketDescriptor bucketDescriptor); - Bucket insertBucket(Bucket bucket); - Optional retrieveRootBucket(ViewName viewName); } diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/BucketisedMemberRepository.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/BucketisedMemberRepository.java deleted file mode 100644 index eaf60df27..000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/BucketisedMemberRepository.java +++ /dev/null @@ -1,8 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; - -public interface BucketisedMemberRepository { - void deleteByViewName(ViewName viewName); - void deleteByCollection(String collectionName); -} diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/services/MemberRetriever.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/services/MemberRetriever.java deleted file mode 100644 index d3bd28e84..000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/services/MemberRetriever.java +++ /dev/null @@ -1,10 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.services; - -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.FragmentationMember; - -import java.util.Optional; - -public interface MemberRetriever { - - Optional findFirstByCollectionNameAndSequenceNrGreaterThanAndInEventSource(String collectionName, long sequenceNr); -} diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/valueobjects/BucketRelation.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/valueobjects/BucketRelation.java index 0dc1bcb72..6453a49df 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/valueobjects/BucketRelation.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/valueobjects/BucketRelation.java @@ -1,17 +1,4 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.RdfConstants; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; - -public record BucketRelation( - Bucket fromBucket, - Bucket toBucket, - String treeRelationType, - String treeValue, - String treeValueType, - String treePath -) { - public static BucketRelation createGenericRelation(Bucket fromBucket, Bucket toBucket) { - return new BucketRelation(fromBucket, toBucket, RdfConstants.GENERIC_TREE_RELATION, "", "", ""); - } -} +public record BucketRelation(String fromPartialUrl, String toPartialUrl, TreeRelation relation) { +} \ No newline at end of file diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/valueobjects/BucketRelationCreatedEvent.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/valueobjects/BucketRelationCreatedEvent.java deleted file mode 100644 index 27295eb78..000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/valueobjects/BucketRelationCreatedEvent.java +++ /dev/null @@ -1,4 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects; - -public record BucketRelationCreatedEvent(BucketRelation bucketRelation) { -} diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/valueobjects/TreeRelation.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/valueobjects/TreeRelation.java new file mode 100644 index 000000000..c3efcde7f --- /dev/null +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/valueobjects/TreeRelation.java @@ -0,0 +1,11 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.RdfConstants; + +public record TreeRelation(String treeRelationType, String treeValue, String treeValueType, String treePath) { + + public static TreeRelation generic() { + return new TreeRelation(RdfConstants.GENERIC_TREE_RELATION, "", "", ""); + } + +} diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/module-info.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/module-info.java index e0c48359e..a139eaa48 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/module-info.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/module-info.java @@ -1,6 +1,5 @@ open module ldes.fragmentation.domain { requires ldes.ingest.domain; - requires ldes.server.retention; requires ldes.domain; requires spring.batch.core; requires spring.batch.infrastructure; @@ -16,7 +15,5 @@ requires spring.core; exports be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities; exports be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository; - exports be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.services; exports be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects; - } \ No newline at end of file diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationServiceTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationServiceTest.java index b9870856e..35af7951d 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationServiceTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationServiceTest.java @@ -33,8 +33,8 @@ @RunWith(SpringRunner.class) @SpringBatchTest @EnableAutoConfiguration -@ContextConfiguration(classes = {SpringBatchConfiguration.class, FragmentationService.class }) -@TestPropertySource(properties = { "ldes-server.fragmentation-cron=*/1 * * * * *"}) +@ContextConfiguration(classes = {SpringBatchConfiguration.class, FragmentationService.class}) +@TestPropertySource(properties = {"ldes-server.fragmentation-cron=*/1 * * * * *"}) class FragmentationServiceTest { @MockBean(name = BUCKETISATION_STEP) Step bucketStep; @@ -87,8 +87,7 @@ void when_unprocessedViews_then_triggerJobsForEachViewThatIsntRunningAlready() t String collection = "collection"; when(memberMetricsRepository.getUnprocessedViews()) - .thenReturn(List.of(new ViewName(collection, "v1"), - new ViewName(collection, "v2"))); + .thenReturn(List.of(new ViewName(collection, "v1"), new ViewName(collection, "v2"))); JobExecution jobExecution = mock(JobExecution.class); JobParameters jobParameters = new JobParametersBuilder() diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchCollectionTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchCollectionTest.java index 6c2aff569..8100a0c52 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchCollectionTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchCollectionTest.java @@ -7,7 +7,6 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewSpecification; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.factory.FragmentationStrategyCreator; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketRepository; import io.micrometer.observation.ObservationRegistry; import org.junit.jupiter.api.Test; @@ -21,10 +20,9 @@ class FragmentationStrategyBatchCollectionTest { private static final String COLLECTION_NAME = "collectionName"; private final FragmentationStrategyCreator fragmentationStrategyCreator = mock(FragmentationStrategyCreator.class); private final ObservationRegistry observationRegistry = mock(ObservationRegistry.class); - private final BucketRepository bucketRepository = mock(BucketRepository.class); private final FragmentationStrategyBatchCollection fragmentationStrategyCollection = new FragmentationStrategyBatchCollection( - bucketRepository, fragmentationStrategyCreator, observationRegistry); + fragmentationStrategyCreator, observationRegistry); @Test void when_ViewAddedEventIsReceived_FragmentationStrategyIsAddedToMap() { @@ -81,9 +79,8 @@ private InitViewAddedResult initAddView() { ViewSpecification viewSpecification = new ViewSpecification(viewName, List.of(), List.of(), 100); FragmentationStrategy fragmentationStrategy = mock(FragmentationStrategy.class); - final var rootBucketRetriever = new RootBucketRetriever(viewName, mock(), observationRegistry); FragmentationStrategyBatchExecutor fragmentationStrategyExecutor = - new FragmentationStrategyBatchExecutor(viewName, fragmentationStrategy, rootBucketRetriever, observationRegistry); + new FragmentationStrategyBatchExecutor(viewName, fragmentationStrategy, observationRegistry); return new InitViewAddedResult(viewName, viewSpecification, fragmentationStrategy, fragmentationStrategyExecutor); diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchExecutorTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchExecutorTest.java index cebc5d811..1f9824a5a 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchExecutorTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyBatchExecutorTest.java @@ -30,8 +30,6 @@ class FragmentationStrategyBatchExecutorTest { private ExecutorService executorService; @Mock private FragmentationStrategy fragmentationStrategy; - @Mock - private RootBucketRetriever rootBucketRetriever; @Nested class ExecuteNext { @@ -41,16 +39,15 @@ class ExecuteNext { @Test void when_ExecuteIsCalled_then_AllLogicIsWrappedByTheExecutorService() { ObservationRegistry observationRegistry = ObservationRegistry.NOOP; - var executor = new FragmentationStrategyBatchExecutor(viewName, fragmentationStrategy, rootBucketRetriever, + var executor = new FragmentationStrategyBatchExecutor(viewName, fragmentationStrategy, observationRegistry); var member = mock(FragmentationMember.class); executor.bucketise(member); - verify(rootBucketRetriever).retrieveRootBucket(any()); verify(fragmentationStrategy).addMemberToBucket(any(), eq(member), any()); - verifyNoMoreInteractions(fragmentationStrategy, rootBucketRetriever); + verifyNoMoreInteractions(fragmentationStrategy); } } @@ -58,7 +55,7 @@ void when_ExecuteIsCalled_then_AllLogicIsWrappedByTheExecutorService() { void isPartOfCollection() { final ViewName viewNameA = ViewName.fromString("col/viewA"); var executorA = new FragmentationStrategyBatchExecutor(viewNameA, null, - null, null); + null); assertTrue(executorA.isPartOfCollection(viewNameA.getCollectionName())); assertFalse(executorA.isPartOfCollection("other")); @@ -68,7 +65,7 @@ void isPartOfCollection() { void getViewName() { final ViewName viewNameA = ViewName.fromString("col/viewA"); var executorA = new FragmentationStrategyBatchExecutor(viewNameA, null, - null, null); + null); assertEquals(viewNameA, executorA.getViewName()); } @@ -88,19 +85,19 @@ static class EqualityTestProvider implements ArgumentsProvider { private static final ViewName viewNameA = ViewName.fromString("col/viewA"); private static final FragmentationStrategyBatchExecutor executorA = new FragmentationStrategyBatchExecutor(viewNameA, - null, null, null); + null, null); @Override public Stream provideArguments(ExtensionContext context) { return Stream.of( Arguments.of(equals(), executorA, executorA), Arguments.of(equals(), executorA, - new FragmentationStrategyBatchExecutor(viewNameA, null, null, null)), + new FragmentationStrategyBatchExecutor(viewNameA, null, null)), Arguments.of(equals(), executorA, new FragmentationStrategyBatchExecutor(viewNameA, mock(FragmentationStrategy.class), - mock(RootBucketRetriever.class), mock(ObservationRegistry.class))), + mock(ObservationRegistry.class))), Arguments.of(notEquals(), executorA, - new FragmentationStrategyBatchExecutor(ViewName.fromString("col/viewB"), null, null, null))); + new FragmentationStrategyBatchExecutor(ViewName.fromString("col/viewB"), null, null))); } private static BiConsumer equals() { diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyCreatorImplTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyCreatorImplTest.java index fe1ce5a60..37c9042f4 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyCreatorImplTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyCreatorImplTest.java @@ -34,7 +34,7 @@ class FragmentationStrategyCreatorImplTest { @BeforeEach void setUp() { applicationContext = mock(ApplicationContext.class); - rootBucketCreator = mock(); + rootBucketCreator = mock(RootBucketCreator.class); fragmentationStrategyCreator = new FragmentationStrategyCreatorImpl(applicationContext, rootBucketCreator); } @@ -45,9 +45,9 @@ void when_ViewSpecificationFragmentationConfigIsNull_FragmentationStrategyImplIs FragmentationStrategy fragmentationStrategy = fragmentationStrategyCreator .createFragmentationStrategyForView(viewSpecification); - assertThat(fragmentationStrategy).isOfAnyClassIn(FragmentationStrategyImpl.class); + assertThat(fragmentationStrategy).isInstanceOf(FragmentationStrategyImpl.class); InOrder inOrder = inOrder(applicationContext, rootBucketCreator); - inOrder.verify(rootBucketCreator).createRootBucketForView(viewSpecification.getName()); + inOrder.verify(rootBucketCreator).createRootBucketForView(VIEW_NAME); inOrder.verifyNoMoreInteractions(); } diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyDecoratorTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyDecoratorTest.java index 73cf04205..f0fda9c0a 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyDecoratorTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyDecoratorTest.java @@ -4,16 +4,12 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.FragmentationMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelationCreatedEvent; import io.micrometer.observation.Observation; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.context.ApplicationEventPublisher; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -23,20 +19,9 @@ class FragmentationStrategyDecoratorTest { private static final ViewName VIEW_NAME = new ViewName("collectionName", "view"); @Mock private FragmentationStrategy fragmentationStrategy; - @Mock - private ApplicationEventPublisher applicationEventPublisher; @InjectMocks private FragmentationStrategyDecoratorTestImpl fragmentationStrategyDecorator; - @Test - void when_ParentDoesNotYetHaveRelationToChild_AddRelationAndSaveToDatabase() { - Bucket parentBucket = new Bucket(BucketDescriptor.empty(), VIEW_NAME); - Bucket childBucket = parentBucket.createChild(new BucketDescriptorPair("key", "value")); - - fragmentationStrategyDecorator.addRelationFromParentToChild(parentBucket, childBucket); - - verify(applicationEventPublisher).publishEvent(new BucketRelationCreatedEvent(BucketRelation.createGenericRelation(parentBucket, childBucket))); - } @Test void when_DecoratorAddsMemberToBucket_WrappedFragmentationStrategyIsCalled() { @@ -50,9 +35,8 @@ void when_DecoratorAddsMemberToBucket_WrappedFragmentationStrategyIsCalled() { } static class FragmentationStrategyDecoratorTestImpl extends FragmentationStrategyDecorator { - protected FragmentationStrategyDecoratorTestImpl(FragmentationStrategy fragmentationStrategy, - ApplicationEventPublisher applicationEventPublisher) { - super(fragmentationStrategy, applicationEventPublisher); + protected FragmentationStrategyDecoratorTestImpl(FragmentationStrategy fragmentationStrategy) { + super(fragmentationStrategy); } } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyImplTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyImplTest.java index e86bd5de9..13ae7e8f0 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyImplTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationStrategyImplTest.java @@ -21,18 +21,14 @@ class FragmentationStrategyImplTest { private final FragmentationStrategyImpl fragmentationStrategy = new FragmentationStrategyImpl(); @Test - void when_memberIsAddedToBucket_FragmentationStrategyImplSavesUpdatedFragment() { - BucketisedMember expectedBucketisedMember = new BucketisedMember(BUCKET_ID, MEMBER_ID); - Bucket bucket = new Bucket(2, BucketDescriptor.empty(), VIEW_NAME); + void when_memberIsAddedToBucket_FragmentationStrategyImplAddsMemberToBucket() { + Bucket bucket = new Bucket(BUCKET_ID, BucketDescriptor.empty(), VIEW_NAME, List.of(), 0); FragmentationMember member = mock(FragmentationMember.class); + BucketisedMember expected = new BucketisedMember(BUCKET_ID, MEMBER_ID); when(member.getMemberId()).thenReturn(MEMBER_ID); - List members = fragmentationStrategy.addMemberToBucket(bucket, member, mock(Observation.class)); + fragmentationStrategy.addMemberToBucket(bucket, member, mock(Observation.class)); - assertThat(members) - .hasSize(1) - .first() - .usingRecursiveComparison() - .isEqualTo(expectedBucketisedMember); + assertThat(bucket.getMember()).contains(expected); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/RootBucketRetrieverTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/RootBucketRetrieverTest.java deleted file mode 100644 index de488a0bc..000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/RootBucketRetrieverTest.java +++ /dev/null @@ -1,69 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.exceptions.MissingRootFragmentException; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; -import io.micrometer.observation.Observation; -import io.micrometer.observation.ObservationRegistry; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.util.Optional; - -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class RootBucketRetrieverTest { - - @Mock - private BucketRepository bucketRepository; - - @Mock - private Observation observation; - private RootBucketRetriever rootBucketRetriever; - private Bucket rootBucket; - private static final ViewName VIEW_NAME = new ViewName("collection", "view"); - - @BeforeEach - void setUp() { - rootBucketRetriever = new RootBucketRetriever(VIEW_NAME, bucketRepository, ObservationRegistry.NOOP); - rootBucket = new Bucket(BucketDescriptor.empty(), VIEW_NAME); - when(bucketRepository.retrieveRootBucket(VIEW_NAME)).thenReturn(Optional.of(rootBucket)); - } - - @Test - void should_FetchRootFragment_when_NotYetInMemory() { - final Bucket result = rootBucketRetriever.retrieveRootBucket(observation); - - assertEquals(rootBucket, result); - verify(bucketRepository).retrieveRootBucket(VIEW_NAME); - } - - @Test - void should_ReturnFragmentFromMemory_when_FetchedEarlier() { - rootBucketRetriever.retrieveRootBucket(observation); - rootBucketRetriever.retrieveRootBucket(observation); - rootBucketRetriever.retrieveRootBucket(observation); - final Bucket result = rootBucketRetriever.retrieveRootBucket(observation); - - assertEquals(rootBucket, result); - verify(bucketRepository).retrieveRootBucket(VIEW_NAME); - } - - @Test - void should_ThrowException_when_RootFragmentIsNotFound() { - when(bucketRepository.retrieveRootBucket(VIEW_NAME)).thenReturn(Optional.empty()); - - assertThatThrownBy(() -> rootBucketRetriever.retrieveRootBucket(observation)) - .isInstanceOf(MissingRootFragmentException.class) - .hasMessage("Could not retrieve root fragment for view %s", VIEW_NAME.asString()); - } -} diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/BucketTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/BucketTest.java index 661651e7a..950be0136 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/BucketTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/BucketTest.java @@ -3,6 +3,9 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.TreeRelation; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import java.util.List; @@ -70,4 +73,51 @@ void testHashCode() { .isNotEqualTo(otherBucket.hashCode()); } + @Nested + class CompositionTest { + private static final int DAY_14_BUCKET_ID = 10; + private static final int DAY_28_BUCKET_ID = 11; + private Bucket rootBucket, year2023Bucket, year2024Bucket, month01Bucket, month03Bucket, day14Bucket, day28Bucket; + + @BeforeEach + void setUp() { + rootBucket = new Bucket(BucketDescriptor.empty(), VIEW_NAME); + year2023Bucket = rootBucket.createChild(new BucketDescriptorPair("year", "2023")); + year2024Bucket = rootBucket.createChild(new BucketDescriptorPair("year", "2024")); + month01Bucket = year2024Bucket.createChild(new BucketDescriptorPair("month", "01")); + month03Bucket = year2024Bucket.createChild(new BucketDescriptorPair("month", "03")); + day14Bucket = new Bucket(DAY_14_BUCKET_ID, month03Bucket.createChild(new BucketDescriptorPair("day", "14")).getBucketDescriptor(), VIEW_NAME, List.of(), 0); + day28Bucket = new Bucket(DAY_28_BUCKET_ID, month03Bucket.createChild(new BucketDescriptorPair("day", "28")).getBucketDescriptor(), VIEW_NAME, List.of(), 0); + month03Bucket.addChildBucket(day14Bucket.withGenericRelation()); + month03Bucket.addChildBucket(day28Bucket.withGenericRelation()); + year2024Bucket.addChildBucket(month01Bucket.withGenericRelation()); + year2024Bucket.addChildBucket(month03Bucket.withGenericRelation()); + rootBucket.addChildBucket(year2023Bucket.withGenericRelation()); + rootBucket.addChildBucket(year2024Bucket.withGenericRelation()); + } + + @Test + void test_GetBucketTree() { + final List descendants = rootBucket.getBucketTree(); + + assertThat(descendants).containsExactlyInAnyOrder( + rootBucket, year2023Bucket, year2024Bucket, month01Bucket, month03Bucket, day14Bucket, day28Bucket + ); + } + + @Test + void test_AddChild() { + rootBucket = Bucket.createRootBucketForView(VIEW_NAME); + rootBucket.addChildBucket(year2024Bucket.withGenericRelation()); + final Bucket duplicateBucket = rootBucket.createChild(new BucketDescriptorPair("year", "2024")); + TreeRelation treeRelation = new TreeRelation("duplicate-type", "duplicate-value", "duplicate-value-type", "duplicate-path"); + + rootBucket.addChildBucket(duplicateBucket.withRelations(treeRelation)); + + assertThat(rootBucket.getChildren()).hasSize(1); + assertThat(rootBucket.getChildRelations()) + .hasSize(2) + .satisfiesOnlyOnce(bucketRelation -> assertThat(bucketRelation.relation()).isEqualTo(treeRelation)); + } + } } \ No newline at end of file diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/FragmentSequenceTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/FragmentSequenceTest.java deleted file mode 100644 index 6ebd67a79..000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/FragmentSequenceTest.java +++ /dev/null @@ -1,19 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentSequence; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.*; - -class FragmentSequenceTest { - - @Test - void createNeverProcessedSequence() { - ViewName viewName = ViewName.fromString("col/view"); - var sequence = FragmentSequence.createNeverProcessedSequence(viewName); - - assertEquals(viewName, sequence.viewName()); - assertEquals(-1, sequence.sequenceNr()); - } -} \ No newline at end of file diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/FragmentTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/FragmentTest.java deleted file mode 100644 index e1885e770..000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/FragmentTest.java +++ /dev/null @@ -1,170 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentPair; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.TreeRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.exceptions.DuplicateFragmentPairException; -import org.junit.jupiter.api.Test; - -import java.util.List; -import java.util.Optional; - -import static be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier.fromFragmentId; -import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Fragment.ROOT; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatExceptionOfType; -import static org.junit.jupiter.api.Assertions.*; - -class FragmentTest { - private static final String VIEW = "mobility-hindrances"; - private static final ViewName VIEW_NAME = new ViewName("collectionName", VIEW); - private static final String FRAGMENTATION_VALUE_1 = "2020-12-28T09:36:09.72Z"; - private static final String GENERATED_AT_TIME = "generatedAtTime"; - private static final String FRAGMENTATION_VALUE_2 = "0/0/0"; - private static final String TILE = "tile"; - public static final FragmentPair PARENT_FRAGMENT_PAIR = new FragmentPair("a", "b"); - public static final FragmentPair CHILD_FRAGMENT_PAIR = new FragmentPair("c", "d"); - - @Test - void when_LdesFragmentIsImmutable_IsImmutableReturnsTrue() { - Fragment fragment = new Fragment(new LdesFragmentIdentifier( - VIEW_NAME, - List.of(new FragmentPair(GENERATED_AT_TIME, FRAGMENTATION_VALUE_1)))); - assertFalse(fragment.isImmutable()); - fragment.makeImmutable(); - assertTrue(fragment.isImmutable()); - } - - @Test - void get_FragmentId() { - Fragment fragment = new Fragment(new LdesFragmentIdentifier( - VIEW_NAME, - List.of(new FragmentPair(GENERATED_AT_TIME, FRAGMENTATION_VALUE_1), - new FragmentPair(TILE, FRAGMENTATION_VALUE_2)))); - - assertEquals("/collectionName/mobility-hindrances?generatedAtTime=2020-12-28T09:36:09.72Z&tile=0/0/0", - fragment.getFragmentIdString()); - assertEquals("/collectionName/mobility-hindrances?generatedAtTime=2020-12-28T09:36:09.72Z", - fragment.getParentIdAsString()); - - fragment = new Fragment(new LdesFragmentIdentifier( - VIEW_NAME, List.of())); - assertEquals("/collectionName/mobility-hindrances", - fragment.getFragmentIdString()); - assertEquals(ROOT, fragment.getParentIdAsString()); - - } - - @Test - void when_ValueIsAbsent_GetValueOfKeyReturnsOptionalEmpty() { - Fragment fragment = new Fragment(new LdesFragmentIdentifier(VIEW_NAME, - List.of(new FragmentPair(GENERATED_AT_TIME, FRAGMENTATION_VALUE_1), - new FragmentPair(TILE, FRAGMENTATION_VALUE_2)))); - assertTrue(fragment.getValueOfKey("unexistingKey").isEmpty()); - assertEquals(Optional.of(FRAGMENTATION_VALUE_1), fragment.getValueOfKey(GENERATED_AT_TIME)); - assertEquals(Optional.of(FRAGMENTATION_VALUE_2), fragment.getValueOfKey(TILE)); - } - - @Test - void when_childIsCreated_ViewIsSameAndFragmentPairsAreExtended() { - ViewName viewName = VIEW_NAME; - Fragment fragment = new Fragment( - new LdesFragmentIdentifier(viewName, List.of(PARENT_FRAGMENT_PAIR))); - Fragment child = fragment.createChild(CHILD_FRAGMENT_PAIR); - assertEquals(List.of(PARENT_FRAGMENT_PAIR, CHILD_FRAGMENT_PAIR), child.getFragmentPairs()); - assertFalse(child.isImmutable()); - assertEquals(viewName, child.getViewName()); - } - - @Test - void when_LdesFragmentIsMadeImmutable_ImmutableTimeStampIsSet() { - Fragment fragment = new Fragment(new LdesFragmentIdentifier(VIEW_NAME, - List.of(PARENT_FRAGMENT_PAIR))); - assertFalse(fragment.isImmutable()); - fragment.makeImmutable(); - assertTrue(fragment.isImmutable()); - } - - @Test - void when_ParentExists_Then_ReturnIdParent() { - Fragment parent = new Fragment(new LdesFragmentIdentifier(VIEW_NAME, List.of(PARENT_FRAGMENT_PAIR))); - Fragment child = parent.createChild(CHILD_FRAGMENT_PAIR); - assertThat(child.getParentId()).contains(parent.getFragmentId()); - assertThat(parent.getFragmentIdString()).isEqualTo(child.getParentIdAsString()); - } - - @Test - void when_FragmentPairExists_Then_ThrowError() { - Fragment parent = new Fragment(new LdesFragmentIdentifier(VIEW_NAME, List.of(PARENT_FRAGMENT_PAIR, CHILD_FRAGMENT_PAIR))); - assertThatExceptionOfType(DuplicateFragmentPairException.class) - .isThrownBy(() -> parent.createChild(CHILD_FRAGMENT_PAIR)) - .withMessage("FragmentId /collectionName/mobility-hindrances?a=b&c=d already contains fragmentkey c"); - } - - @Test - void when_ParentDoesNotExists_Then_ReturnEmpty() { - Fragment rootFragment = new Fragment(new LdesFragmentIdentifier(VIEW_NAME, List.of())); - assertEquals(Optional.empty(), rootFragment.getParentId()); - assertEquals(ROOT, rootFragment.getParentIdAsString()); - } - - @Test - void when_removeRelationToIdentifier_Then_ExpectItemToBeRemoved() { - Fragment a = new Fragment(fromFragmentId("/c/v")); - a.addRelation(createEmptyRelationForFragment(fromFragmentId("/c/v1?k=1"))); - a.addRelation(createEmptyRelationForFragment(fromFragmentId("/c/v1?k=2"))); - a.addRelation(createEmptyRelationForFragment(fromFragmentId("/c/v1?k=3"))); - a.addRelation(createEmptyRelationForFragment(fromFragmentId("/c/v2?k=1"))); - - LdesFragmentIdentifier toRemove = fromFragmentId("/c/v1?k=2"); - - assertEquals(4, a.getRelations().size()); - a.removeRelationToIdentifier(toRemove); - assertFalse(a.getRelations().stream().anyMatch(treeRelation -> treeRelation.treeNode().equals(toRemove))); - assertEquals(3, a.getRelations().size()); - } - - @Test - void testEquals() { - Fragment a = new Fragment(new LdesFragmentIdentifier(new ViewName("collectionName", "a"), List.of())); - Fragment a2 = new Fragment(new LdesFragmentIdentifier(new ViewName("collectionName", "a"), List.of())); - Fragment c = new Fragment(new LdesFragmentIdentifier(new ViewName("collectionName", "c"), List.of())); - - assertEquals(a, a2); - assertEquals(a2, a); - assertNotEquals(a, c); - } - - @Test - void testHashCode() { - Fragment a = new Fragment(new LdesFragmentIdentifier(new ViewName("collectionName", "a"), List.of())); - Fragment a2 = new Fragment(new LdesFragmentIdentifier(new ViewName("collectionName", "a"), List.of())); - Fragment c = new Fragment(new LdesFragmentIdentifier(new ViewName("collectionName", "c"), List.of())); - - assertEquals(a.hashCode(), a2.hashCode()); - assertEquals(a2.hashCode(), a.hashCode()); - assertNotEquals(a.hashCode(), c.hashCode()); - } - - @Test - void isConnectedTo() { - Fragment f3 = new Fragment(fromFragmentId("collection/3"), true, 0, List.of(), - null); - Fragment f2 = new Fragment(fromFragmentId("collection/1"), true, 0, List.of(), - null); - Fragment f1 = new Fragment(fromFragmentId("collection/2"), true, 0, - List.of(new TreeRelation(null, f2.getFragmentId(), null, null, null)), - null); - - assertFalse(f2.isConnectedTo(f1)); - assertTrue(f1.isConnectedTo(f2)); - assertFalse(f3.isConnectedTo(f2)); - assertFalse(f3.isConnectedTo(f1)); - } - - private TreeRelation createEmptyRelationForFragment(LdesFragmentIdentifier fragmentIdentifier) { - return new TreeRelation(null, fragmentIdentifier, null, null, null); - } - -} diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategy.java b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategy.java index 547717ded..d579364c0 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategy.java +++ b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategy.java @@ -3,17 +3,11 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategy; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategyDecorator; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.FragmentationMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.bucketising.GeospatialBucketiser; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.fragments.GeospatialBucketCreator; import io.micrometer.observation.Observation; import io.micrometer.observation.ObservationRegistry; -import org.springframework.context.ApplicationEventPublisher; - -import java.util.Collection; -import java.util.List; -import java.util.Set; import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.ServerConstants.DEFAULT_BUCKET_STRING; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.constants.GeospatialConstants.FRAGMENT_KEY_TILE_ROOT; @@ -24,54 +18,38 @@ public class GeospatialFragmentationStrategy extends FragmentationStrategyDecora private final GeospatialBucketCreator bucketCreator; private final ObservationRegistry observationRegistry; - private Bucket rootTileBucket = null; - public GeospatialFragmentationStrategy(FragmentationStrategy fragmentationStrategy, GeospatialBucketiser geospatialBucketiser, GeospatialBucketCreator bucketCreator, - ObservationRegistry observationRegistry, - ApplicationEventPublisher applicationEventPublisher) { - super(fragmentationStrategy, applicationEventPublisher); + ObservationRegistry observationRegistry) { + super(fragmentationStrategy); this.geospatialBucketiser = geospatialBucketiser; this.bucketCreator = bucketCreator; this.observationRegistry = observationRegistry; } @Override - public List addMemberToBucket(Bucket parentBucket, FragmentationMember member, - Observation parentObservation) { + public void addMemberToBucket(Bucket parentBucket, FragmentationMember member, Observation parentObservation) { Observation geospatialFragmentationObservation = Observation .createNotStarted("geospatial bucketisation", observationRegistry) .parentObservation(parentObservation) .start(); - setRootTileBucket(parentBucket); - - Set tiles = geospatialBucketiser.bucketise(member.getSubject(), member.getVersionModel()); - List buckets = tiles + geospatialBucketiser.createTiles(member.getSubject(), member.getVersionModel()) .stream() .map(tile -> { if (tile.equals(DEFAULT_BUCKET_STRING)) { - return bucketCreator.getOrCreateTileBucket(parentBucket, tile, parentBucket); + return bucketCreator.createTileBucket(parentBucket, tile, parentBucket); } else { - return bucketCreator.getOrCreateTileBucket(parentBucket, tile, rootTileBucket); + Bucket rootTileBucket = createRootTileBucket(parentBucket); + return bucketCreator.createTileBucket(parentBucket, tile, rootTileBucket); } - }).toList(); - - List members = buckets - .parallelStream() - .map(bucket -> super.addMemberToBucket(bucket, member, geospatialFragmentationObservation)) - .flatMap(Collection::stream) - .toList(); + }) + .forEach(bucket -> super.addMemberToBucket(bucket, member, geospatialFragmentationObservation)); geospatialFragmentationObservation.stop(); - return members; } - - private void setRootTileBucket(Bucket parentBucket) { - if (rootTileBucket == null) { - rootTileBucket = bucketCreator.getOrCreateRootBucket(parentBucket, FRAGMENT_KEY_TILE_ROOT); - super.addRelationFromParentToChild(parentBucket, rootTileBucket); - } + private Bucket createRootTileBucket(Bucket parentBucket) { + return parentBucket.addChildBucket(bucketCreator.createRootBucket(parentBucket, FRAGMENT_KEY_TILE_ROOT).withGenericRelation()); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategyWrapper.java b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategyWrapper.java index 249527ba9..574cf44af 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategyWrapper.java +++ b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategyWrapper.java @@ -3,7 +3,6 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ConfigProperties; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategy; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategyWrapper; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.bucketising.GeospatialBucketiser; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.config.GeospatialConfig; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.connected.relations.TileBucketRelationsAttributer; @@ -18,15 +17,14 @@ public class GeospatialFragmentationStrategyWrapper implements FragmentationStra public FragmentationStrategy wrapFragmentationStrategy(ApplicationContext applicationContext, FragmentationStrategy fragmentationStrategy, ConfigProperties fragmentationProperties) { ObservationRegistry observationRegistry = applicationContext.getBean(ObservationRegistry.class); - BucketRepository bucketRepository = applicationContext.getBean(BucketRepository.class); - TileBucketRelationsAttributer tileBucketRelationsAttributer = new TileBucketRelationsAttributer(applicationContext); + TileBucketRelationsAttributer tileBucketRelationsAttributer = new TileBucketRelationsAttributer(); GeospatialConfig geospatialConfig = createGeospatialConfig(fragmentationProperties); GeospatialBucketiser geospatialBucketiser = new GeospatialBucketiser(geospatialConfig); - GeospatialBucketCreator geospatialBucketCreator = new GeospatialBucketCreator(bucketRepository, tileBucketRelationsAttributer); + GeospatialBucketCreator geospatialBucketCreator = new GeospatialBucketCreator(tileBucketRelationsAttributer); return new GeospatialFragmentationStrategy(fragmentationStrategy, - geospatialBucketiser, geospatialBucketCreator, observationRegistry, applicationContext); + geospatialBucketiser, geospatialBucketCreator, observationRegistry); } private GeospatialConfig createGeospatialConfig(ConfigProperties properties) { diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/bucketising/GeospatialBucketiser.java b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/bucketising/GeospatialBucketiser.java index f92fffdbf..e22e41e84 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/bucketising/GeospatialBucketiser.java +++ b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/bucketising/GeospatialBucketiser.java @@ -26,11 +26,11 @@ public GeospatialBucketiser(GeospatialConfig geospatialConfig) { this.geospatialConfig = geospatialConfig; } - public Set bucketise(String memberId, Model memberModel) { + public Set createTiles(String memberId, Model memberModel) { try { Set tiles = getFragmentationObjects(memberModel, geospatialConfig.fragmenterSubjectFilter(), geospatialConfig.fragmentationPath()) .flatMap(geometryWrapper -> calculateTiles(geometryWrapper.getXYGeometry().toText(), geospatialConfig.maxZoom()).stream()) - .collect(Collectors.toSet()); + .collect(Collectors.toSet()); if(tiles.isEmpty()) { tiles.add(DEFAULT_BUCKET_STRING); diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/GeospatialRelationsAttributer.java b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/GeospatialRelationsAttributer.java deleted file mode 100644 index 4c9d4b027..000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/GeospatialRelationsAttributer.java +++ /dev/null @@ -1,53 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.connected.relations; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.TreeRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Fragment; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.exceptions.MissingFragmentValueException; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.relations.RelationsAttributer; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.connected.BoundingBox; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.converter.BoundingBoxConverter; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.converter.TileConverter; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.model.Tile; - -import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.constants.GeospatialConstants.*; - -public class GeospatialRelationsAttributer implements RelationsAttributer { - - public TreeRelation getRelationToParentFragment(Fragment childFragment) { - String targetWKT = getWKT(childFragment); - - return new TreeRelation(GEOSPARQL_AS_WKT, childFragment.getFragmentId(), - WGS_84 + " " + targetWKT, WKT_DATA_TYPE, TREE_GEOSPATIALLY_CONTAINS_RELATION); - } - - public BucketRelation createRelationBetween(Bucket parentBucket, Bucket childBucket) { - final String treeValue = WGS_84 + " " + getWKT(childBucket); - return new BucketRelation( - parentBucket, - childBucket, - TREE_GEOSPATIALLY_CONTAINS_RELATION, - treeValue, - WKT_DATA_TYPE, - GEOSPARQL_AS_WKT - ); - } - - private String getWKT(Fragment currentFragment) { - String fragmentWKT = currentFragment.getValueOfKey(FRAGMENT_KEY_TILE).orElseThrow( - () -> new MissingFragmentValueException(currentFragment.getFragmentIdString(), FRAGMENT_KEY_TILE)); - Tile currentTile = TileConverter.fromString(fragmentWKT); - BoundingBox currentBoundingBox = new BoundingBox(currentTile); - return BoundingBoxConverter.toWkt(currentBoundingBox); - } - - private String getWKT(Bucket currentBucket) { - String fragmentWKT = currentBucket.getValueForKey(FRAGMENT_KEY_TILE).orElseThrow( - () -> new MissingFragmentValueException(currentBucket.getBucketDescriptorAsString(), FRAGMENT_KEY_TILE)); - Tile currentTile = TileConverter.fromString(fragmentWKT); - BoundingBox currentBoundingBox = new BoundingBox(currentTile); - return BoundingBoxConverter.toWkt(currentBoundingBox); - } - -} diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/TileBucketRelationsAttributer.java b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/TileBucketRelationsAttributer.java index 4f5a9d754..2706ab63f 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/TileBucketRelationsAttributer.java +++ b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/TileBucketRelationsAttributer.java @@ -1,25 +1,40 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.connected.relations; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelationCreatedEvent; -import org.springframework.context.ApplicationEventPublisher; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.exceptions.MissingFragmentValueException; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.relations.RelationsAttributer; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.TreeRelation; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.connected.BoundingBox; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.converter.BoundingBoxConverter; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.converter.TileConverter; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.model.Tile; import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.ServerConstants.DEFAULT_BUCKET_STRING; -import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.constants.GeospatialConstants.FRAGMENT_KEY_TILE; +import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.constants.GeospatialConstants.*; -public class TileBucketRelationsAttributer { +public class TileBucketRelationsAttributer implements RelationsAttributer { - private final GeospatialRelationsAttributer relationsAttributer = new GeospatialRelationsAttributer(); - private final ApplicationEventPublisher applicationEventPublisher; + public Bucket addRelationsFromRootToBottom(Bucket rootBucket, Bucket tileBucket) { + boolean isDefaultBucket = tileBucket.getValueForKey(FRAGMENT_KEY_TILE).orElse("").equals(DEFAULT_BUCKET_STRING); + TreeRelation treeRelation = isDefaultBucket ? TreeRelation.generic() : createGeospatialRelationToParent(tileBucket); + return rootBucket.addChildBucket(tileBucket.withRelations(treeRelation)); + } - public TileBucketRelationsAttributer(ApplicationEventPublisher applicationEventPublisher) { - this.applicationEventPublisher = applicationEventPublisher; + private TreeRelation createGeospatialRelationToParent(Bucket childBucket) { + final String treeValue = WGS_84 + " " + getWKT(childBucket); + return new TreeRelation( + TREE_GEOSPATIALLY_CONTAINS_RELATION, + treeValue, + WKT_DATA_TYPE, + GEOSPARQL_AS_WKT + ); } - public void addRelationsFromRootToBottom(Bucket rootBucket, Bucket tileBucket) { - boolean isDefaultBucket = tileBucket.getValueForKey(FRAGMENT_KEY_TILE).orElse("").equals(DEFAULT_BUCKET_STRING); - BucketRelation bucketRelation = isDefaultBucket ? BucketRelation.createGenericRelation(rootBucket, tileBucket) : relationsAttributer.createRelationBetween(rootBucket, tileBucket); - applicationEventPublisher.publishEvent(new BucketRelationCreatedEvent(bucketRelation)); + private String getWKT(Bucket currentBucket) { + String fragmentWKT = currentBucket.getValueForKey(FRAGMENT_KEY_TILE).orElseThrow( + () -> new MissingFragmentValueException(currentBucket.getBucketDescriptorAsString(), FRAGMENT_KEY_TILE)); + Tile currentTile = TileConverter.fromString(fragmentWKT); + BoundingBox currentBoundingBox = new BoundingBox(currentTile); + return BoundingBoxConverter.toWkt(currentBoundingBox); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/exceptions/RdfGeometryException.java b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/exceptions/RdfGeometryException.java deleted file mode 100644 index ae293f31b..000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/exceptions/RdfGeometryException.java +++ /dev/null @@ -1,19 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.exceptions; - -import org.apache.jena.geosparql.implementation.GeometryWrapper; - -public class RdfGeometryException extends RuntimeException { - private final GeometryWrapper geometryWrapper; - private final String srsFormat; - - public RdfGeometryException(GeometryWrapper geometryWrapper, String srsFormat, Exception e) { - super(e); - this.geometryWrapper = geometryWrapper; - this.srsFormat = srsFormat; - } - - @Override - public String getMessage() { - return "unable to convert member" + geometryWrapper + " to " + srsFormat + "format"; - } -} diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/fragments/GeospatialBucketCreator.java b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/fragments/GeospatialBucketCreator.java index c0590a62f..1abd97787 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/fragments/GeospatialBucketCreator.java +++ b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/fragments/GeospatialBucketCreator.java @@ -1,45 +1,27 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.fragments; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.connected.relations.TileBucketRelationsAttributer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.constants.GeospatialConstants.FRAGMENT_KEY_TILE; public class GeospatialBucketCreator { - private final BucketRepository bucketRepository; private final TileBucketRelationsAttributer tileBucketRelationsAttributer; - private static final Logger LOGGER = LoggerFactory.getLogger(GeospatialBucketCreator.class); - public GeospatialBucketCreator(BucketRepository bucketRepository, TileBucketRelationsAttributer tileBucketRelationsAttributer) { - this.bucketRepository = bucketRepository; + public GeospatialBucketCreator(TileBucketRelationsAttributer tileBucketRelationsAttributer) { this.tileBucketRelationsAttributer = tileBucketRelationsAttributer; } - public Bucket getOrCreateTileBucket(Bucket parentBucket, String tile, Bucket rootTileFragment) { + public Bucket createTileBucket(Bucket parentBucket, String tile, Bucket rootTileFragment) { final BucketDescriptorPair childDescriptorPair = new BucketDescriptorPair(FRAGMENT_KEY_TILE, tile); - return bucketRepository - .retrieveBucket(parentBucket.getViewName(), parentBucket.createChildDescriptor(childDescriptorPair)) - .orElseGet(() -> { - final Bucket childBucket = bucketRepository.insertBucket(parentBucket.createChild(childDescriptorPair)); - tileBucketRelationsAttributer.addRelationsFromRootToBottom(rootTileFragment, childBucket); - LOGGER.debug("Geospatial fragment created with id: {}", childBucket.getBucketDescriptorAsString()); - return childBucket; - }); + final Bucket childBucket = parentBucket.createChild(childDescriptorPair); + return tileBucketRelationsAttributer.addRelationsFromRootToBottom(rootTileFragment, childBucket); } - public Bucket getOrCreateRootBucket(Bucket parentBucket, String tile) { + public Bucket createRootBucket(Bucket parentBucket, String tile) { final BucketDescriptorPair childDescriptorPair = new BucketDescriptorPair(FRAGMENT_KEY_TILE, tile); - return bucketRepository - .retrieveBucket(parentBucket.getViewName(), parentBucket.createChildDescriptor(childDescriptorPair)) - .orElseGet(() -> { - final Bucket childBucket = bucketRepository.insertBucket(parentBucket.createChild(childDescriptorPair)); - LOGGER.debug("Geospatial rootfragment created with id: {}", childBucket.getBucketDescriptorAsString()); - return childBucket; - }); + return parentBucket.createChild(childDescriptorPair); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategyTest.java b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategyTest.java index 2e9436c09..56a95fc3f 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategyTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategyTest.java @@ -18,14 +18,15 @@ import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.ServerConstants.DEFAULT_BUCKET_STRING; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.constants.GeospatialConstants.FRAGMENT_KEY_TILE; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.constants.GeospatialConstants.FRAGMENT_KEY_TILE_ROOT; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.*; class GeospatialFragmentationStrategyTest { private static final ViewName VIEW_NAME = new ViewName("collectionName", "view"); - private static final Bucket PARENT_BUCKET = new Bucket(BucketDescriptor.empty(), VIEW_NAME); - private static final Bucket ROOT_TILE_BUCKET = PARENT_BUCKET.createChild(new BucketDescriptorPair(FRAGMENT_KEY_TILE, FRAGMENT_KEY_TILE_ROOT)); + private Bucket parentBucket; + private Bucket rootTileBucket; private GeospatialBucketiser geospatialBucketiser; private GeospatialBucketCreator bucketCreator; private FragmentationStrategy decoratedFragmentationStrategy; @@ -33,29 +34,46 @@ class GeospatialFragmentationStrategyTest { @BeforeEach void setUp() { + parentBucket = new Bucket(BucketDescriptor.empty(), VIEW_NAME); + rootTileBucket = parentBucket.createChild(new BucketDescriptorPair(FRAGMENT_KEY_TILE, FRAGMENT_KEY_TILE_ROOT)); + geospatialBucketiser = mock(GeospatialBucketiser.class); bucketCreator = mock(GeospatialBucketCreator.class); decoratedFragmentationStrategy = mock(FragmentationStrategy.class); - when(bucketCreator.getOrCreateRootBucket(PARENT_BUCKET, FRAGMENT_KEY_TILE_ROOT)).thenReturn(ROOT_TILE_BUCKET); + when(bucketCreator.createRootBucket(parentBucket, FRAGMENT_KEY_TILE_ROOT)).thenReturn(rootTileBucket); geospatialFragmentationStrategy = new GeospatialFragmentationStrategy(decoratedFragmentationStrategy, - geospatialBucketiser, bucketCreator, ObservationRegistry.create(), - mock()); + geospatialBucketiser, bucketCreator, ObservationRegistry.create()); } - + + @Test + void when_RootTileBucketIsNotSet_then_SetRootTileBucket() { + FragmentationMember member = mock(FragmentationMember.class); + + when(geospatialBucketiser.createTiles(member.getSubject(), member.getVersionModel())).thenReturn(Set.of("dummy")); + when(bucketCreator.createTileBucket(parentBucket, "dummy", parentBucket)) + .thenReturn(mock(Bucket.class)); + + geospatialFragmentationStrategy.addMemberToBucket(parentBucket, member, mock(Observation.class)); + + verify(decoratedFragmentationStrategy).addMemberToBucket(any(), any(), any(Observation.class)); + verifyNoMoreInteractions(decoratedFragmentationStrategy); + assertThat(parentBucket.getChildren()) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyInAnyOrder(rootTileBucket.withGenericRelation()); + } + @Test void when_MemberIsAddedToDefaultFragment_GeospatialFragmentationIsApplied() { FragmentationMember member = mock(FragmentationMember.class); - when(geospatialBucketiser.bucketise(member.getSubject(), member.getVersionModel())).thenReturn(Set.of(DEFAULT_BUCKET_STRING)); - Bucket defaultTileBucket = PARENT_BUCKET.createChild(new BucketDescriptorPair(FRAGMENT_KEY_TILE, DEFAULT_BUCKET_STRING)); - when(bucketCreator.getOrCreateTileBucket(PARENT_BUCKET, DEFAULT_BUCKET_STRING, PARENT_BUCKET)) + when(geospatialBucketiser.createTiles(member.getSubject(), member.getVersionModel())).thenReturn(Set.of(DEFAULT_BUCKET_STRING)); + Bucket defaultTileBucket = parentBucket.createChild(new BucketDescriptorPair(FRAGMENT_KEY_TILE, DEFAULT_BUCKET_STRING)); + when(bucketCreator.createTileBucket(parentBucket, DEFAULT_BUCKET_STRING, parentBucket)) .thenReturn(defaultTileBucket); - geospatialFragmentationStrategy.addMemberToBucket(PARENT_BUCKET, member, mock(Observation.class)); + geospatialFragmentationStrategy.addMemberToBucket(parentBucket, member, mock(Observation.class)); - verify(decoratedFragmentationStrategy, - times(1)).addMemberToBucket(eq(defaultTileBucket), - any(), any(Observation.class)); + verify(decoratedFragmentationStrategy).addMemberToBucket(eq(defaultTileBucket), any(), any(Observation.class)); verifyNoMoreInteractions(decoratedFragmentationStrategy); } @@ -63,13 +81,13 @@ void when_MemberIsAddedToDefaultFragment_GeospatialFragmentationIsApplied() { void when_MemberIsAddedToBucket_GeospatialFragmentationIsApplied() { FragmentationMember member = mock(FragmentationMember.class); - when(geospatialBucketiser.bucketise(member.getSubject(), member.getVersionModel())).thenReturn(Set.of("1/1/1", + when(geospatialBucketiser.createTiles(member.getSubject(), member.getVersionModel())).thenReturn(Set.of("1/1/1", "2/2/2", "3/3/3")); Bucket tileBucketOne = mockCreationGeospatialBucket("1/1/1"); Bucket tileBucketTwo = mockCreationGeospatialBucket("2/2/2"); Bucket tileBucketThree = mockCreationGeospatialBucket("3/3/3"); - geospatialFragmentationStrategy.addMemberToBucket(PARENT_BUCKET, member, mock(Observation.class)); + geospatialFragmentationStrategy.addMemberToBucket(parentBucket, member, mock(Observation.class)); verify(decoratedFragmentationStrategy).addMemberToBucket(eq(tileBucketOne), any(), any(Observation.class)); verify(decoratedFragmentationStrategy).addMemberToBucket(eq(tileBucketTwo), any(), any(Observation.class)); @@ -77,24 +95,9 @@ void when_MemberIsAddedToBucket_GeospatialFragmentationIsApplied() { verifyNoMoreInteractions(decoratedFragmentationStrategy); } - @Test - void when_MemberIsAddedToDefaultBucket_GeospatialFragmentationIsApplied() { - FragmentationMember member = mock(FragmentationMember.class); - - when(geospatialBucketiser.bucketise(member.getSubject(), member.getVersionModel())).thenReturn(Set.of(DEFAULT_BUCKET_STRING)); - Bucket defaultTileBucket = PARENT_BUCKET.createChild(new BucketDescriptorPair(FRAGMENT_KEY_TILE, DEFAULT_BUCKET_STRING)); - when(bucketCreator.getOrCreateTileBucket(PARENT_BUCKET, DEFAULT_BUCKET_STRING, PARENT_BUCKET)) - .thenReturn(defaultTileBucket); - - geospatialFragmentationStrategy.addMemberToBucket(PARENT_BUCKET, member, mock(Observation.class)); - - verify(decoratedFragmentationStrategy).addMemberToBucket(eq(defaultTileBucket), any(), any(Observation.class)); - verifyNoMoreInteractions(decoratedFragmentationStrategy); - } - private Bucket mockCreationGeospatialBucket(String tile) { - Bucket tileBucket = PARENT_BUCKET.createChild(new BucketDescriptorPair(FRAGMENT_KEY_TILE, tile)); - when(bucketCreator.getOrCreateTileBucket(PARENT_BUCKET, tile, ROOT_TILE_BUCKET)).thenReturn(tileBucket); + Bucket tileBucket = parentBucket.createChild(new BucketDescriptorPair(FRAGMENT_KEY_TILE, tile)); + when(bucketCreator.createTileBucket(parentBucket, tile, rootTileBucket)).thenReturn(tileBucket); return tileBucket; } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategyWrapperTest.java b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategyWrapperTest.java index bf1dc9986..94407154b 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategyWrapperTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/GeospatialFragmentationStrategyWrapperTest.java @@ -8,7 +8,7 @@ import java.util.Map; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; class GeospatialFragmentationStrategyWrapperTest { @@ -26,8 +26,10 @@ void setUp() { void when_FragmentationStrategyIsUpdated_GeospatialFragmentationStrategyIsReturned() { ConfigProperties properties = new ConfigProperties( Map.of("maxZoom", "15", "fragmentationPath", "http://www.opengis.net/ont/geosparql#asWKT")); + FragmentationStrategy decoratedFragmentationStrategy = geospatialFragmentationUpdater .wrapFragmentationStrategy(applicationContext, fragmentationStrategy, properties); - assertTrue(decoratedFragmentationStrategy instanceof GeospatialFragmentationStrategy); + + assertThat(decoratedFragmentationStrategy).isInstanceOf(GeospatialFragmentationStrategy.class); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/bucketising/GeospatialBucketiserTest.java b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/bucketising/GeospatialBucketiserTest.java index ddac4cc0e..349dfbea2 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/bucketising/GeospatialBucketiserTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/bucketising/GeospatialBucketiserTest.java @@ -40,7 +40,7 @@ void when_MemberIsBucketized_CorrectBucketsAreReturned() throws URISyntaxExcepti FragmentationMember member = readLdesMemberFromFile(getClass().getClassLoader(), "examples/ldes-member-bucketising.nq"); - Set actualBuckets = bucketiser.bucketise(member.getSubject(), member.getVersionModel()); + Set actualBuckets = bucketiser.createTiles(member.getSubject(), member.getVersionModel()); assertEquals(4, actualBuckets.size()); assertEquals(expectedBuckets, actualBuckets); @@ -56,7 +56,7 @@ void when_MemberWith2GeoPropertiesIsBucketized_CorrectBucketsAreReturned() FragmentationMember member = readLdesMemberFromFile(getClass().getClassLoader(), "examples/ldes-member-2-geo-props-bucketising.nq"); - Set actualBuckets = bucketiser.bucketise(member.getSubject(), member.getVersionModel()); + Set actualBuckets = bucketiser.createTiles(member.getSubject(), member.getVersionModel()); assertEquals(2, actualBuckets.size()); assertEquals(expectedBuckets, actualBuckets); @@ -72,7 +72,7 @@ void when_MemberWithIncorrectGeospatialProperty_Then_DefaultBucketIsReturned() FragmentationMember member = readLdesMemberFromFile(getClass().getClassLoader(), "examples/ldes-member-bucketising-faulty.nq"); - Set actualBuckets = bucketiser.bucketise(member.getSubject(), member.getVersionModel()); + Set actualBuckets = bucketiser.createTiles(member.getSubject(), member.getVersionModel()); assertEquals(1, actualBuckets.size()); assertEquals(expectedBuckets, actualBuckets); @@ -88,7 +88,7 @@ void when_MemberWith1FaultyGeoPropertyIsBucketized_CorrectBucketsAreReturned() FragmentationMember member = readLdesMemberFromFile(getClass().getClassLoader(), "examples/ldes-member-2-geo-props-bucketising-faulty.nq"); - Set actualBuckets = bucketiser.bucketise(member.getSubject(), member.getVersionModel()); + Set actualBuckets = bucketiser.createTiles(member.getSubject(), member.getVersionModel()); assertEquals(1, actualBuckets.size()); assertEquals(expectedBuckets, actualBuckets); @@ -103,7 +103,7 @@ void when_MemberHasInvalidCoordinates_DefaultBucketIsReturned() throws URISyntax FragmentationMember member = readLdesMemberFromFile(getClass().getClassLoader(), "examples/ldes-member-bucketising-invalid-coordinates.nq"); - Set actualBuckets = bucketiser.bucketise(member.getSubject(), member.getVersionModel()); + Set actualBuckets = bucketiser.createTiles(member.getSubject(), member.getVersionModel()); assertEquals(1, actualBuckets.size()); assertEquals(expectedBuckets, actualBuckets); diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/GeospatialRelationsAttributerTest.java b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/GeospatialRelationsAttributerTest.java deleted file mode 100644 index a74d7a1e8..000000000 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/GeospatialRelationsAttributerTest.java +++ /dev/null @@ -1,35 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.connected.relations; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentPair; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.TreeRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Fragment; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.constants.GeospatialConstants.FRAGMENT_KEY_TILE; -import static org.junit.jupiter.api.Assertions.assertEquals; - -class GeospatialRelationsAttributerTest { - - private final GeospatialRelationsAttributer geospatialRelationsAttributer = new GeospatialRelationsAttributer(); - private static final ViewName VIEW_NAME = new ViewName("collectionName", "view"); - private static final Fragment CHILD_FRAGMENT = new Fragment(new LdesFragmentIdentifier(VIEW_NAME, - List.of(new FragmentPair(FRAGMENT_KEY_TILE, - "1/1/1")))); - - private static final TreeRelation EXPECTED_RELATION = new TreeRelation("http://www.opengis.net/ont/geosparql#asWKT", - LdesFragmentIdentifier.fromFragmentId("/collectionName/view?tile=1/1/1"), - " POLYGON ((180 0, 180 -85.0511287798066, 0 -85.0511287798066, 0 0, 180 0))", - "http://www.opengis.net/ont/geosparql#wktLiteral", - "https://w3id.org/tree#GeospatiallyContainsRelation"); - - @Test - void when_getRelation_GeospatialRelationIsReturned() { - TreeRelation relationToParentFragment = geospatialRelationsAttributer.getRelationToParentFragment( - CHILD_FRAGMENT); - assertEquals(EXPECTED_RELATION, relationToParentFragment); - } -} diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/TileBucketRelationsAttributerTest.java b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/TileBucketRelationsAttributerTest.java index a9b687f84..2f5b01406 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/TileBucketRelationsAttributerTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/connected/relations/TileBucketRelationsAttributerTest.java @@ -4,37 +4,34 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelationCreatedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.TreeRelation; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.context.ApplicationEventPublisher; import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.ServerConstants.DEFAULT_BUCKET_STRING; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.constants.GeospatialConstants.FRAGMENT_KEY_TILE; -import static org.mockito.Mockito.verify; +import static org.assertj.core.api.Assertions.assertThat; @ExtendWith(MockitoExtension.class) class TileBucketRelationsAttributerTest { private static final ViewName VIEW_NAME = new ViewName("collectionName", "view"); private static final Bucket PARENT_BUCKET = new Bucket(BucketDescriptor.empty(), VIEW_NAME); - @Mock - private ApplicationEventPublisher applicationEventPublisher; - @InjectMocks + private TileBucketRelationsAttributer tileBucketRelationsAttributer; + @BeforeEach + void setUp() { + tileBucketRelationsAttributer = new TileBucketRelationsAttributer(); + } @Test void when_TileFragmentsAreCreated_RelationsBetweenRootAndCreatedFragmentsAreAdded() { Bucket rootBucket = createTileBucket("0/0/0"); Bucket tileBucket = createTileBucket("1/1/1"); - BucketRelation bucketRelation = new BucketRelation( - rootBucket, - tileBucket, + TreeRelation treeRelation = new TreeRelation( "https://w3id.org/tree#GeospatiallyContainsRelation", " POLYGON ((180 0, 180 -85.0511287798066, 0 -85.0511287798066, 0 0, 180 0))", "http://www.opengis.net/ont/geosparql#wktLiteral", @@ -43,19 +40,21 @@ void when_TileFragmentsAreCreated_RelationsBetweenRootAndCreatedFragmentsAreAdde tileBucketRelationsAttributer.addRelationsFromRootToBottom(rootBucket, tileBucket); - verify(applicationEventPublisher).publishEvent(new BucketRelationCreatedEvent(bucketRelation)); + assertThat(rootBucket.getChildren()) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyInAnyOrder(tileBucket.withRelations(treeRelation)); } @Test void when_DefaultFragmentIsCreated_RelationsBetweenRootAndCreatedFragmentIsAdded() { Bucket rootBucket = createTileBucket("0/0/0"); Bucket tileBucket = createTileBucket(DEFAULT_BUCKET_STRING); - BucketRelation bucketRelation = BucketRelation.createGenericRelation(rootBucket, tileBucket); tileBucketRelationsAttributer.addRelationsFromRootToBottom(rootBucket, tileBucket); - verify(applicationEventPublisher).publishEvent(new BucketRelationCreatedEvent(bucketRelation)); - + assertThat(rootBucket.getChildren()) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyInAnyOrder(tileBucket.withGenericRelation()); } private Bucket createTileBucket(String tile) { diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/fragments/GeospatialBucketCreatorTest.java b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/fragments/GeospatialBucketCreatorTest.java index bc99da427..83d242e41 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/fragments/GeospatialBucketCreatorTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-geospatial/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/geospatial/fragments/GeospatialBucketCreatorTest.java @@ -2,16 +2,16 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.connected.relations.TileBucketRelationsAttributer; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import java.util.Optional; - import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.ServerConstants.DEFAULT_BUCKET_STRING; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.constants.GeospatialConstants.FRAGMENT_KEY_TILE; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.geospatial.constants.GeospatialConstants.FRAGMENT_KEY_TILE_ROOT; @@ -22,110 +22,57 @@ class GeospatialBucketCreatorTest { private static final ViewName VIEW_NAME = new ViewName("collectionName", "view"); private static final BucketDescriptorPair timebasedPair = new BucketDescriptorPair("year", "2023"); - private static final BucketDescriptorPair geoRootPair = new BucketDescriptorPair(FRAGMENT_KEY_TILE, FRAGMENT_KEY_TILE_ROOT); - private static final BucketDescriptorPair geoPair = new BucketDescriptorPair(FRAGMENT_KEY_TILE, "15/101/202"); - private static final BucketDescriptorPair defaultPair = new BucketDescriptorPair(FRAGMENT_KEY_TILE, DEFAULT_BUCKET_STRING); + private BucketDescriptorPair geoRootPair; + private BucketDescriptorPair geoPair; + private BucketDescriptorPair defaultPair; - private BucketRepository bucketRepository; + @Mock + private TileBucketRelationsAttributer tileBucketRelationsAttributer; + @InjectMocks private GeospatialBucketCreator geospatialBucketCreator; @BeforeEach void setUp() { - bucketRepository = mock(BucketRepository.class); - geospatialBucketCreator = new GeospatialBucketCreator(bucketRepository, mock()); + geoRootPair = new BucketDescriptorPair(FRAGMENT_KEY_TILE, FRAGMENT_KEY_TILE_ROOT); + geoPair = new BucketDescriptorPair(FRAGMENT_KEY_TILE, "15/101/202"); + defaultPair = new BucketDescriptorPair(FRAGMENT_KEY_TILE, DEFAULT_BUCKET_STRING); } @Test - void when_TileFragmentDoesNotExist_NewTileFragmentIsCreatedAndSaved() { - Bucket bucket = new Bucket(BucketDescriptor.of(timebasedPair), VIEW_NAME); - Bucket rootBucket = bucket.createChild(geoRootPair); - Bucket childBucket = bucket.createChild(geoPair); - - when(bucketRepository.retrieveBucket(VIEW_NAME, childBucket.getBucketDescriptor())) - .thenReturn(Optional.empty()); - when(bucketRepository.insertBucket(any())).thenReturn(childBucket); + void test_createTileBucket() { + Bucket rootBucket = new Bucket(BucketDescriptor.of(timebasedPair), VIEW_NAME); + Bucket rootTileBucket = rootBucket.createChild(geoRootPair); + Bucket tileBucket = rootBucket.createChild(geoPair); + String tile = "15/101/202"; + when(tileBucketRelationsAttributer.addRelationsFromRootToBottom(rootTileBucket, tileBucket)) + .thenReturn(rootBucket.createChild(new BucketDescriptorPair(FRAGMENT_KEY_TILE, tile))); - Bucket returnedBucket = geospatialBucketCreator.getOrCreateTileBucket(bucket, "15/101/202", rootBucket); - - assertThat(returnedBucket) - .describedAs("Child instance must be the same, to assure the bucket instance from the db is returned") - .isSameAs(childBucket); - verify(bucketRepository).retrieveBucket(VIEW_NAME, childBucket.getBucketDescriptor()); - } - - @Test - void when_TileFragmentDoesNotExist_RetrievedTileFragmentIsReturned() { - Bucket bucket = new Bucket(BucketDescriptor.of(timebasedPair), VIEW_NAME); - Bucket rootBucket = bucket.createChild(geoRootPair); - Bucket tileBucket = bucket.createChild(geoPair); - - when(bucketRepository.retrieveBucket(VIEW_NAME, tileBucket.getBucketDescriptor())) - .thenReturn(Optional.of(tileBucket)); - - Bucket childBucket = geospatialBucketCreator.getOrCreateTileBucket(bucket, "15/101/202", rootBucket); + Bucket childBucket = geospatialBucketCreator.createTileBucket(rootBucket, "15/101/202", rootTileBucket); assertThat(childBucket.getBucketDescriptorAsString()).isEqualTo("year=2023&tile=15/101/202"); - verify(bucketRepository).retrieveBucket(VIEW_NAME, tileBucket.getBucketDescriptor()); - verifyNoMoreInteractions(bucketRepository); + verify(tileBucketRelationsAttributer).addRelationsFromRootToBottom(rootTileBucket, tileBucket); } @Test - void when_RootFragmentDoesNotExist_NewRootFragmentIsCreatedAndSaved() { + void test_CreateRootBucket() { Bucket bucket = new Bucket(BucketDescriptor.of(timebasedPair), VIEW_NAME); - Bucket rootBucket = bucket.createChild(geoRootPair); - when(bucketRepository.retrieveBucket(VIEW_NAME, rootBucket.getBucketDescriptor())).thenReturn(Optional.empty()); - when(bucketRepository.insertBucket(any())).thenReturn(rootBucket); - - Bucket returnedBucket = geospatialBucketCreator.getOrCreateRootBucket(bucket, FRAGMENT_KEY_TILE_ROOT); - assertThat(returnedBucket) - .describedAs("Child instance must be the same, to assure the bucket instance from the db is returned") - .isSameAs(rootBucket); - verify(bucketRepository).retrieveBucket(VIEW_NAME, rootBucket.getBucketDescriptor()); - } - - @Test - void when_RootFragmentDoesNotExist_RetrievedRootFragmentIsReturned() { - Bucket bucket = new Bucket(BucketDescriptor.of(timebasedPair), VIEW_NAME); - Bucket rootBucket = bucket.createChild(geoRootPair); - when(bucketRepository.retrieveBucket(VIEW_NAME, rootBucket.getBucketDescriptor())) - .thenReturn(Optional.of(rootBucket)); + Bucket returnedBucket = geospatialBucketCreator.createRootBucket(bucket, FRAGMENT_KEY_TILE_ROOT); - Bucket returnedBucket = geospatialBucketCreator.getOrCreateRootBucket(bucket, FRAGMENT_KEY_TILE_ROOT); assertThat(returnedBucket.getBucketDescriptorAsString()).isEqualTo("year=2023&tile=0/0/0"); - verify(bucketRepository).retrieveBucket(VIEW_NAME, rootBucket.getBucketDescriptor()); - verifyNoMoreInteractions(bucketRepository); + verifyNoInteractions(tileBucketRelationsAttributer); } @Test - void when_DefaultFragmentDoesNotExist_NewDefaultFragmentIsCreatedAndSaved() { - Bucket bucket = new Bucket(BucketDescriptor.of(timebasedPair), VIEW_NAME); - Bucket rootBucket = bucket.createChild(geoRootPair); - Bucket defaultBucket = bucket.createChild(defaultPair); - when(bucketRepository.retrieveBucket(VIEW_NAME, defaultBucket.getBucketDescriptor())).thenReturn(Optional.empty()); - when(bucketRepository.insertBucket(any())).thenReturn(defaultBucket); - - Bucket returnedBucket = geospatialBucketCreator.getOrCreateTileBucket(bucket, DEFAULT_BUCKET_STRING, rootBucket); - - assertThat(returnedBucket) - .describedAs("Child instance must be the same, to assure the bucket instance from the db is returned") - .isSameAs(defaultBucket); - assertThat(returnedBucket.getBucketDescriptor()).isEqualTo(defaultBucket.getBucketDescriptor()); - verify(bucketRepository).retrieveBucket(VIEW_NAME, defaultBucket.getBucketDescriptor()); - } - - @Test - void when_DefaultFragmentDoesNotExist_RetrievedDefaultFragmentIsReturned() { - Bucket bucket = new Bucket(BucketDescriptor.of(timebasedPair), VIEW_NAME); - Bucket rootBucket = bucket.createChild(geoRootPair); - Bucket defaultBucket = bucket.createChild(defaultPair); - when(bucketRepository.retrieveBucket(VIEW_NAME, defaultBucket.getBucketDescriptor())) - .thenReturn(Optional.of(defaultBucket)); + void test_GetOrCreateDefaultTileBucket() { + Bucket rootbucket = new Bucket(BucketDescriptor.of(timebasedPair), VIEW_NAME); + Bucket rootTileBucket = rootbucket.createChild(geoRootPair); + Bucket defaultBucket = rootbucket.createChild(defaultPair); + when(tileBucketRelationsAttributer.addRelationsFromRootToBottom(rootTileBucket, defaultBucket)).thenReturn(defaultBucket); - Bucket returnedBucket = geospatialBucketCreator.getOrCreateTileBucket(bucket, DEFAULT_BUCKET_STRING, rootBucket); + Bucket returnedBucket = geospatialBucketCreator.createTileBucket(rootbucket, DEFAULT_BUCKET_STRING, rootTileBucket); assertThat(returnedBucket.getBucketDescriptorAsString()).isEqualTo("year=2023&tile=unknown"); - verify(bucketRepository).retrieveBucket(VIEW_NAME, defaultBucket.getBucketDescriptor()); - verifyNoMoreInteractions(bucketRepository); + verify(tileBucketRelationsAttributer).addRelationsFromRootToBottom(rootTileBucket, defaultBucket); } } \ No newline at end of file diff --git a/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategy.java b/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategy.java index 10ec7ada6..ecc504aa6 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategy.java +++ b/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategy.java @@ -3,68 +3,55 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategy; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategyDecorator; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.FragmentationMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.bucketising.ReferenceBucketiser; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.fragmentation.ReferenceBucketCreator; import io.micrometer.observation.Observation; import io.micrometer.observation.ObservationRegistry; -import org.springframework.context.ApplicationEventPublisher; - -import java.util.Collection; -import java.util.List; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.fragmentation.ReferenceBucketCreator.FRAGMENT_KEY_REFERENCE_ROOT; public class ReferenceFragmentationStrategy extends FragmentationStrategyDecorator { - public static final String REFERENCE_FRAGMENTATION = "ReferenceFragmentation"; - - private final ReferenceBucketiser referenceBucketiser; - private final ReferenceBucketCreator bucketCreator; - private final ObservationRegistry observationRegistry; - - public ReferenceFragmentationStrategy(FragmentationStrategy fragmentationStrategy, - ReferenceBucketiser referenceBucketiser, - ReferenceBucketCreator bucketCreator, - ObservationRegistry observationRegistry, - ApplicationEventPublisher applicationEventPublisher) { - super(fragmentationStrategy, applicationEventPublisher); - this.referenceBucketiser = referenceBucketiser; - this.bucketCreator = bucketCreator; - this.observationRegistry = observationRegistry; - } - - @Override - public List addMemberToBucket(Bucket parentBucket, FragmentationMember member, - Observation parentObservation) { - final var fragmentationObservation = startObservation(parentObservation); - final var rootFragment = getOrCreateRootBucket(parentBucket); - var fragments = - referenceBucketiser - .bucketise(member.getSubject(), member.getVersionModel()) - .stream() - .map(reference -> bucketCreator.getOrCreateBucket(parentBucket, reference, rootFragment)) - .toList(); - - List members = fragments.parallelStream() - .map(bucket -> super.addMemberToBucket(bucket, member, fragmentationObservation)) - .flatMap(Collection::stream) - .toList(); - fragmentationObservation.stop(); - return members; - } - - private Observation startObservation(Observation parentObservation) { - return Observation.createNotStarted("reference fragmentation", observationRegistry) - .parentObservation(parentObservation) - .start(); - } - - private Bucket getOrCreateRootBucket(Bucket parentBucket) { - Bucket referenceRootFragment = bucketCreator.getOrCreateRootBucket(parentBucket, FRAGMENT_KEY_REFERENCE_ROOT); - super.addRelationFromParentToChild(parentBucket, referenceRootFragment); - return referenceRootFragment; - } + public static final String REFERENCE_FRAGMENTATION = "ReferenceFragmentation"; + + private final ReferenceBucketiser referenceBucketiser; + private final ReferenceBucketCreator bucketCreator; + private final ObservationRegistry observationRegistry; + + public ReferenceFragmentationStrategy(FragmentationStrategy fragmentationStrategy, + ReferenceBucketiser referenceBucketiser, + ReferenceBucketCreator bucketCreator, + ObservationRegistry observationRegistry) { + super(fragmentationStrategy); + this.referenceBucketiser = referenceBucketiser; + this.bucketCreator = bucketCreator; + this.observationRegistry = observationRegistry; + } + + @Override + public void addMemberToBucket(Bucket parentBucket, FragmentationMember member, Observation parentObservation) { + final var fragmentationObservation = startObservation(parentObservation); + final var rootBucket = getOrCreateRootBucket(parentBucket); + + referenceBucketiser + .createReferences(member.getSubject(), member.getVersionModel()) + .stream() + .map(reference -> bucketCreator.getOrCreateBucket(parentBucket, reference, rootBucket)) + .forEach(bucket -> super.addMemberToBucket(bucket, member, fragmentationObservation)); + + fragmentationObservation.stop(); + } + + private Observation startObservation(Observation parentObservation) { + return Observation.createNotStarted("reference fragmentation", observationRegistry) + .parentObservation(parentObservation) + .start(); + } + + private Bucket getOrCreateRootBucket(Bucket parentBucket) { + Bucket referenceRootFragment = bucketCreator.getOrCreateRootBucket(parentBucket, FRAGMENT_KEY_REFERENCE_ROOT); + return parentBucket.addChildBucket(referenceRootFragment.withGenericRelation()); + } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategyWrapper.java b/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategyWrapper.java index 523ecf66f..6bbae30ef 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategyWrapper.java +++ b/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategyWrapper.java @@ -3,7 +3,6 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ConfigProperties; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategy; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategyWrapper; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.bucketising.ReferenceBucketiser; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.config.ReferenceConfig; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.fragmentation.ReferenceBucketCreator; @@ -23,20 +22,19 @@ public class ReferenceFragmentationStrategyWrapper implements FragmentationStrat public FragmentationStrategy wrapFragmentationStrategy(ApplicationContext applicationContext, FragmentationStrategy fragmentationStrategy, ConfigProperties properties) { final var fragmentationPath = properties.getOrDefault(FRAGMENTATION_PATH, DEFAULT_FRAGMENTATION_PATH); - final var bucketRepository = applicationContext.getBean(BucketRepository.class); final var observationRegistry = applicationContext.getBean(ObservationRegistry.class); final var referenceConfig = new ReferenceConfig(fragmentationPath); final var referenceBucketiser = new ReferenceBucketiser(referenceConfig); final var fragmentationKey = properties.getOrDefault(FRAGMENTATION_KEY, DEFAULT_FRAGMENTATION_KEY); - final var relationsAttributer = new ReferenceFragmentRelationsAttributer(applicationContext, fragmentationPath, fragmentationKey); + final var relationsAttributer = new ReferenceFragmentRelationsAttributer(fragmentationPath, fragmentationKey); - final var referenceBucketCreator = new ReferenceBucketCreator(bucketRepository, relationsAttributer, fragmentationKey); + final var referenceBucketCreator = new ReferenceBucketCreator(relationsAttributer, fragmentationKey); return new ReferenceFragmentationStrategy( fragmentationStrategy, referenceBucketiser, referenceBucketCreator, - observationRegistry, - applicationContext); + observationRegistry + ); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/bucketising/ReferenceBucketiser.java b/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/bucketising/ReferenceBucketiser.java index f6aa98b99..6e0843c4a 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/bucketising/ReferenceBucketiser.java +++ b/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/bucketising/ReferenceBucketiser.java @@ -25,7 +25,7 @@ public ReferenceBucketiser(ReferenceConfig referenceConfig) { this.fragmentationPath = referenceConfig.fragmentationPath(); } - public Set bucketise(@NotNull String subject, @NotNull Model memberModel) { + public Set createReferences(@NotNull String subject, @NotNull Model memberModel) { try { Set references = memberModel .listObjectsOfProperty(createResource(subject), createProperty(fragmentationPath)) diff --git a/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/fragmentation/ReferenceBucketCreator.java b/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/fragmentation/ReferenceBucketCreator.java index 0d94e3243..e2d3da832 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/fragmentation/ReferenceBucketCreator.java +++ b/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/fragmentation/ReferenceBucketCreator.java @@ -1,55 +1,35 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.fragmentation; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.relations.ReferenceFragmentRelationsAttributer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.ServerConstants.DEFAULT_BUCKET_STRING; public class ReferenceBucketCreator { - private final String fragmentKeyReference; public static final String FRAGMENT_KEY_REFERENCE_ROOT = ""; - private static final Logger LOGGER = LoggerFactory.getLogger(ReferenceBucketCreator.class); - private final BucketRepository fragmentRepository; private final ReferenceFragmentRelationsAttributer relationsAttributer; - public ReferenceBucketCreator(BucketRepository fragmentRepository, - ReferenceFragmentRelationsAttributer relationsAttributer, + public ReferenceBucketCreator(ReferenceFragmentRelationsAttributer relationsAttributer, String fragmentKeyReference) { - this.fragmentRepository = fragmentRepository; this.relationsAttributer = relationsAttributer; this.fragmentKeyReference = fragmentKeyReference; } public Bucket getOrCreateBucket(Bucket parentBucket, String reference, Bucket rootBucket) { final BucketDescriptorPair childDescriptorPair = new BucketDescriptorPair(fragmentKeyReference, reference); - return fragmentRepository - .retrieveBucket(parentBucket.getViewName(), parentBucket.createChildDescriptor(childDescriptorPair)) - .orElseGet(() -> { - final Bucket childBucket = fragmentRepository.insertBucket(parentBucket.createChild(childDescriptorPair)); - if (reference.equals(DEFAULT_BUCKET_STRING)) { - relationsAttributer.addDefaultRelation(parentBucket, childBucket); - } else { - relationsAttributer.addRelationFromRootToBottom(rootBucket, childBucket); - } - LOGGER.debug("Reference fragment created with id: {}", childBucket.getBucketDescriptorAsString()); - return childBucket; - }); + final Bucket childBucket = parentBucket.createChild(childDescriptorPair); + if (reference.equals(DEFAULT_BUCKET_STRING)) { + return relationsAttributer.addDefaultRelation(parentBucket, childBucket); + } else { + return relationsAttributer.addRelationFromRootToBottom(rootBucket, childBucket); + } } public Bucket getOrCreateRootBucket(Bucket parentBucket, String reference) { final BucketDescriptorPair childDescriptorPair = new BucketDescriptorPair(fragmentKeyReference, reference); - return fragmentRepository - .retrieveBucket(parentBucket.getViewName(), parentBucket.createChildDescriptor(childDescriptorPair)) - .orElseGet(() -> { - final Bucket childBucket = fragmentRepository.insertBucket(parentBucket.createChild(childDescriptorPair)); - LOGGER.debug("Reference fragment created with id: {}", childBucket.getBucketDescriptorAsString()); - return childBucket; - }); + return parentBucket.createChild(childDescriptorPair); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/relations/ReferenceFragmentRelationsAttributer.java b/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/relations/ReferenceFragmentRelationsAttributer.java index 8fe48e310..4301a4b65 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/relations/ReferenceFragmentRelationsAttributer.java +++ b/ldes-fragmentisers/ldes-fragmentisers-reference/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/relations/ReferenceFragmentRelationsAttributer.java @@ -3,10 +3,8 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.exceptions.MissingFragmentValueException; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.relations.RelationsAttributer; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelationCreatedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.TreeRelation; import org.apache.jena.datatypes.xsd.XSDDatatype; -import org.springframework.context.ApplicationEventPublisher; import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.RdfConstants.TREE; @@ -14,34 +12,27 @@ public class ReferenceFragmentRelationsAttributer implements RelationsAttributer public static final String TREE_REFERENCE_EQUALS_RELATION = TREE + "EqualToRelation"; - private final ApplicationEventPublisher applicationEventPublisher; private final String fragmentationPath; private final String fragmentKeyReference; - public ReferenceFragmentRelationsAttributer(ApplicationEventPublisher applicationEventPublisher, - String fragmentationPath, - String fragmentKeyReference) { - this.applicationEventPublisher = applicationEventPublisher; + public ReferenceFragmentRelationsAttributer(String fragmentationPath, String fragmentKeyReference) { this.fragmentationPath = fragmentationPath; this.fragmentKeyReference = fragmentKeyReference; } - public void addRelationFromRootToBottom(Bucket rootBucket, Bucket referenceBucket) { - final BucketRelation bucketRelation = new BucketRelation( - rootBucket, - referenceBucket, + public Bucket addRelationFromRootToBottom(Bucket rootBucket, Bucket referenceBucket) { + final TreeRelation treeRelation = new TreeRelation( TREE_REFERENCE_EQUALS_RELATION, getTreeValue(referenceBucket), XSDDatatype.XSDanyURI.getURI(), fragmentationPath ); - applicationEventPublisher.publishEvent(new BucketRelationCreatedEvent(bucketRelation)); + return rootBucket.addChildBucket(referenceBucket.withRelations(treeRelation)); } - public void addDefaultRelation(Bucket rootBucket, Bucket referenceBucket) { - final BucketRelation defaultRelation = BucketRelation.createGenericRelation(rootBucket, referenceBucket); - applicationEventPublisher.publishEvent(new BucketRelationCreatedEvent(defaultRelation)); + public Bucket addDefaultRelation(Bucket rootBucket, Bucket referenceBucket) { + return rootBucket.addChildBucket(referenceBucket.withGenericRelation()); } private String getTreeValue(Bucket currentBucket) { diff --git a/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategyTest.java b/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategyTest.java index cfa6b07c0..af59ef9d7 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategyTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/ReferenceFragmentationStrategyTest.java @@ -24,8 +24,11 @@ class ReferenceFragmentationStrategyTest { private static final ViewName VIEW_NAME = new ViewName("collectionName", "view"); - private static final Bucket PARENT_BUCKET = new Bucket(BucketDescriptor.empty(), VIEW_NAME); - private static final Bucket ROOT_TILE_BUCKET = PARENT_BUCKET.createChild(new BucketDescriptorPair(DEFAULT_FRAGMENTATION_KEY, FRAGMENT_KEY_REFERENCE_ROOT)); + private static final String TYPE_PARCEL = "https://basisregisters.vlaanderen.be/implementatiemodel/gebouwenregister#Perceel"; + private static final String TYPE_BUILDING = "https://basisregisters.vlaanderen.be/implementatiemodel/gebouwenregister#Gebouw"; + private static final String TYPE_ADDRESS = "https://basisregisters.vlaanderen.be/implementatiemodel/gebouwenregister#Adres"; + private Bucket parentBucket; + private Bucket rootTileBucket; private ReferenceBucketiser referenceBucketiser; private ReferenceBucketCreator bucketCreator; @@ -34,60 +37,28 @@ class ReferenceFragmentationStrategyTest { @BeforeEach void setUp() { + parentBucket = new Bucket(BucketDescriptor.empty(), VIEW_NAME); + rootTileBucket = parentBucket.createChild(new BucketDescriptorPair(DEFAULT_FRAGMENTATION_KEY, FRAGMENT_KEY_REFERENCE_ROOT)); + referenceBucketiser = mock(ReferenceBucketiser.class); bucketCreator = mock(ReferenceBucketCreator.class); decoratedFragmentationStrategy = mock(FragmentationStrategy.class); - when(bucketCreator.getOrCreateRootBucket(PARENT_BUCKET, FRAGMENT_KEY_REFERENCE_ROOT)) - .thenReturn(ROOT_TILE_BUCKET); + when(bucketCreator.getOrCreateRootBucket(parentBucket, FRAGMENT_KEY_REFERENCE_ROOT)) + .thenReturn(rootTileBucket); referenceFragmentationStrategy = new ReferenceFragmentationStrategy(decoratedFragmentationStrategy, - referenceBucketiser, bucketCreator, ObservationRegistry.create(), mock()); - } - - @Test - void when_MemberIsAddedToFragment_ThenReferenceFragmentationIsApplied() { - FragmentationMember member = mock(FragmentationMember.class); - - final var typePerceel = "https://basisregisters.vlaanderen.be/implementatiemodel/gebouwenregister#Perceel"; - final var typeGebouw = "https://basisregisters.vlaanderen.be/implementatiemodel/gebouwenregister#Gebouw"; - final var typeAdres = "https://basisregisters.vlaanderen.be/implementatiemodel/gebouwenregister#Adres"; - - when(referenceBucketiser.bucketise(member.getSubject(), member.getVersionModel())) - .thenReturn(Set.of(typePerceel, typeGebouw, typeAdres)); - Bucket referenceBucketOne = mockCreationReferenceBucket(typePerceel); - Bucket referenceBucketTwo = mockCreationReferenceBucket(typeGebouw); - Bucket referenceBucketThree = mockCreationReferenceBucket(typeAdres); - - referenceFragmentationStrategy - .addMemberToBucket(PARENT_BUCKET, member, mock(Observation.class)); - - verify(decoratedFragmentationStrategy, - times(1)).addMemberToBucket(eq(referenceBucketOne), - any(), any(Observation.class)); - verify(decoratedFragmentationStrategy, - times(1)).addMemberToBucket(eq(referenceBucketTwo), - any(), any(Observation.class)); - verify(decoratedFragmentationStrategy, - times(1)).addMemberToBucket(eq(referenceBucketThree), - any(), any(Observation.class)); - verifyNoMoreInteractions(decoratedFragmentationStrategy); + referenceBucketiser, bucketCreator, ObservationRegistry.create()); } @Test void when_MemberIsAddedToBucket_ThenReferenceFragmentationIsApplied() { FragmentationMember member = mock(FragmentationMember.class); + when(referenceBucketiser.createReferences(member.getSubject(), member.getVersionModel())) + .thenReturn(Set.of(TYPE_PARCEL, TYPE_BUILDING, TYPE_ADDRESS)); + Bucket referenceBucketOne = mockCreationReferenceBucket(TYPE_PARCEL); + Bucket referenceBucketTwo = mockCreationReferenceBucket(TYPE_BUILDING); + Bucket referenceBucketThree = mockCreationReferenceBucket(TYPE_ADDRESS); - final var typeParcel = "https://basisregisters.vlaanderen.be/implementatiemodel/gebouwenregister#Perceel"; - final var typeBuilding = "https://basisregisters.vlaanderen.be/implementatiemodel/gebouwenregister#Gebouw"; - final var typeAddress = "https://basisregisters.vlaanderen.be/implementatiemodel/gebouwenregister#Adres"; - - when(referenceBucketiser.bucketise(member.getSubject(), member.getVersionModel())) - .thenReturn(Set.of(typeParcel, typeBuilding, typeAddress)); - Bucket referenceBucketOne = mockCreationReferenceBucket(typeParcel); - Bucket referenceBucketTwo = mockCreationReferenceBucket(typeBuilding); - Bucket referenceBucketThree = mockCreationReferenceBucket(typeAddress); - - referenceFragmentationStrategy - .addMemberToBucket(PARENT_BUCKET, member, mock(Observation.class)); + referenceFragmentationStrategy.addMemberToBucket(parentBucket, member, mock(Observation.class)); verify(decoratedFragmentationStrategy).addMemberToBucket(eq(referenceBucketOne), any(), any(Observation.class)); verify(decoratedFragmentationStrategy).addMemberToBucket(eq(referenceBucketTwo), any(), any(Observation.class)); @@ -96,8 +67,8 @@ void when_MemberIsAddedToBucket_ThenReferenceFragmentationIsApplied() { } private Bucket mockCreationReferenceBucket(String tile) { - Bucket referenceBucket = PARENT_BUCKET.createChild(new BucketDescriptorPair(DEFAULT_FRAGMENTATION_KEY, tile)); - when(bucketCreator.getOrCreateBucket(PARENT_BUCKET, tile, ROOT_TILE_BUCKET)) + Bucket referenceBucket = parentBucket.createChild(new BucketDescriptorPair(DEFAULT_FRAGMENTATION_KEY, tile)); + when(bucketCreator.getOrCreateBucket(parentBucket, tile, rootTileBucket)) .thenReturn(referenceBucket); return referenceBucket; } diff --git a/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/bucketising/ReferenceBucketiserTest.java b/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/bucketising/ReferenceBucketiserTest.java index 02f5295b3..9ac87a25f 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/bucketising/ReferenceBucketiserTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/bucketising/ReferenceBucketiserTest.java @@ -26,7 +26,7 @@ void setUp() { void shouldReturnSetOfFoundResources() { Model model = RDFParser.source("member-with-two-types.ttl").toModel(); - assertThat(referenceBucketiser.bucketise(memberId, model)) + assertThat(referenceBucketiser.createReferences(memberId, model)) .hasSize(2) .contains("https://basisregisters.vlaanderen.be/implementatiemodel/gebouwenregister#Perceel") .contains("https://basisregisters.vlaanderen.be/implementatiemodel/gebouwenregister#Gebouw"); @@ -36,7 +36,7 @@ void shouldReturnSetOfFoundResources() { void shouldReturnDefaultBucketString() { Model model = RDFParser.source("member-with-two-types.ttl").toModel(); - assertThat(referenceBucketiser.bucketise("faulty", model)) + assertThat(referenceBucketiser.createReferences("faulty", model)) .hasSize(1) .contains(DEFAULT_BUCKET_STRING); } @@ -45,7 +45,7 @@ void shouldReturnDefaultBucketString() { void when_MemberHasInvalidURI_Then_ReturnOnlyCorrectBucket() { Model model = RDFParser.source("member-with-two-types-faulty.ttl").toModel(); - assertThat(referenceBucketiser.bucketise(memberId, model)) + assertThat(referenceBucketiser.createReferences(memberId, model)) .hasSize(1) .contains("https://basisregisters.vlaanderen.be/implementatiemodel/gebouwenregister#Perceel"); } @@ -54,7 +54,7 @@ void when_MemberHasInvalidURI_Then_ReturnOnlyCorrectBucket() { void shouldSkipResultsThatAreNotUris() { Model model = RDFParser.source("member-with-string-type.ttl").toModel(); - assertThat(referenceBucketiser.bucketise(memberId, model)).hasSize(1) + assertThat(referenceBucketiser.createReferences(memberId, model)).hasSize(1) .contains(DEFAULT_BUCKET_STRING); } diff --git a/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/fragmentation/ReferenceBucketCreatorTest.java b/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/fragmentation/ReferenceBucketCreatorTest.java index 75fa16ae5..405e5650c 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/fragmentation/ReferenceBucketCreatorTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/fragmentation/ReferenceBucketCreatorTest.java @@ -2,7 +2,6 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.relations.ReferenceFragmentRelationsAttributer; @@ -13,8 +12,6 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import java.util.Optional; - import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.ServerConstants.DEFAULT_BUCKET_STRING; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.ReferenceFragmentationStrategyWrapper.DEFAULT_FRAGMENTATION_KEY; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.fragmentation.ReferenceBucketCreator.FRAGMENT_KEY_REFERENCE_ROOT; @@ -28,38 +25,13 @@ class ReferenceBucketCreatorTest { private static final BucketDescriptorPair referenceRootPair = new BucketDescriptorPair(DEFAULT_FRAGMENTATION_KEY, FRAGMENT_KEY_REFERENCE_ROOT); private static final BucketDescriptorPair referencePair = new BucketDescriptorPair(DEFAULT_FRAGMENTATION_KEY, RDF.type.getURI()); private static final BucketDescriptorPair defaultPair = new BucketDescriptorPair(DEFAULT_FRAGMENTATION_KEY, DEFAULT_BUCKET_STRING); - private ReferenceBucketCreator referenceBucketCreator; - - @Mock - private BucketRepository bucketRepository; - @Mock private ReferenceFragmentRelationsAttributer relationsAttributer; @BeforeEach void setUp() { - referenceBucketCreator = - new ReferenceBucketCreator(bucketRepository, relationsAttributer, DEFAULT_FRAGMENTATION_KEY); - } - - @Test - void when_ReferenceFragmentDoesNotExist_NewReferenceFragmentIsCreatedAndSaved() { - Bucket bucket = new Bucket(BucketDescriptor.of(timebasedPair), viewName); - Bucket rootBucket = bucket.createChild(referenceRootPair); - Bucket childBucket = bucket.createChild(referencePair); - - when(bucketRepository.retrieveBucket(viewName, childBucket.getBucketDescriptor())).thenReturn(Optional.empty()); - when(bucketRepository.insertBucket(any())).thenReturn(childBucket); - - Bucket returnedBucket = referenceBucketCreator.getOrCreateBucket(bucket, RDF.type.getURI(), rootBucket); - - assertThat(returnedBucket) - .describedAs("Child instance must be the same, to assure the bucket instance from the db is returned") - .isSameAs(childBucket); - assertThat(returnedBucket.getBucketDescriptorAsString()).isEqualTo("year=2023&reference=%s", RDF.type.getURI()); - verify(bucketRepository).retrieveBucket(viewName, childBucket.getBucketDescriptor()); - verify(bucketRepository).insertBucket(returnedBucket); + referenceBucketCreator = new ReferenceBucketCreator(relationsAttributer, DEFAULT_FRAGMENTATION_KEY); } @Test @@ -67,62 +39,33 @@ void when_ReferenceFragmentDoesExist_RetrievedReferenceFragmentIsReturned() { Bucket bucket = new Bucket(BucketDescriptor.of(timebasedPair), viewName); Bucket rootBucket = bucket.createChild(referenceRootPair); Bucket referenceBucket = bucket.createChild(referencePair); - - when(bucketRepository.retrieveBucket(viewName, referenceBucket.getBucketDescriptor())).thenReturn(Optional.of(referenceBucket)); + when(relationsAttributer.addRelationFromRootToBottom(rootBucket, bucket.createChild(referencePair))).thenReturn(referenceBucket); Bucket childBucket = referenceBucketCreator.getOrCreateBucket(bucket, RDF.type.getURI(), rootBucket); assertThat(childBucket.getBucketDescriptorAsString()).isEqualTo("year=2023&reference=%s", RDF.type.getURI()); - verify(bucketRepository).retrieveBucket(viewName, referenceBucket.getBucketDescriptor()); - verifyNoMoreInteractions(bucketRepository); + verify(relationsAttributer).addRelationFromRootToBottom(rootBucket, referenceBucket); } @Test void when_RootFragmentDoesNotExist_NewRootFragmentIsCreatedAndSaved() { Bucket bucket = new Bucket(BucketDescriptor.of(timebasedPair), viewName); - Bucket rootBucket = bucket.createChild(referenceRootPair); - - when(bucketRepository.retrieveBucket(viewName, rootBucket.getBucketDescriptor())).thenReturn(Optional.empty()); - when(bucketRepository.insertBucket(any())).thenReturn(rootBucket); Bucket returnedBucket = referenceBucketCreator.getOrCreateRootBucket(bucket, FRAGMENT_KEY_REFERENCE_ROOT); - assertThat(returnedBucket) - .describedAs("Child instance must be the same, to assure the bucket instance from the db is returned") - .isSameAs(rootBucket); - verify(bucketRepository).retrieveBucket(viewName, rootBucket.getBucketDescriptor()); - verify(bucketRepository).insertBucket(returnedBucket); + assertThat(returnedBucket.getBucketDescriptorAsString()).isEqualTo("year=2023&reference="); + verifyNoInteractions(relationsAttributer); } @Test void when_RootFragmentDoesNotExist_RetrievedRootFragmentIsReturned() { Bucket bucket = new Bucket(BucketDescriptor.of(timebasedPair), viewName); Bucket rootBucket = bucket.createChild(referenceRootPair); - when(bucketRepository.retrieveBucket(viewName, rootBucket.getBucketDescriptor())).thenReturn(Optional.of(rootBucket)); + when(relationsAttributer.addRelationFromRootToBottom(rootBucket, bucket.createChild(referenceRootPair))).thenReturn(bucket.createChild(referenceRootPair)); Bucket returnedBucket = referenceBucketCreator.getOrCreateBucket(bucket, FRAGMENT_KEY_REFERENCE_ROOT, rootBucket); assertThat(returnedBucket.getBucketDescriptorAsString()).isEqualTo("year=2023&reference="); - verify(bucketRepository).retrieveBucket(viewName, rootBucket.getBucketDescriptor()); - verifyNoMoreInteractions(bucketRepository); - } - - @Test - void when_DefaultFragmentDoesNotExist_DefaultFragmentIsCreatedAndSaved() { - Bucket bucket = new Bucket(BucketDescriptor.of(timebasedPair), viewName); - Bucket rootBucket = bucket.createChild(referenceRootPair); - Bucket defaultBucket = bucket.createChild(defaultPair); - - when(bucketRepository.retrieveBucket(viewName, defaultBucket.getBucketDescriptor())).thenReturn(Optional.empty()); - when(bucketRepository.insertBucket(any())).thenReturn(defaultBucket); - - Bucket childBucket = referenceBucketCreator.getOrCreateBucket(bucket, DEFAULT_BUCKET_STRING, rootBucket); - - assertThat(childBucket) - .describedAs("Child instance must be the same, to assure the bucket instance from the db is returned") - .isSameAs(defaultBucket); - verify(bucketRepository).retrieveBucket(viewName, defaultBucket.getBucketDescriptor()); - verify(bucketRepository).insertBucket(childBucket); } @Test @@ -130,14 +73,12 @@ void when_DefaultFragmentDoesExist_RetrievedDefaultFragmentIsReturned() { Bucket bucket = new Bucket(BucketDescriptor.of(timebasedPair), viewName); Bucket rootBucket = bucket.createChild(referenceRootPair); Bucket defaultBucket = bucket.createChild(defaultPair); - - when(bucketRepository.retrieveBucket(viewName, defaultBucket.getBucketDescriptor())).thenReturn(Optional.of(defaultBucket)); + when(relationsAttributer.addDefaultRelation(bucket, bucket.createChild(defaultPair))).thenReturn(defaultBucket); Bucket childBucket = referenceBucketCreator.getOrCreateBucket(bucket, DEFAULT_BUCKET_STRING, rootBucket); assertThat(childBucket.getBucketDescriptorAsString()).isEqualTo("year=2023&reference=unknown"); - verify(bucketRepository).retrieveBucket(viewName, defaultBucket.getBucketDescriptor()); - verifyNoMoreInteractions(bucketRepository); + verify(relationsAttributer).addDefaultRelation(bucket, defaultBucket); } } \ No newline at end of file diff --git a/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/relations/ReferenceFragmentRelationsAttributerTest.java b/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/relations/ReferenceFragmentRelationsAttributerTest.java index a18a9cf6f..42c5a8ab5 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/relations/ReferenceFragmentRelationsAttributerTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-reference/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/reference/relations/ReferenceFragmentRelationsAttributerTest.java @@ -5,33 +5,27 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.exceptions.MissingFragmentValueException; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelationCreatedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.TreeRelation; import org.apache.jena.datatypes.xsd.XSDDatatype; import org.apache.jena.vocabulary.RDF; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.context.ApplicationEventPublisher; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.ReferenceFragmentationStrategyWrapper.DEFAULT_FRAGMENTATION_KEY; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.reference.relations.ReferenceFragmentRelationsAttributer.TREE_REFERENCE_EQUALS_RELATION; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; -import static org.mockito.Mockito.verify; @ExtendWith(MockitoExtension.class) class ReferenceFragmentRelationsAttributerTest { - private static final String fragmentReference = + private static final String FRAGMENT_REFERENCE = "https://basisregisters.vlaanderen.be/implementatiemodel/gebouwenregister#Perceel"; private static final ViewName viewName = new ViewName("collectionName", "view"); private static final Bucket parentBucket = new Bucket(BucketDescriptor.empty(), viewName); - @Mock - private ApplicationEventPublisher applicationEventPublisher; - private final String fragmentationPath = RDF.type.getURI(); private ReferenceFragmentRelationsAttributer relationsAttributer; @@ -39,17 +33,14 @@ class ReferenceFragmentRelationsAttributerTest { @BeforeEach void setUp() { relationsAttributer = - new ReferenceFragmentRelationsAttributer(applicationEventPublisher, fragmentationPath, DEFAULT_FRAGMENTATION_KEY); + new ReferenceFragmentRelationsAttributer(fragmentationPath, DEFAULT_FRAGMENTATION_KEY); } @Test void when_ReferenceFragmentsAreCreated_RelationsBetweenRootAndCreatedFragmentsAreAdded() { Bucket rootBucket = createReferenceBuket(""); - Bucket referenceBucket = createReferenceBuket(fragmentReference); - - BucketRelation expectedRelation = new BucketRelation( - rootBucket, - referenceBucket, + Bucket referenceBucket = createReferenceBuket(FRAGMENT_REFERENCE); + TreeRelation expectedRelation = new TreeRelation( TREE_REFERENCE_EQUALS_RELATION, "https://basisregisters.vlaanderen.be/implementatiemodel/gebouwenregister#Perceel", XSDDatatype.XSDanyURI.getURI(), @@ -58,13 +49,27 @@ void when_ReferenceFragmentsAreCreated_RelationsBetweenRootAndCreatedFragmentsAr relationsAttributer.addRelationFromRootToBottom(rootBucket, referenceBucket); - verify(applicationEventPublisher).publishEvent(new BucketRelationCreatedEvent(expectedRelation)); + assertThat(rootBucket.getChildren()) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyInAnyOrder(referenceBucket.withRelations(expectedRelation)); + } + + @Test + void when_UnknownReferenceFragmentsAreCreated_RelationsBetweenRootAndCreatedFragmentsAreAdded() { + Bucket rootBucket = createReferenceBuket(""); + Bucket referenceBucket = createReferenceBuket("unknown"); + + relationsAttributer.addDefaultRelation(rootBucket, referenceBucket); + + assertThat(rootBucket.getChildren()) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyInAnyOrder(referenceBucket.withGenericRelation()); } @Test void when_ReferenceFragmentHasNoReference_ThenAnExceptionIsThrown() { Bucket rootBucket = createReferenceBuket(""); - Bucket referenceBucket = parentBucket.createChild(new BucketDescriptorPair("invalid-key", fragmentReference)); + Bucket referenceBucket = parentBucket.createChild(new BucketDescriptorPair("invalid-key", FRAGMENT_REFERENCE)); assertThatExceptionOfType(MissingFragmentValueException.class) .isThrownBy(() -> relationsAttributer.addRelationFromRootToBottom(rootBucket, referenceBucket)) diff --git a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategy.java b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategy.java index 8cc5d40f4..255490f20 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategy.java +++ b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategy.java @@ -4,7 +4,6 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategy; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategyDecorator; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.FragmentationMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.config.TimeBasedConfig; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.constants.Granularity; @@ -15,10 +14,8 @@ import org.apache.jena.rdf.model.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.context.ApplicationEventPublisher; import java.time.LocalDateTime; -import java.util.List; import java.util.Optional; public class HierarchicalTimeBasedFragmentationStrategy extends FragmentationStrategyDecorator { @@ -33,29 +30,27 @@ public class HierarchicalTimeBasedFragmentationStrategy extends FragmentationStr public HierarchicalTimeBasedFragmentationStrategy(FragmentationStrategy fragmentationStrategy, ObservationRegistry observationRegistry, TimeBasedBucketFinder bucketFinder, - ApplicationEventPublisher applicationEventPublisher, TimeBasedConfig config) { - super(fragmentationStrategy, applicationEventPublisher); + super(fragmentationStrategy); this.observationRegistry = observationRegistry; this.bucketFinder = bucketFinder; this.config = config; } @Override - public List addMemberToBucket(Bucket parentBucket, FragmentationMember member, Observation parentObservation) { + public void addMemberToBucket(Bucket parentBucket, FragmentationMember member, Observation parentObservation) { final Observation bucketisationObservation = startFragmentationObservation(parentObservation); - Bucket bucket = getFragmentationTimestamp(member.getSubject(), member.getVersionModel()) + Bucket childBucket = getFragmentationTimestamp(member.getSubject(), member.getVersionModel()) .map(timestamp -> bucketFinder.getLowestBucket(parentBucket, timestamp, Granularity.YEAR)) .orElseGet(() -> bucketFinder.getDefaultFragment(parentBucket)); - List members = super.addMemberToBucket(bucket, member, parentObservation); + super.addMemberToBucket(childBucket, member, parentObservation); bucketisationObservation.stop(); - return members; } private Optional getFragmentationTimestamp(String subject, Model memberModel) { - try{ + try { Optional timeStamp = getFragmentationObjectLocalDateTime(memberModel, config.getFragmenterSubjectFilter(), config.getFragmentationPath()); diff --git a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategyWrapper.java b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategyWrapper.java index e7fef9c7c..5b95f4338 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategyWrapper.java +++ b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategyWrapper.java @@ -3,7 +3,6 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ConfigProperties; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategy; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategyWrapper; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.config.TimeBasedConfig; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.constants.Granularity; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.services.TimeBasedBucketCreator; @@ -18,22 +17,18 @@ public class HierarchicalTimeBasedFragmentationStrategyWrapper implements Fragme public FragmentationStrategy wrapFragmentationStrategy(ApplicationContext applicationContext, FragmentationStrategy fragmentationStrategy, ConfigProperties fragmentationProperties) { - BucketRepository bucketRepository = applicationContext.getBean(BucketRepository.class); ObservationRegistry observationRegistry = applicationContext.getBean(ObservationRegistry.class); TimeBasedConfig config = createConfig(fragmentationProperties); - TimeBasedRelationsAttributer relationsAttributer = new TimeBasedRelationsAttributer( - applicationContext, config); - TimeBasedBucketCreator bucketCreator = new TimeBasedBucketCreator(bucketRepository, relationsAttributer); + TimeBasedRelationsAttributer relationsAttributer = new TimeBasedRelationsAttributer(config); + TimeBasedBucketCreator bucketCreator = new TimeBasedBucketCreator(relationsAttributer); TimeBasedBucketFinder bucketFinder = new TimeBasedBucketFinder(bucketCreator, config); - return new HierarchicalTimeBasedFragmentationStrategy(fragmentationStrategy, - observationRegistry, bucketFinder, applicationContext, config); + return new HierarchicalTimeBasedFragmentationStrategy(fragmentationStrategy, observationRegistry, bucketFinder, config); } private TimeBasedConfig createConfig(ConfigProperties properties) { return new TimeBasedConfig(properties.getOrDefault(FRAGMENTATION_SUBJECT_FILTER, ".*"), - properties.get(FRAGMENTATION_PATH), Granularity.from(properties.get(MAX_GRANULARITY)), - Boolean.parseBoolean(properties.getOrDefault(LINEAR_TIME_CACHING_ENABLED, "false"))); + properties.get(FRAGMENTATION_PATH), Granularity.from(properties.get(MAX_GRANULARITY))); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/config/TimeBasedConfig.java b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/config/TimeBasedConfig.java index 4555029e3..f8bf59e5e 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/config/TimeBasedConfig.java +++ b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/config/TimeBasedConfig.java @@ -5,15 +5,12 @@ public class TimeBasedConfig { private final String fragmenterSubjectFilter; private final String fragmentationPath; - private final boolean linearTimeCachingEnabled; private final Granularity maxGranularity; - public TimeBasedConfig(String fragmenterSubjectFilter, String fragmentationPath, Granularity maxGranularity, - boolean linearTimeCachingEnabled) { + public TimeBasedConfig(String fragmenterSubjectFilter, String fragmentationPath, Granularity maxGranularity) { this.fragmenterSubjectFilter = fragmenterSubjectFilter; this.fragmentationPath = fragmentationPath; this.maxGranularity = maxGranularity; - this.linearTimeCachingEnabled = linearTimeCachingEnabled; } public String getFragmentationPath() { @@ -27,9 +24,4 @@ public String getFragmenterSubjectFilter() { public Granularity getMaxGranularity() { return maxGranularity; } - - public boolean isLinearTimeCachingEnabled() { - return linearTimeCachingEnabled; - } - } diff --git a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketCreator.java b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketCreator.java index 3aa07d168..32a53a177 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketCreator.java +++ b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketCreator.java @@ -1,7 +1,6 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.services; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.constants.Granularity; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.model.FragmentationTimestamp; @@ -16,38 +15,31 @@ import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.HierarchicalTimeBasedFragmentationStrategy.TIMEBASED_FRAGMENTATION_HIERARCHICAL; public class TimeBasedBucketCreator { - private final BucketRepository bucketRepository; private final TimeBasedRelationsAttributer relationsAttributer; private static final Logger LOGGER = LoggerFactory.getLogger(TimeBasedBucketCreator.class); - public TimeBasedBucketCreator(BucketRepository bucketRepository, TimeBasedRelationsAttributer relationsAttributer) { - this.bucketRepository = bucketRepository; + public TimeBasedBucketCreator(TimeBasedRelationsAttributer relationsAttributer) { this.relationsAttributer = relationsAttributer; } - public Bucket getOrCreateBucket(Bucket parentFragment, - FragmentationTimestamp fragmentationTimestamp, - Granularity granularity) { - return getOrCreateBucket(parentFragment, fragmentationTimestamp.getTimeValueForGranularity(granularity), granularity); + public Bucket createBucket(Bucket parentBucket, + FragmentationTimestamp fragmentationTimestamp, + Granularity granularity) { + return createBucket(parentBucket, fragmentationTimestamp.getTimeValueForGranularity(granularity), granularity); } - public Bucket getOrCreateBucket(Bucket parentBucket, String timeValue, Granularity granularity) { + public Bucket createBucket(Bucket parentBucket, String timeValue, Granularity granularity) { final BucketDescriptorPair childDescriptorPair = new BucketDescriptorPair(granularity.getValue(), timeValue); - return bucketRepository - .retrieveBucket(parentBucket.getViewName(), parentBucket.createChildDescriptor(childDescriptorPair)) - .orElseGet(() -> { - final Bucket childBucket = bucketRepository.insertBucket(parentBucket.createChild(childDescriptorPair)); - addRelationToParent(parentBucket, childBucket); - logBucketisation(parentBucket, childBucket); - return childBucket; - }); + final Bucket childBucket = parentBucket.createChild(childDescriptorPair); + logBucketisation(parentBucket, childBucket); + return addRelationToParent(parentBucket, childBucket); } - private void addRelationToParent(Bucket parentBucket, Bucket childBucket) { - if(isDefaultBucket(childBucket)) { - relationsAttributer.addDefaultRelation(parentBucket, childBucket); + private Bucket addRelationToParent(Bucket parentBucket, Bucket childBucket) { + if (isDefaultBucket(childBucket)) { + return relationsAttributer.addDefaultRelation(parentBucket, childBucket); } else { - relationsAttributer.addInBetweenRelation(parentBucket, childBucket); + return relationsAttributer.addInBetweenRelation(parentBucket, childBucket); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketFinder.java b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketFinder.java index d0f48a43f..f20eacdea 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketFinder.java +++ b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketFinder.java @@ -22,16 +22,15 @@ public Bucket getLowestBucket(Bucket parentFragment, FragmentationTimestamp frag return parentFragment; } return getLowestBucket( - bucketCreator.getOrCreateBucket(parentFragment, fragmentationTimestamp, granularity), + bucketCreator.createBucket(parentFragment, fragmentationTimestamp, granularity), fragmentationTimestamp, granularity.getChild()); } public Bucket getDefaultFragment(Bucket rootFragment) { - return bucketCreator.getOrCreateBucket(rootFragment, DEFAULT_BUCKET_STRING, Granularity.YEAR); + return bucketCreator.createBucket(rootFragment, DEFAULT_BUCKET_STRING, Granularity.YEAR); } private boolean isLowest(Bucket bucket) { - return bucket.getBucketDescriptorPairs().stream() - .anyMatch(descriptorPair -> descriptorPair.key().equals(config.getMaxGranularity().getValue())); + return bucket.getValueForKey(config.getMaxGranularity().getValue()).isPresent(); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedRelationsAttributer.java b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedRelationsAttributer.java index 61c0dab5c..3e1d2cb2d 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedRelationsAttributer.java +++ b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedRelationsAttributer.java @@ -1,85 +1,60 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.services; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.fragmentation.TimeBasedLinearCachingTriggered; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.relations.RelationsAttributer; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelationCreatedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.TreeRelation; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.config.TimeBasedConfig; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.constants.Granularity; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.model.FragmentationTimestamp; import org.jetbrains.annotations.NotNull; -import org.springframework.context.ApplicationEventPublisher; import java.time.LocalDateTime; import java.util.Arrays; import java.util.Map; +import java.util.Optional; +import java.util.function.Function; import java.util.stream.Collectors; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.constants.TimeBasedConstants.*; public class TimeBasedRelationsAttributer implements RelationsAttributer { - private final ApplicationEventPublisher applicationEventPublisher; - private final TimeBasedConfig config; - public TimeBasedRelationsAttributer(ApplicationEventPublisher applicationEventPublisher, TimeBasedConfig config) { - this.applicationEventPublisher = applicationEventPublisher; + public TimeBasedRelationsAttributer(TimeBasedConfig config) { this.config = config; } - public void addInBetweenRelation(Bucket parentBucket, Bucket childBucket) { + public Bucket addInBetweenRelation(Bucket parentBucket, Bucket childBucket) { FragmentationTimestamp timestamp = timestampFromFragmentPairs(childBucket); - BucketRelation parentGteRelation = new BucketRelation( - parentBucket, - childBucket, - TREE_GTE_RELATION, - timestamp.getTime().toString(), - XSD_DATETIME, - config.getFragmentationPath() - ); - BucketRelation parentLtRelation = new BucketRelation( - parentBucket, - childBucket, - TREE_LT_RELATION, - timestamp.getLtBoundary().toString(), - XSD_DATETIME, - config.getFragmentationPath() - ); - saveRelation(parentGteRelation, timestamp.getNextUpdateTs()); - saveRelation(parentLtRelation, timestamp.getNextUpdateTs()); - } - - public void addDefaultRelation(Bucket parentBucket, Bucket childBucket) { - final BucketRelation defaultRelation = BucketRelation.createGenericRelation(parentBucket, childBucket); - saveRelation(defaultRelation, null); + return parentBucket.addChildBucket(childBucket.withRelations(createTimeBasedRelations(timestamp))); } - private void saveRelation(BucketRelation bucketRelation, LocalDateTime nextUpdateTs) { - if (config.isLinearTimeCachingEnabled()) { - applicationEventPublisher.publishEvent(new TimeBasedLinearCachingTriggered(bucketRelation.fromBucket().getBucketId(), nextUpdateTs)); - } - applicationEventPublisher.publishEvent(new BucketRelationCreatedEvent(bucketRelation)); - + public Bucket addDefaultRelation(Bucket parentBucket, Bucket childBucket) { + return parentBucket.addChildBucket(childBucket.withGenericRelation()); } private FragmentationTimestamp timestampFromFragmentPairs(Bucket bucket) { - Map timeMap = bucket.getBucketDescriptorPairs().stream() - .filter(descriptorPair -> Arrays.stream(Granularity.values()).map(Granularity::getValue) - .anyMatch(t -> t.equals(descriptorPair.key()))) - .collect(Collectors.toMap(BucketDescriptorPair::key, pair -> Integer.parseInt(pair.value()))); + Map> timeMap = Arrays.stream(Granularity.values()) + .map(Granularity::getValue) + .collect(Collectors.toMap(Function.identity(), key -> bucket.getValueForKey(key).map(Integer::parseInt))); return createTimestampFromMap(timeMap); } - private static @NotNull FragmentationTimestamp createTimestampFromMap(Map timeMap) { - LocalDateTime time = LocalDateTime.of(timeMap.getOrDefault(Granularity.YEAR.getValue(), 0), - timeMap.getOrDefault(Granularity.MONTH.getValue(), 1), - timeMap.getOrDefault(Granularity.DAY.getValue(), 1), - timeMap.getOrDefault(Granularity.HOUR.getValue(), 0), - timeMap.getOrDefault(Granularity.MINUTE.getValue(), 0), - timeMap.getOrDefault(Granularity.SECOND.getValue(), 0)); - return new FragmentationTimestamp(time, Granularity.fromIndex(timeMap.size() - 1)); + private TreeRelation[] createTimeBasedRelations(FragmentationTimestamp timestamp) { + return new TreeRelation[]{ + new TreeRelation(TREE_GTE_RELATION, timestamp.getTime().toString(), XSD_DATETIME, config.getFragmentationPath()), + new TreeRelation(TREE_LT_RELATION, timestamp.getLtBoundary().toString(), XSD_DATETIME, config.getFragmentationPath()), + }; + } + + private static @NotNull FragmentationTimestamp createTimestampFromMap(Map> timeMap) { + LocalDateTime time = LocalDateTime.of(timeMap.get(Granularity.YEAR.getValue()).orElse(0), + timeMap.get(Granularity.MONTH.getValue()).orElse(1), + timeMap.get(Granularity.DAY.getValue()).orElse(1), + timeMap.get(Granularity.HOUR.getValue()).orElse(0), + timeMap.get(Granularity.MINUTE.getValue()).orElse(0), + timeMap.get(Granularity.SECOND.getValue()).orElse(0)); + return new FragmentationTimestamp(time, Granularity.fromIndex((int) (timeMap.values().stream().filter(Optional::isPresent).count() - 1))); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategyTest.java b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategyTest.java index 9bfa454da..466855288 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategyTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/HierarchicalTimeBasedFragmentationStrategyTest.java @@ -30,37 +30,24 @@ class HierarchicalTimeBasedFragmentationStrategyTest { private static final ViewName VIEW_NAME = new ViewName("collectionName", "view"); - private static final Bucket PARENT_BUCKET = new Bucket(BucketDescriptor.empty(), VIEW_NAME); - private static final Bucket CHILD_BUCKET = new Bucket(new BucketDescriptor(List.of(new BucketDescriptorPair("is", "child"))), VIEW_NAME); private static final LocalDateTime TIME = LocalDateTime.of(2023, 1, 1, 0, 0, 0); private static final Granularity GRANULARITY = Granularity.SECOND; private static final EventStreamProperties EVENT_STREAM_PROPERTIES = new EventStreamProperties("collectionName", "versionOf", "timestampPath", false); private HierarchicalTimeBasedFragmentationStrategy fragmentationStrategy; private FragmentationStrategy decoratedFragmentationStrategy; private TimeBasedBucketFinder bucketFinder; + private Bucket parentBucket; + private Bucket childBucket; @BeforeEach void setUp() { - TimeBasedConfig config = new TimeBasedConfig(".*", "http://purl.org/dc/terms/created", GRANULARITY, false); + TimeBasedConfig config = new TimeBasedConfig(".*", "http://purl.org/dc/terms/created", GRANULARITY); bucketFinder = mock(TimeBasedBucketFinder.class); decoratedFragmentationStrategy = mock(FragmentationStrategy.class); fragmentationStrategy = new HierarchicalTimeBasedFragmentationStrategy(decoratedFragmentationStrategy, - ObservationRegistry.create(), bucketFinder, mock(), config); - } - - @Test - void when_FragmentationCalled_Then_FunctionsAreCalled() { - Model model = loadModel("member_with_created_property.nq"); - FragmentationMember fragmentationMember = new FragmentationMember(1, "subject", "versionOf", TIME, EVENT_STREAM_PROPERTIES, model); - FragmentationTimestamp fragmentationTimestamp = new FragmentationTimestamp(TIME, GRANULARITY); - when(bucketFinder.getLowestBucket(PARENT_BUCKET, fragmentationTimestamp, Granularity.YEAR)) - .thenReturn(CHILD_BUCKET); - - fragmentationStrategy.addMemberToBucket(PARENT_BUCKET, fragmentationMember, mock()); - - InOrder inOrder = Mockito.inOrder(bucketFinder, decoratedFragmentationStrategy); - inOrder.verify(bucketFinder).getLowestBucket(PARENT_BUCKET, fragmentationTimestamp, Granularity.YEAR); - inOrder.verify(decoratedFragmentationStrategy).addMemberToBucket(eq(CHILD_BUCKET), any(), any()); + ObservationRegistry.create(), bucketFinder, config); + parentBucket = new Bucket(BucketDescriptor.empty(), VIEW_NAME); + childBucket = new Bucket(new BucketDescriptor(List.of(new BucketDescriptorPair("is", "child"))), VIEW_NAME); } @Test @@ -68,44 +55,26 @@ void when_BucketisationCalled_Then_FunctionsAreCalled() { Model model = loadModel("member_with_created_property.nq"); FragmentationMember member = new FragmentationMember(1, "subject", "versionOf", TIME, EVENT_STREAM_PROPERTIES, model); FragmentationTimestamp fragmentationTimestamp = new FragmentationTimestamp(TIME, GRANULARITY); - when(bucketFinder.getLowestBucket(PARENT_BUCKET, fragmentationTimestamp, Granularity.YEAR)) - .thenReturn(CHILD_BUCKET); - - fragmentationStrategy.addMemberToBucket(PARENT_BUCKET, member, mock(Observation.class)); - - InOrder inOrder = Mockito.inOrder(bucketFinder, decoratedFragmentationStrategy); - inOrder.verify(bucketFinder).getLowestBucket(PARENT_BUCKET, fragmentationTimestamp, Granularity.YEAR); - inOrder.verify(decoratedFragmentationStrategy, - times(1)).addMemberToBucket(eq(CHILD_BUCKET), any(), - any(Observation.class)); - } - - @Test - void when_FragmentationCalledForMemberWithMissingTimestamp_Then_FunctionsAreCalled() { - FragmentationMember member = mock(FragmentationMember.class); - when(bucketFinder.getDefaultFragment(PARENT_BUCKET)).thenReturn(CHILD_BUCKET); + when(bucketFinder.getLowestBucket(parentBucket, fragmentationTimestamp, Granularity.YEAR)) + .thenReturn(childBucket); - fragmentationStrategy.addMemberToBucket(PARENT_BUCKET, member, mock()); + fragmentationStrategy.addMemberToBucket(parentBucket, member, mock(Observation.class)); InOrder inOrder = Mockito.inOrder(bucketFinder, decoratedFragmentationStrategy); - inOrder.verify(bucketFinder).getDefaultFragment(PARENT_BUCKET); - inOrder.verify(decoratedFragmentationStrategy).addMemberToBucket(eq(CHILD_BUCKET), any(), any()); + inOrder.verify(bucketFinder).getLowestBucket(parentBucket, fragmentationTimestamp, Granularity.YEAR); + inOrder.verify(decoratedFragmentationStrategy).addMemberToBucket(eq(childBucket), any(), any(Observation.class)); } @Test void when_BucketisationCalledForMemberWithMissingTimestamp_Then_FunctionsAreCalled() { FragmentationMember member = mock(FragmentationMember.class); - when(bucketFinder.getDefaultFragment(PARENT_BUCKET)) - .thenReturn(CHILD_BUCKET); + when(bucketFinder.getDefaultFragment(parentBucket)).thenReturn(childBucket); - fragmentationStrategy.addMemberToBucket(PARENT_BUCKET, member, - mock(Observation.class)); + fragmentationStrategy.addMemberToBucket(parentBucket, member, mock()); InOrder inOrder = Mockito.inOrder(bucketFinder, decoratedFragmentationStrategy); - inOrder.verify(bucketFinder).getDefaultFragment(PARENT_BUCKET); - inOrder.verify(decoratedFragmentationStrategy, - times(1)).addMemberToBucket(eq(CHILD_BUCKET), any(), - any(Observation.class)); + inOrder.verify(bucketFinder).getDefaultFragment(parentBucket); + inOrder.verify(decoratedFragmentationStrategy).addMemberToBucket(eq(childBucket), any(), any(Observation.class)); } } diff --git a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketCreatorTest.java b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketCreatorTest.java index 60b4aaee8..81b372b31 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketCreatorTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketCreatorTest.java @@ -2,7 +2,6 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.constants.Granularity; @@ -12,7 +11,6 @@ import java.time.LocalDateTime; import java.util.List; -import java.util.Optional; import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.ServerConstants.DEFAULT_BUCKET_STRING; import static org.assertj.core.api.Assertions.assertThat; @@ -21,73 +19,44 @@ class TimeBasedBucketCreatorTest { private static final ViewName VIEW_NAME = new ViewName("collectionName", "view"); private static final BucketDescriptorPair timePair = new BucketDescriptorPair(Granularity.YEAR.getValue(), "2023"); - private static final Bucket PARENT = new Bucket(new BucketDescriptor(List.of(timePair)), VIEW_NAME); - private static final Bucket ROOT = new Bucket(BucketDescriptor.empty(), VIEW_NAME); + private Bucket parent; + private Bucket root; private static final FragmentationTimestamp TIME = new FragmentationTimestamp(LocalDateTime.of(2023, 1, 1, 0, 0, 0), Granularity.MONTH); - private BucketRepository bucketRepository; private TimeBasedRelationsAttributer relationsAttributer; private TimeBasedBucketCreator bucketCreator; @BeforeEach void setUp() { - bucketRepository = mock(BucketRepository.class); relationsAttributer = mock(TimeBasedRelationsAttributer.class); - bucketCreator = new TimeBasedBucketCreator(bucketRepository, relationsAttributer); + bucketCreator = new TimeBasedBucketCreator(relationsAttributer); + parent = new Bucket(new BucketDescriptor(List.of(timePair)), VIEW_NAME); + root = Bucket.createRootBucketForView(VIEW_NAME); } @Test - void when_FragmentDoesNotExist_Then_NewFragmentIsCreated() { - final BucketDescriptor expectedBucketDescriptor = new BucketDescriptor(List.of(timePair, new BucketDescriptorPair(Granularity.MONTH.getValue(), "01"))); - final Bucket expectedChild = new Bucket(expectedBucketDescriptor, VIEW_NAME); - when(bucketRepository.retrieveBucket(VIEW_NAME, expectedBucketDescriptor)).thenReturn(Optional.empty()); - when(bucketRepository.insertBucket(any())).thenReturn(expectedChild); + void test_GetOrCreateInBetweenBucket() { + final BucketDescriptor childDescriptor = new BucketDescriptor(List.of(timePair, new BucketDescriptorPair(Granularity.MONTH.getValue(), "01"))); + when(relationsAttributer.addInBetweenRelation(parent, new Bucket(childDescriptor, VIEW_NAME))) + .thenReturn(new Bucket(childDescriptor, VIEW_NAME)); - Bucket child = bucketCreator.getOrCreateBucket(PARENT, TIME, Granularity.MONTH); + Bucket child = bucketCreator.createBucket(parent, TIME, Granularity.MONTH); - assertThat(child) - .describedAs("Child instance must be the same, to assure the bucket instance from the db is returned") - .isSameAs(expectedChild) - .extracting(Bucket::getBucketDescriptor) - .isEqualTo(expectedBucketDescriptor); - verify(bucketRepository).retrieveBucket(VIEW_NAME, expectedBucketDescriptor); - verify(relationsAttributer).addInBetweenRelation(PARENT, child); - verify(bucketRepository).insertBucket(child); - verifyNoMoreInteractions(bucketRepository); + verify(relationsAttributer).addInBetweenRelation(eq(parent), any()); + assertThat(child.getBucketDescriptor()).isEqualTo(childDescriptor); } @Test - void when_FragmentDoesNotExistAndIsDefaultFragment_Then_NewFragmentIsCreated() { - BucketDescriptor expectedBucketDescriptor = new BucketDescriptor( + void test_GetOrCreateDefaultBucket() { + BucketDescriptor childDescriptor = new BucketDescriptor( List.of(new BucketDescriptorPair(Granularity.YEAR.getValue(), DEFAULT_BUCKET_STRING))); - final Bucket expectedChild = new Bucket(expectedBucketDescriptor, VIEW_NAME); - when(bucketRepository.retrieveBucket(VIEW_NAME, expectedBucketDescriptor)).thenReturn(Optional.empty()); - when(bucketRepository.insertBucket(any())).thenReturn(expectedChild); + when(relationsAttributer.addDefaultRelation(root, new Bucket(childDescriptor, VIEW_NAME))) + .thenReturn(new Bucket(childDescriptor, VIEW_NAME)); - Bucket child = bucketCreator.getOrCreateBucket(new Bucket(BucketDescriptor.empty(), VIEW_NAME), DEFAULT_BUCKET_STRING, Granularity.YEAR); + Bucket child = bucketCreator.createBucket(root, DEFAULT_BUCKET_STRING, Granularity.YEAR); - assertThat(child) - .describedAs("Child instance must be the same, to assure the bucket inserted into the db is returned") - .isSameAs(expectedChild) - .extracting(Bucket::getBucketDescriptor) - .isEqualTo(expectedBucketDescriptor); - verify(bucketRepository).retrieveBucket(VIEW_NAME, expectedBucketDescriptor); - verify(relationsAttributer).addDefaultRelation(ROOT, child); - verify(bucketRepository).insertBucket(child); - verifyNoMoreInteractions(bucketRepository); - } - - @Test - void when_FragmentDoesExist_Then_FragmentIsRetrieved() { - Bucket expectedChild = PARENT.createChild(new BucketDescriptorPair(Granularity.MONTH.getValue(), "01")); - when(bucketRepository.retrieveBucket(VIEW_NAME, expectedChild.getBucketDescriptor())).thenReturn(Optional.of(expectedChild)); - - Bucket child = bucketCreator.getOrCreateBucket(PARENT, TIME, Granularity.MONTH); - - assertThat(child.getBucketDescriptorAsString()).isEqualTo(expectedChild.getBucketDescriptorAsString()); - verify(bucketRepository).retrieveBucket(VIEW_NAME, expectedChild.getBucketDescriptor()); - verifyNoInteractions(relationsAttributer); - verifyNoMoreInteractions(bucketRepository); + verify(relationsAttributer).addDefaultRelation(eq(root), any()); + assertThat(child.getBucketDescriptor()).isEqualTo(childDescriptor); } } \ No newline at end of file diff --git a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketFinderTest.java b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketFinderTest.java index 14d87fe46..98acb2d5a 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketFinderTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedBucketFinderTest.java @@ -26,7 +26,7 @@ class TimeBasedBucketFinderTest { @BeforeEach void setUp() { - TimeBasedConfig config = new TimeBasedConfig(".*", "", Granularity.DAY, false); + TimeBasedConfig config = new TimeBasedConfig(".*", "", Granularity.DAY); bucketCreator = mock(TimeBasedBucketCreator.class); bucketFinder = new TimeBasedBucketFinder(bucketCreator, config); @@ -43,9 +43,9 @@ void when_GetLowestIsCalled_Then_ReturnExpectedBucket() { Bucket yearBucket = PARENT.createChild(new BucketDescriptorPair(Granularity.YEAR.getValue(), "2023")); Bucket monthBucket = yearBucket.createChild(new BucketDescriptorPair(Granularity.MONTH.getValue(), "01")); Bucket dayBucket = monthBucket.createChild(new BucketDescriptorPair(Granularity.DAY.getValue(), "01")); - when(bucketCreator.getOrCreateBucket(PARENT, TIME, Granularity.YEAR)).thenReturn(yearBucket); - when(bucketCreator.getOrCreateBucket(yearBucket, TIME, Granularity.MONTH)).thenReturn(monthBucket); - when(bucketCreator.getOrCreateBucket(monthBucket, TIME, Granularity.DAY)).thenReturn(dayBucket); + when(bucketCreator.createBucket(PARENT, TIME, Granularity.YEAR)).thenReturn(yearBucket); + when(bucketCreator.createBucket(yearBucket, TIME, Granularity.MONTH)).thenReturn(monthBucket); + when(bucketCreator.createBucket(monthBucket, TIME, Granularity.DAY)).thenReturn(dayBucket); Bucket actual = bucketFinder.getLowestBucket(PARENT, TIME, Granularity.YEAR); @@ -56,7 +56,7 @@ void when_GetLowestIsCalled_Then_ReturnExpectedBucket() { void when_GetDefaultIsCalled_Then_ReturnExpectedFragment() { bucketFinder.getDefaultFragment(PARENT); - verify(bucketCreator).getOrCreateBucket(PARENT, DEFAULT_BUCKET_STRING, Granularity.YEAR); + verify(bucketCreator).createBucket(PARENT, DEFAULT_BUCKET_STRING, Granularity.YEAR); } diff --git a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedRelationsAttributerTest.java b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedRelationsAttributerTest.java index 4ccc1bba6..72e920aca 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedRelationsAttributerTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentisers/timebasedhierarchical/services/TimeBasedRelationsAttributerTest.java @@ -4,97 +4,66 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelationCreatedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.TreeRelation; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.config.TimeBasedConfig; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.constants.Granularity; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.springframework.context.ApplicationEventPublisher; import java.time.LocalDateTime; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentisers.timebasedhierarchical.constants.TimeBasedConstants.*; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; +import static org.assertj.core.api.Assertions.assertThat; class TimeBasedRelationsAttributerTest { private static final ViewName VIEW_NAME = new ViewName("collectionName", "view"); private static final BucketDescriptorPair timePair = new BucketDescriptorPair(Granularity.YEAR.getValue(), "2023"); private static final BucketDescriptorPair monthPair = new BucketDescriptorPair(Granularity.MONTH.getValue(), "02"); - private static final Bucket PARENT_BUCKET = new Bucket(BucketDescriptor.of(timePair), VIEW_NAME); + private Bucket parentBucket; private TimeBasedRelationsAttributer relationsAttributer; - private ApplicationEventPublisher applicationEventPublisher; private TimeBasedConfig config; @BeforeEach void setUp() { - applicationEventPublisher = mock(ApplicationEventPublisher.class); - config = new TimeBasedConfig(".*", "", Granularity.SECOND, false); - relationsAttributer = new TimeBasedRelationsAttributer(applicationEventPublisher, config); + config = new TimeBasedConfig(".*", "", Granularity.SECOND); + relationsAttributer = new TimeBasedRelationsAttributer(config); + parentBucket = new Bucket(BucketDescriptor.of(timePair), VIEW_NAME); } @Test void when_RelationNotPresent_AndCachingDisabled_ThenRelationIsAdded_NextUpdateTsIsNotSet_ChildrenStayMutable() { - Bucket child = PARENT_BUCKET.createChild(monthPair); + Bucket child = parentBucket.createChild(monthPair); - BucketRelation gteRelation = new BucketRelation( - PARENT_BUCKET, - child, + TreeRelation gteRelation = new TreeRelation( TREE_GTE_RELATION, LocalDateTime.of(2023,2,1,0,0).toString(), XSD_DATETIME, config.getFragmentationPath()); - BucketRelation ltRelation = new BucketRelation( - PARENT_BUCKET, - child, + TreeRelation ltRelation = new TreeRelation( TREE_LT_RELATION, LocalDateTime.of(2023,3,1,0,0).toString(), XSD_DATETIME, config.getFragmentationPath()); - relationsAttributer.addInBetweenRelation(PARENT_BUCKET, child); + relationsAttributer.addInBetweenRelation(parentBucket, child); - verify(applicationEventPublisher).publishEvent(new BucketRelationCreatedEvent(gteRelation)); - verify(applicationEventPublisher).publishEvent(new BucketRelationCreatedEvent(ltRelation)); - } - - @Test - void when_RelationNotPresent_AndCachingEnabled_ThenRelationIsAdded_NextUpdateTsIsSet_AndChildrenBecomeImmutable() { - config = new TimeBasedConfig(".*", "", Granularity.SECOND, true); - relationsAttributer = new TimeBasedRelationsAttributer(applicationEventPublisher, config); - Bucket child = PARENT_BUCKET.createChild(monthPair); - - BucketRelation gteRelation = new BucketRelation( - PARENT_BUCKET, - child, - TREE_GTE_RELATION, - LocalDateTime.of(2023,2,1,0,0).toString(), - XSD_DATETIME, - config.getFragmentationPath()); - BucketRelation ltRelation = new BucketRelation( - PARENT_BUCKET, - child, - TREE_LT_RELATION, - LocalDateTime.of(2023,3,1,0,0).toString(), - XSD_DATETIME, - config.getFragmentationPath()); - - relationsAttributer.addInBetweenRelation(PARENT_BUCKET, child); + assertThat(parentBucket.getChildren()) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyInAnyOrder( + child.withRelations(gteRelation, ltRelation) + ); - verify(applicationEventPublisher).publishEvent(new BucketRelationCreatedEvent(gteRelation)); - verify(applicationEventPublisher).publishEvent(new BucketRelationCreatedEvent(ltRelation)); } @Test void when_RelationNotPresent_Then_AddDefaultRelation() { - Bucket child = PARENT_BUCKET.createChild(monthPair); - BucketRelation expected = BucketRelation.createGenericRelation(PARENT_BUCKET, child); + Bucket child = parentBucket.createChild(monthPair); - relationsAttributer.addDefaultRelation(PARENT_BUCKET, child); + relationsAttributer.addDefaultRelation(parentBucket, child); - verify(applicationEventPublisher).publishEvent(new BucketRelationCreatedEvent(expected)); + assertThat(parentBucket.getChildren()) + .containsExactly(child.withGenericRelation()); } } diff --git a/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentationConfigCompaction.java b/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentationConfigCompaction.java index 8901e7954..0b79fad27 100644 --- a/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentationConfigCompaction.java +++ b/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentationConfigCompaction.java @@ -2,7 +2,6 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategy; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategyImpl; -import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -10,7 +9,7 @@ public class FragmentationConfigCompaction { @Bean("compactionFragmentation") - public FragmentationStrategy compactionFragmentationStrategy(ApplicationEventPublisher eventPublisher) { + public FragmentationStrategy compactionFragmentationStrategy() { return new FragmentationStrategyImpl(); } } diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/compaction/FragmentsCompactedEvent.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/compaction/FragmentsCompactedEvent.java deleted file mode 100644 index 7e2d49830..000000000 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/compaction/FragmentsCompactedEvent.java +++ /dev/null @@ -1,8 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.events.compaction; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; - -import java.util.List; - -public record FragmentsCompactedEvent(List compactedFragments) { -} diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/BulkFragmentDeletedEvent.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/BulkFragmentDeletedEvent.java deleted file mode 100644 index 31600aa51..000000000 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/BulkFragmentDeletedEvent.java +++ /dev/null @@ -1,8 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.events.fragmentation; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; - -import java.util.Set; - -public record BulkFragmentDeletedEvent(Set ldesFragmentIdentifiers) { -} diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/BulkMemberAllocatedEvent.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/BulkMemberAllocatedEvent.java deleted file mode 100644 index 66e13a359..000000000 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/BulkMemberAllocatedEvent.java +++ /dev/null @@ -1,6 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.events.fragmentation; - -import java.util.List; - -public record BulkMemberAllocatedEvent(List membersOfCompactedFragments, String collectionName, String viewName, String fragmentId) { -} diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/MemberAllocatedEvent.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/MemberAllocatedEvent.java deleted file mode 100644 index c80c049aa..000000000 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/MemberAllocatedEvent.java +++ /dev/null @@ -1,5 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.events.fragmentation; - - -public record MemberAllocatedEvent(String memberId, String collectionName, String viewName, String fragmentId) { -} diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/TimeBasedLinearCachingTriggered.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/TimeBasedLinearCachingTriggered.java deleted file mode 100644 index 3c95ea824..000000000 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/TimeBasedLinearCachingTriggered.java +++ /dev/null @@ -1,6 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.events.fragmentation; - -import java.time.LocalDateTime; - -public record TimeBasedLinearCachingTriggered(long bucketId, LocalDateTime nextUpdateTs) { -} diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/ViewNeedsRebucketisationEvent.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/ViewNeedsRebucketisationEvent.java deleted file mode 100644 index 521dca13e..000000000 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/fragmentation/ViewNeedsRebucketisationEvent.java +++ /dev/null @@ -1,12 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.events.fragmentation; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.ViewAddedEvent; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.ViewInitializationEvent; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; - -/** - * This event is published as a response to {@link ViewInitializationEvent} or - * to {@link ViewAddedEvent}. This will trigger rebucketisation of the member for a view. - */ -public record ViewNeedsRebucketisationEvent(ViewName viewName) { -} diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/ingest/MembersIngestedEvent.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/ingest/MembersIngestedEvent.java deleted file mode 100644 index 551d6d4b7..000000000 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/events/ingest/MembersIngestedEvent.java +++ /dev/null @@ -1,9 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.events.ingest; - -import java.time.LocalDateTime; -import java.util.List; - -public record MembersIngestedEvent(String collectionName, List members) { - public record MemberProperties(String id, String versionOf, LocalDateTime timestamp) { - } -} diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/exceptions/DataConversionException.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/exceptions/DataConversionException.java deleted file mode 100644 index 0a229a85f..000000000 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/exceptions/DataConversionException.java +++ /dev/null @@ -1,32 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.exceptions; - -@SuppressWarnings("java:S3740") -public class DataConversionException extends RuntimeException { - private final Class conversionClass; - private final String operation; - - public DataConversionException(Class conversionClass, boolean serialising, Exception e) { - super(e); - this.conversionClass = conversionClass; - this.operation = serialising ? "serializing" : "deserializing"; - } - - private DataConversionException(Class conversionClass, String operation, Exception e) { - super(e); - this.conversionClass = conversionClass; - this.operation = operation; - } - - @Override - public String getMessage() { - return "Failed %s class %s failed.".formatted(operation, conversionClass); - } - - public static DataConversionException serializationFailed(Class conversionClass, Exception e) { - return new DataConversionException(conversionClass, "serializing", e); - } - - public static DataConversionException deserializationFailed(Class conversionClass, Exception e) { - return new DataConversionException(conversionClass, "deserializing", e); - } -} diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/FragmentPair.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/FragmentPair.java deleted file mode 100644 index 289c16a01..000000000 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/FragmentPair.java +++ /dev/null @@ -1,4 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.model; - -public record FragmentPair(String fragmentKey, String fragmentValue) { -} diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/FragmentSequence.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/FragmentSequence.java deleted file mode 100644 index c3d02660f..000000000 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/FragmentSequence.java +++ /dev/null @@ -1,13 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.model; - -public record FragmentSequence(ViewName viewName, long sequenceNr) { - - /** - * Creates a sequence with a negative number. The ingest module starts sequencing from 0. - * This negative sequenceNr indicates that this view has never been processed for fragmentation. - */ - public static FragmentSequence createNeverProcessedSequence(ViewName viewName) { - return new FragmentSequence(viewName, -1); - } - -} diff --git a/ldes-server-domain/src/main/java/module-info.java b/ldes-server-domain/src/main/java/module-info.java index f8f372d29..ec845fd5a 100644 --- a/ldes-server-domain/src/main/java/module-info.java +++ b/ldes-server-domain/src/main/java/module-info.java @@ -7,11 +7,8 @@ exports be.vlaanderen.informatievlaanderen.ldes.server.domain.rest; // Events - exports be.vlaanderen.informatievlaanderen.ldes.server.domain.events.fragmentation; exports be.vlaanderen.informatievlaanderen.ldes.server.domain.events.retention; - exports be.vlaanderen.informatievlaanderen.ldes.server.domain.events.ingest; exports be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin; - exports be.vlaanderen.informatievlaanderen.ldes.server.domain.events.compaction; exports be.vlaanderen.informatievlaanderen.ldes.server.domain.services; requires spring.web; diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepository.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepository.java index f5d186581..5bf5169fd 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepository.java +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepository.java @@ -8,10 +8,12 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.mapper.BucketMapper; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.repository.BucketEntityRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.BucketRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; +import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -19,32 +21,36 @@ public class BucketPostgresRepository implements BucketRepository { private final ViewEntityRepository viewEntityRepository; private final BucketEntityRepository bucketEntityRepository; + private final NamedParameterJdbcTemplate jdbcTemplate; - public BucketPostgresRepository(ViewEntityRepository viewEntityRepository, BucketEntityRepository bucketEntityRepository) { + public BucketPostgresRepository(ViewEntityRepository viewEntityRepository, BucketEntityRepository bucketEntityRepository, NamedParameterJdbcTemplate jdbcTemplate) { this.viewEntityRepository = viewEntityRepository; this.bucketEntityRepository = bucketEntityRepository; - } - - @Override - public Optional retrieveBucket(ViewName viewName, BucketDescriptor bucketDescriptor) { - return bucketEntityRepository - .findBucketEntityByBucketDescriptor(viewName.asString(), bucketDescriptor.asDecodedString()) - .map(BucketMapper::fromProjection); + this.jdbcTemplate = jdbcTemplate; } @Override @Transactional public Bucket insertBucket(Bucket bucket) { + final String sql = """ + INSERT INTO pages (bucket_id, expiration, partial_url) + VALUES (:bucketId, NULL, :partialUrl) + ON CONFLICT DO NOTHING + """; ViewEntity view = viewEntityRepository.findByViewName(bucket.getViewName().getCollectionName(), bucket.getViewName().getViewName()) .orElseThrow(); BucketEntity bucketEntity = new BucketEntity(view, bucket.getBucketDescriptorAsString()); bucketEntity = bucketEntityRepository.save(bucketEntity); + long bucketId = Objects.requireNonNull(bucketEntity.getBucketId()); + jdbcTemplate.update(sql, Map.of("bucketId", bucketId, "partialUrl", bucket.createPartialUrl())); return new Bucket( - Objects.requireNonNull(bucketEntity.getBucketId()), + bucketId, bucket.getBucketDescriptor(), - bucket.getViewName() + bucket.getViewName(), + List.of(), + 0 ); } diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/BucketisationItemWriter.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/BucketisationItemWriter.java new file mode 100644 index 000000000..0fac843a4 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/BucketisationItemWriter.java @@ -0,0 +1,56 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch; + +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.chunk.ChunkCollector; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; +import org.springframework.batch.item.Chunk; +import org.springframework.batch.item.ItemWriter; +import org.springframework.stereotype.Component; + +import java.util.Optional; + +@Component +public class BucketisationItemWriter implements ItemWriter { + private final ItemWriter bucketItemWriter; + private final ItemWriter pageItemWriter; + private final ItemWriter bucketisedMemberItemWriter; + private final ItemWriter bucketRelationWriter; + + public BucketisationItemWriter(ItemWriter bucketItemWriter, + ItemWriter pageItemWriter, + ItemWriter bucketisedMemberItemWriter, + ItemWriter bucketRelationWriter) { + this.bucketItemWriter = bucketItemWriter; + this.pageItemWriter = pageItemWriter; + this.bucketisedMemberItemWriter = bucketisedMemberItemWriter; + this.bucketRelationWriter = bucketRelationWriter; + } + + @Override + public void write(Chunk rootBucketChunk) throws Exception { + for(var rootbucket : rootBucketChunk) { + final Chunk flatBucketChunk = new Chunk<>(rootbucket.getBucketTree()); + bucketItemWriter.write(flatBucketChunk); + pageItemWriter.write(flatBucketChunk); + bucketRelationWriter.write(extractAllBucketRelations(rootbucket)); + bucketisedMemberItemWriter.write(extractAllMembers(flatBucketChunk)); + } + } + + private static Chunk extractAllMembers(Chunk flatBucketChunk) { + return flatBucketChunk.getItems().stream() + .map(Bucket::getMember) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(new ChunkCollector<>()); + } + + private static Chunk extractAllBucketRelations(Bucket rootbucket) { + return rootbucket.getBucketTree().stream() + .flatMap(bucket -> bucket.getChildRelations().stream()) + .distinct() + .collect(new ChunkCollector<>()); + } + +} diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/BucketisedMemberWriter.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/BucketisedMemberWriter.java deleted file mode 100644 index eae657569..000000000 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/BucketisedMemberWriter.java +++ /dev/null @@ -1,54 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch; - -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; -import org.springframework.batch.item.Chunk; -import org.springframework.batch.item.ItemWriter; -import org.springframework.batch.item.database.JdbcBatchItemWriter; -import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.stereotype.Component; - -import javax.sql.DataSource; -import java.util.List; - -@Component -public class BucketisedMemberWriter implements ItemWriter> { - private final JdbcBatchItemWriter delegateWriter; - - public BucketisedMemberWriter(JdbcBatchItemWriter delegateWriter) { - this.delegateWriter = delegateWriter; - } - - @Override - public void write(Chunk> chunk) throws Exception { - Chunk bucketisedMembers = new Chunk<>(chunk.getItems() - .stream() - .flatMap(List::stream) - .toList()); - - if (!bucketisedMembers.isEmpty()) { - delegateWriter.write(bucketisedMembers); - } - } - - @Configuration - public static class BatchBucketWriterConfig { - private static final String SQL = """ - INSERT INTO page_members (bucket_id, member_id) - VALUES (?, ?) - """; - - @Bean - JdbcBatchItemWriter batchBucketWriter(DataSource dataSource) { - return new JdbcBatchItemWriterBuilder() - .dataSource(dataSource) - .sql(SQL) - .itemPreparedStatementSetter((item, ps) -> { - ps.setLong(1, item.bucketId()); - ps.setLong(2, item.memberId()); - }) - .build(); - } - } -} diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/chunk/ChunkCollector.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/chunk/ChunkCollector.java new file mode 100644 index 000000000..3d148b8c1 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/chunk/ChunkCollector.java @@ -0,0 +1,40 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.chunk; + +import com.google.common.collect.Streams; +import org.springframework.batch.item.Chunk; + +import java.util.Collections; +import java.util.EnumSet; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.BinaryOperator; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collector; + +public class ChunkCollector implements Collector, Chunk> { + @Override + public Supplier> supplier() { + return Chunk::new; + } + + @Override + public BiConsumer, T> accumulator() { + return Chunk::add; + } + + @Override + public BinaryOperator> combiner() { + return (left, right) -> new Chunk<>(Streams.concat(left.getItems().stream(), right.getItems().stream()).toList()); + } + + @Override + public Function, Chunk> finisher() { + return Function.identity(); + } + + @Override + public Set characteristics() { + return Collections.unmodifiableSet(EnumSet.of(Collector.Characteristics.UNORDERED, Collector.Characteristics.IDENTITY_FINISH)); + } +} diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketItemWriter.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketItemWriter.java new file mode 100644 index 000000000..fb4337a8b --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketItemWriter.java @@ -0,0 +1,49 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.delegates; + +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; +import org.springframework.batch.item.Chunk; +import org.springframework.batch.item.ItemWriter; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.jdbc.support.GeneratedKeyHolder; +import org.springframework.stereotype.Component; + +import java.util.Map; +import java.util.Objects; + +@Component +public class BucketItemWriter implements ItemWriter { + private static final String SQL = """ + INSERT INTO buckets (bucket, view_id) + SELECT :bucket, view_id + FROM views v + JOIN collections c USING (collection_id) + WHERE c.name = :collectionName + AND v.name = :viewName + ON CONFLICT (bucket, view_id) DO UPDATE SET bucket_id = buckets.bucket_id + """; + + private final NamedParameterJdbcTemplate jdbcTemplate; + + public BucketItemWriter(NamedParameterJdbcTemplate jdbcTemplate) { + this.jdbcTemplate = jdbcTemplate; + } + + @Override + public void write(Chunk chunk) { + for (var bucket : chunk) { + GeneratedKeyHolder keyHolder = new GeneratedKeyHolder(); + jdbcTemplate.update( + SQL, + new MapSqlParameterSource(Map.of( + "bucket", bucket.getBucketDescriptorAsString(), + "collectionName", bucket.getViewName().getCollectionName(), + "viewName", bucket.getViewName().getViewName() + )), + keyHolder, + new String[]{"bucket_id"} + ); + bucket.setBucketId(Objects.requireNonNull(keyHolder.getKey()).longValue()); + } + } +} diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketPageItemWriterConfig.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketPageItemWriterConfig.java new file mode 100644 index 000000000..43b892498 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketPageItemWriterConfig.java @@ -0,0 +1,34 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.delegates; + +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; +import org.springframework.batch.item.ItemWriter; +import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; + +import javax.sql.DataSource; +import java.util.Map; + +@Configuration +public class BucketPageItemWriterConfig { + private static final String SQL = """ + INSERT INTO pages (bucket_id, expiration, partial_url) + VALUES (:bucketId, NULL, :partialUrl) + ON CONFLICT DO NOTHING + """; + + @Bean + ItemWriter pageItemWriter(DataSource dataSource) { + return new JdbcBatchItemWriterBuilder() + .dataSource(dataSource) + .sql(SQL) + .assertUpdates(false) + .itemSqlParameterSourceProvider(item -> new MapSqlParameterSource(Map.of( + "bucketId", item.getBucketId(), + "partialUrl", item.createPartialUrl() + ))) + .build(); + + } +} diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketisedMemberItemWriterConfig.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketisedMemberItemWriterConfig.java new file mode 100644 index 000000000..900c9acc1 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketisedMemberItemWriterConfig.java @@ -0,0 +1,32 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.delegates; + +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; +import org.springframework.batch.item.ItemWriter; +import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; + +import javax.sql.DataSource; +import java.util.Map; + +@Configuration +public class BucketisedMemberItemWriterConfig { + private static final String SQL = """ + INSERT INTO page_members (bucket_id, member_id, view_id) + SELECT :bucketId, :memberId, view_id + FROM buckets WHERE bucket_id = :bucketId + """; + + @Bean + public ItemWriter bucketisedMemberWriter(DataSource dataSource) { + return new JdbcBatchItemWriterBuilder() + .dataSource(dataSource) + .sql(SQL) + .itemSqlParameterSourceProvider(item -> new MapSqlParameterSource(Map.of( + "bucketId", item.bucketId(), + "memberId", item.memberId() + ))) + .build(); + } +} diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/PageRelationItemWriterConfig.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/PageRelationItemWriterConfig.java new file mode 100644 index 000000000..c1ddd8288 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/PageRelationItemWriterConfig.java @@ -0,0 +1,37 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.delegates; + +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; +import org.springframework.batch.item.ItemWriter; +import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; + +import javax.sql.DataSource; +import java.util.Map; + +@Configuration +public class PageRelationItemWriterConfig { + + @Bean + public ItemWriter bucketRelationItemWriter(DataSource dataSource) { + final String sql = """ + INSERT INTO page_relations (from_page_id, to_page_id, relation_type, value, value_type, path) + SELECT (SELECT page_id FROM pages WHERE partial_url = :fromPartialUrl), + (SELECT page_id FROM pages WHERE partial_url = :toPartialUrl), + :treeRelationType, :treeValue, :treeValueType, :treePath + """; + return new JdbcBatchItemWriterBuilder() + .dataSource(dataSource) + .sql(sql) + .itemSqlParameterSourceProvider(item -> new MapSqlParameterSource(Map.of( + "fromPartialUrl", item.fromPartialUrl(), + "toPartialUrl", item.toPartialUrl(), + "treeRelationType", item.relation().treeRelationType(), + "treeValue", item.relation().treeValue(), + "treeValueType", item.relation().treeValueType(), + "treePath", item.relation().treePath() + ))) + .build(); + } +} diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/entity/MemberBucketEntity.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/entity/MemberBucketEntity.java deleted file mode 100644 index 7765c62bf..000000000 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/entity/MemberBucketEntity.java +++ /dev/null @@ -1,54 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.entity; - -import jakarta.persistence.*; - -import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.entity.MemberBucketEntity.BUCKETISATION; - -@Entity -@Table(name = BUCKETISATION, indexes = { - @Index(columnList = "viewName"), - @Index(columnList = "fragmentId"), - @Index(columnList = "sequenceNr") -}) -public class MemberBucketEntity { - public static final String BUCKETISATION = "fragmentation_bucketisation"; - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - @Column(columnDefinition = "serial") - private long id; - private String viewName; - private String fragmentId; - private String memberId; - private long sequenceNr; - - public MemberBucketEntity(String viewName, String fragmentId, String memberId, long sequenceNr) { - this.viewName = viewName; - this.fragmentId = fragmentId; - this.memberId = memberId; - this.sequenceNr = sequenceNr; - } - - protected MemberBucketEntity() { - - } - - public long getId() { - return id; - } - - public String getViewName() { - return viewName; - } - - public String getFragmentId() { - return fragmentId; - } - - public String getMemberId() { - return memberId; - } - - public long getSequenceNr() { - return sequenceNr; - } -} diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/entity/TreeRelationEntity.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/entity/TreeRelationEntity.java deleted file mode 100644 index dedbbf15c..000000000 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/entity/TreeRelationEntity.java +++ /dev/null @@ -1,77 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.entity; - - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.TreeRelation; -import jakarta.persistence.Embeddable; - -@Embeddable -public class TreeRelationEntity { - private String treePath; - private String treeNode; - private String treeValue; - private String treeValueType; - private String relation; - - protected TreeRelationEntity() { - } - - public TreeRelationEntity(String treePath, String treeNode, String treeValue, String treeValueType, - String relation) { - this.treePath = treePath; - this.treeNode = treeNode; - this.treeValue = treeValue; - this.treeValueType = treeValueType; - this.relation = relation; - } - - public TreeRelation toTreeRelation() { - return new TreeRelation(treePath, LdesFragmentIdentifier.fromFragmentId(treeNode), treeValue, - treeValueType, relation); - } - - public static TreeRelationEntity toEntity(TreeRelation treeRelation) { - return new TreeRelationEntity(treeRelation.treePath(), treeRelation.treeNode().asDecodedFragmentId(), - treeRelation.treeValue(), treeRelation.treeValueType(), treeRelation.relation()); - } - - public String getTreePath() { - return treePath; - } - - public void setTreePath(String treePath) { - this.treePath = treePath; - } - - public String getTreeNode() { - return treeNode; - } - - public void setTreeNode(String treeNode) { - this.treeNode = treeNode; - } - - public String getTreeValue() { - return treeValue; - } - - public void setTreeValue(String treeValue) { - this.treeValue = treeValue; - } - - public String getTreeValueType() { - return treeValueType; - } - - public void setTreeValueType(String treeValueType) { - this.treeValueType = treeValueType; - } - - public String getRelation() { - return relation; - } - - public void setRelation(String relation) { - this.relation = relation; - } -} diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/mapper/BucketMapper.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/mapper/BucketMapper.java index 1a514422f..9fe3ff222 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/mapper/BucketMapper.java +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/mapper/BucketMapper.java @@ -2,7 +2,6 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.entity.BucketEntity; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.projections.BucketProjection; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; @@ -12,18 +11,9 @@ private BucketMapper() { public static Bucket fromProjection(BucketProjection projection) { return new Bucket( - projection.getBucketId(), BucketDescriptor.fromString(projection.getBucketDescriptor()), ViewName.fromString(projection.getViewName()) ); } - public static Bucket fromEntity(BucketEntity entity) { - return new Bucket( - entity.getBucketId(), - BucketDescriptor.fromString(entity.getBucketDescriptor()), - ViewName.fromString(entity.getView().getComposedViewName()) - ); - } - } diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepositoryTest.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepositoryTest.java index caec72356..2ad37cf08 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepositoryTest.java +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepositoryTest.java @@ -8,30 +8,29 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.jdbc.Sql; +import java.util.List; import java.util.Optional; import static org.assertj.core.api.Assertions.assertThat; class BucketPostgresRepositoryTest extends PostgresBucketisationIntegrationTest { private static final ViewName VIEW_NAME = new ViewName("collection", "name"); - private static final BucketDescriptor BUCKET_DESCRIPTOR = BucketDescriptor.fromString("key=value&k=v"); @Autowired private BucketPostgresRepository bucketPostgresRepository; - @Test @Sql("./init-bucketReader.sql") void test_BucketRetrieval() { - final Bucket expectedBucket = new Bucket(BUCKET_DESCRIPTOR, VIEW_NAME); + final Bucket expectedBucket = Bucket.createRootBucketForView(VIEW_NAME); - final Optional retrievedBucket = bucketPostgresRepository.retrieveBucket(VIEW_NAME, BUCKET_DESCRIPTOR); + final Optional retrievedBucket = bucketPostgresRepository.retrieveRootBucket(VIEW_NAME); assertThat(retrievedBucket).contains(expectedBucket); } @Test void test_EmptyRetrieval() { - final Optional retrievedBucket = bucketPostgresRepository.retrieveBucket(VIEW_NAME, BUCKET_DESCRIPTOR); + final Optional retrievedBucket = bucketPostgresRepository.retrieveRootBucket(VIEW_NAME); assertThat(retrievedBucket).isEmpty(); } @@ -40,7 +39,7 @@ void test_EmptyRetrieval() { @Sql("./init-bucketWriter.sql") void test_Insertion() { final Bucket bucketToSave = new Bucket(BucketDescriptor.of(new BucketDescriptorPair("key", "value"), new BucketDescriptorPair("k", "v")), VIEW_NAME); - final Bucket expectedSavedBucket = new Bucket(1L, BucketDescriptor.of(new BucketDescriptorPair("key", "value"), new BucketDescriptorPair("k", "v")), VIEW_NAME); + final Bucket expectedSavedBucket = new Bucket(1L, BucketDescriptor.of(new BucketDescriptorPair("key", "value"), new BucketDescriptorPair("k", "v")), VIEW_NAME, List.of(), 0); final Bucket result = bucketPostgresRepository.insertBucket(bucketToSave); diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/PostgresBucketisationIntegrationTest.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/PostgresBucketisationIntegrationTest.java index f858f3bb0..aed04baff 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/PostgresBucketisationIntegrationTest.java +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/PostgresBucketisationIntegrationTest.java @@ -4,9 +4,8 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.services.MemberMetricsRepository; import be.vlaanderen.informatievlaanderen.ldes.server.domain.services.ServerMetrics; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationService; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.BucketisedMemberWriter; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.delegates.BucketisedMemberItemWriterConfig; import be.vlaanderen.informatievlaanderen.ldes.server.ingest.postgres.repository.MemberEntityRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.MemberPropertiesRepository; import io.cucumber.spring.CucumberContextConfiguration; import io.micrometer.observation.ObservationRegistry; import io.zonky.test.db.AutoConfigureEmbeddedDatabase; @@ -28,7 +27,7 @@ @CucumberContextConfiguration @EnableAutoConfiguration(exclude = FragmentationService.class) @DataJpaTest -@AutoConfigureEmbeddedDatabase +@AutoConfigureEmbeddedDatabase(refresh = AutoConfigureEmbeddedDatabase.RefreshMode.BEFORE_EACH_TEST_METHOD) @EnableBatchProcessing @ActiveProfiles("postgres-test") @EntityScan(basePackages = {"be.vlaanderen.informatievlaanderen.ldes.server"}) @@ -40,7 +39,7 @@ "be.vlaanderen.informatievlaanderen.ldes.server.admin.postgres.view", "be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres" }) -@ContextConfiguration(classes = {BucketisedMemberWriter.class}) +@ContextConfiguration(classes = {BucketisedMemberItemWriterConfig.class}) @Import(PostgresBucketisationIntegrationTest.EventStreamControllerTestConfiguration.class) @SuppressWarnings("java:S2187") public class PostgresBucketisationIntegrationTest { @@ -55,11 +54,6 @@ public ObservationRegistry observationRegistry() { return ObservationRegistry.NOOP; } - @Bean - public MemberPropertiesRepository memberPropertiesRepository() { - return mock(MemberPropertiesRepository.class); - } - @Bean ServerMetrics serverMetrics() { return new ServerMetrics(mock(FragmentationMetricsRepository.class), mock(MemberMetricsRepository.class)); diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/BucketisationItemWriterTest.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/BucketisationItemWriterTest.java new file mode 100644 index 000000000..65d8aca47 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/BucketisationItemWriterTest.java @@ -0,0 +1,124 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.chunk.ChunkCollector; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.delegates.TestBucketSupplier; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.TreeRelation; +import org.assertj.core.api.InstanceOfAssertFactories; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.batch.item.Chunk; +import org.springframework.batch.item.ItemWriter; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.assertArg; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class BucketisationItemWriterTest { + private static final ViewName BY_TIME_VIEW_NAME = new ViewName("test", "by-time"); + private static final BucketDescriptorPair year2024Pair = new BucketDescriptorPair("year", "2024"); + private static final BucketDescriptorPair month09Pair = new BucketDescriptorPair("month", "09"); + private static final BucketDescriptorPair month08Pair = new BucketDescriptorPair("month", "08"); + private static final BucketDescriptorPair day23Pair = new BucketDescriptorPair("day", "23"); + private static final BucketDescriptorPair day26Pair = new BucketDescriptorPair("day", "26"); + + @Mock + private ItemWriter bucketItemWriter; + @Mock + private ItemWriter pageItemWriter; + @Mock + private ItemWriter bucketMemberItemWriter; + @Mock + private ItemWriter bucketRelationItemWriter; + private BucketisationItemWriter bucketisationItemWriter; + + @BeforeEach + void setUp() { + bucketisationItemWriter = new BucketisationItemWriter(bucketItemWriter, pageItemWriter, bucketMemberItemWriter, bucketRelationItemWriter); + } + + @Test + void test_ChunkOf4RootBuckets() throws Exception { + AtomicInteger memberId = new AtomicInteger(); + final Chunk chunk = Stream.of( + new BucketDescriptorPair[]{year2024Pair, month09Pair, day23Pair}, + new BucketDescriptorPair[]{year2024Pair, month09Pair, day26Pair}, + new BucketDescriptorPair[]{year2024Pair, month08Pair, day26Pair}, + new BucketDescriptorPair[]{year2024Pair, month08Pair, day23Pair} + ) + .map(pair -> new TestBucketSupplier(BY_TIME_VIEW_NAME, pair, memberId.incrementAndGet()).get()) + .collect(new ChunkCollector<>()); + + bucketisationItemWriter.write(chunk); + + verify(bucketItemWriter, times(4)).write(assertArg(actual -> assertThat(actual).hasSize(4))); + verify(pageItemWriter, times(4)).write(assertArg(actual -> assertThat(actual).hasSize(4))); + verify(bucketRelationItemWriter, times(4)).write(assertArg(actual -> assertThat(actual).hasSize(3))); + verify(bucketMemberItemWriter, times(4)).write(assertArg(actual -> assertThat(actual).hasSize(1))); + } + + @Test + void test_ChunkOf1RootBucket() throws Exception { + final int memberId = 11; + final Bucket rootBucket = new TestBucketSupplier(BY_TIME_VIEW_NAME, new BucketDescriptorPair[]{year2024Pair, month09Pair, day23Pair}, memberId).get(); + final List bucketTree = List.of( + rootBucket, + rootBucket.getChildren().getFirst(), + rootBucket.getChildren().getFirst().getChildren().getFirst(), + rootBucket.getChildren().getFirst().getChildren().getFirst().getChildren().getFirst() + ); + final List relations = new ArrayList<>(); + for (int i = 1; i < bucketTree.size(); i++) { + relations.add(new BucketRelation( + bucketTree.get(i - 1).createPartialUrl(), + bucketTree.get(i).createPartialUrl(), + TreeRelation.generic() + )); + + } + doAnswer(invocation -> { + final Chunk buckets = (Chunk) invocation.getArgument(0, Chunk.class); + List items = buckets.getItems(); + for (int i = 0; i < items.size(); i++) { + Bucket item = items.get(i); + assertThat(item.getBucketId()).isZero(); + item.setBucketId(i + 1); + } + return null; + }).when(bucketItemWriter).write(any()); + + bucketisationItemWriter.write(Chunk.of(rootBucket)); + + verify(bucketItemWriter).write(assertArg(actual -> assertThat(actual.getItems()) + .asInstanceOf(InstanceOfAssertFactories.list(Bucket.class)) + .containsExactlyInAnyOrderElementsOf(bucketTree) + )); + verify(pageItemWriter).write(assertArg(actual -> assertThat(actual.getItems()) + .asInstanceOf(InstanceOfAssertFactories.list(Bucket.class)) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyInAnyOrderElementsOf(bucketTree) + .allSatisfy(bucket -> assertThat(bucket.getBucketId()).isNotZero()) + )); + verify(bucketRelationItemWriter).write(assertArg(actual -> assertThat(actual.getItems()) + .asInstanceOf(InstanceOfAssertFactories.list(BucketRelation.class)) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyInAnyOrderElementsOf(relations) + )); + verify(bucketMemberItemWriter).write(assertArg(actual -> assertThat(actual.getItems()) + .first() + .isEqualTo(new BucketisedMember(bucketTree.size(), memberId)))); + } +} \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketItemWriterTest.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketItemWriterTest.java new file mode 100644 index 000000000..164c9cc11 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketItemWriterTest.java @@ -0,0 +1,36 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.delegates; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.PostgresBucketisationIntegrationTest; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; +import org.junit.jupiter.api.Test; +import org.springframework.batch.item.Chunk; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.jdbc.Sql; + +import static org.assertj.core.api.Assertions.assertThat; + +class BucketItemWriterTest extends PostgresBucketisationIntegrationTest { + private static final ViewName VIEW_NAME = new ViewName("mobility-hindrances", "by-hour"); + @Autowired + private BucketItemWriter bucketItemWriter; + @Autowired + JdbcTemplate jdbcTemplate; + + @Test + @Sql("./init-collection-and-view.sql") + void testWriter() { + final BucketDescriptorPair[] pairs = BucketDescriptor.fromString("year=2024&month=09&day=14&hour=09").getDescriptorPairs().toArray(new BucketDescriptorPair[0]); + final Bucket rootBucket = new TestBucketSupplier(VIEW_NAME, pairs, 12).get(); + final Chunk bucketTreeChunk = new Chunk<>(rootBucket.getBucketTree()); + + bucketItemWriter.write(bucketTreeChunk); + + var count = jdbcTemplate.queryForObject("SELECT COUNT(*) FROM buckets", Integer.class); + assertThat(count).isEqualTo(5); + assertThat(bucketTreeChunk.getItems()).allSatisfy(bucket -> assertThat(bucket.getBucketId()).isNotZero()); + } +} \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketPageItemWriterTest.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketPageItemWriterTest.java new file mode 100644 index 000000000..432d5c690 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketPageItemWriterTest.java @@ -0,0 +1,38 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.delegates; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.PostgresBucketisationIntegrationTest; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; +import org.junit.jupiter.api.Test; +import org.springframework.batch.item.Chunk; +import org.springframework.batch.item.ItemWriter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.jdbc.Sql; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class BucketPageItemWriterTest extends PostgresBucketisationIntegrationTest { + private static final ViewName VIEW_NAME = new ViewName("mobility-hindrances", "by-hour"); + @Autowired + private ItemWriter pageItemWriter; + @Autowired + JdbcTemplate jdbcTemplate; + + @Test + @Sql({"./init-collection-and-view.sql", "./init-writer-test.sql"}) + void testWriter() throws Exception { + final BucketDescriptorPair[] pairs = BucketDescriptor.fromString("year=2023&month=06").getDescriptorPairs().toArray(new BucketDescriptorPair[0]); + final List bucketTree = new TestBucketSupplier(VIEW_NAME, pairs, 12, true).getBucketTree(); + final Chunk bucketTreeChunk = new Chunk<>(bucketTree); + + pageItemWriter.write(bucketTreeChunk); + + var count = jdbcTemplate.queryForObject("SELECT COUNT(*) FROM pages", Integer.class); + assertThat(count).isEqualTo(3); + } +} \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/BucketisedMemberWriterTest.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketisedMemberWriterTest.java similarity index 65% rename from ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/BucketisedMemberWriterTest.java rename to ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketisedMemberWriterTest.java index b3765e3fe..b658e7786 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/BucketisedMemberWriterTest.java +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/BucketisedMemberWriterTest.java @@ -1,41 +1,38 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch; +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.delegates; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.BucketisedMember; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.PostgresBucketisationIntegrationTest; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.chunk.ChunkCollector; import org.junit.jupiter.api.Test; import org.springframework.batch.item.Chunk; +import org.springframework.batch.item.ItemWriter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.test.context.jdbc.Sql; import javax.sql.DataSource; -import java.util.List; import java.util.stream.IntStream; import static org.assertj.core.api.Assertions.assertThat; class BucketisedMemberWriterTest extends PostgresBucketisationIntegrationTest { @Autowired - BucketisedMemberWriter writer; + ItemWriter writer; @Autowired DataSource dataSource; @Test - @Sql("./init-writer-test.sql") + @Sql({"./init-collection-and-view.sql", "./init-writer-test.sql"}) void testWriter() throws Exception { - List bucketisedMembers = initBucketisedMembers(); - writer.write(Chunk.of(List.of(bucketisedMembers.get(0), bucketisedMembers.get(1)), - List.of(bucketisedMembers.get(2)))); + final long bucketId = 3; + final Chunk members = IntStream.range(1, 4) + .mapToObj(memberId -> new BucketisedMember(bucketId, memberId)) + .collect(new ChunkCollector<>()); - var count = new JdbcTemplate(dataSource).queryForObject("SELECT COUNT(*) FROM page_members", Integer.class); + writer.write(members); + var count = new JdbcTemplate(dataSource).queryForObject("SELECT COUNT(*) FROM page_members", Integer.class); assertThat(count).isEqualTo(3); } - - private static List initBucketisedMembers() { - return IntStream.range(1, 4) - .mapToObj(id -> new BucketisedMember(1, id)) - .toList(); - } } diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/PageRelationItemWriterTest.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/PageRelationItemWriterTest.java new file mode 100644 index 000000000..9502c8023 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/PageRelationItemWriterTest.java @@ -0,0 +1,41 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.delegates; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.PostgresBucketisationIntegrationTest; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.chunk.ChunkCollector; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptor; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.TreeRelation; +import org.junit.jupiter.api.Test; +import org.springframework.batch.item.Chunk; +import org.springframework.batch.item.ItemWriter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.jdbc.Sql; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +class PageRelationItemWriterTest extends PostgresBucketisationIntegrationTest { + private static final ViewName VIEW_NAME = new ViewName("mobility-hindrances", "by-hour"); + + @Autowired + private ItemWriter relationItemWriter; + @Autowired + private JdbcTemplate jdbcTemplate; + + @Test + @Sql({"./init-collection-and-view.sql", "./init-writer-test.sql", "./insert-pages.sql"}) + void testWriter() throws Exception { + final BucketDescriptorPair[] pairs = BucketDescriptor.fromString("year=2023&month=06").getDescriptorPairs().toArray(new BucketDescriptorPair[0]); + final Chunk chunk = new TestBucketSupplier(VIEW_NAME, pairs, 12, true).getBucketTree().stream() + .flatMap(bucket -> bucket.getChildRelations().stream()) + .collect(new ChunkCollector<>()); + chunk.add(new BucketRelation("/mobility-hindrances/by-hour?year=2023", "/mobility-hindrances/by-hour?year=2023&month=07", TreeRelation.generic())); + + relationItemWriter.write(chunk); + + var count = jdbcTemplate.queryForObject("SELECT COUNT(*) FROM page_relations", Integer.class); + assertThat(count).isEqualTo(3); + } +} \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/TestBucketSupplier.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/TestBucketSupplier.java new file mode 100644 index 000000000..1d18161e3 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/TestBucketSupplier.java @@ -0,0 +1,54 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.batch.delegates; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Bucket; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketDescriptorPair; + +import java.util.List; +import java.util.function.Supplier; + +public class TestBucketSupplier implements Supplier { + private final ViewName viewName; + private final BucketDescriptorPair[] bucketDescriptorPairs; + private final long memberId; + private final boolean idGenerationEnabled; + private int currentId = 0; + + public TestBucketSupplier(ViewName viewName, BucketDescriptorPair[] bucketDescriptorPairs, long memberId, boolean idGenerationEnabled) { + this.viewName = viewName; + this.bucketDescriptorPairs = bucketDescriptorPairs; + this.memberId = memberId; + this.idGenerationEnabled = idGenerationEnabled; + } + + public TestBucketSupplier(ViewName viewName, BucketDescriptorPair[] bucketDescriptorPairs, long memberId) { + this(viewName, bucketDescriptorPairs, memberId, false); + } + + @Override + public Bucket get() { + Bucket rootBucket = Bucket.createRootBucketForView(viewName); + assignBucketIdIfNecessary(rootBucket); + Bucket nextParent = rootBucket; + for (final BucketDescriptorPair bucketDescriptorPair : bucketDescriptorPairs) { + nextParent = createChildBucket(nextParent, bucketDescriptorPair); + assignBucketIdIfNecessary(nextParent); + } + nextParent.assignMember(memberId); + return rootBucket; + } + + public List getBucketTree() { + return get().getBucketTree(); + } + + private Bucket createChildBucket(Bucket parentBucket, BucketDescriptorPair childDescriptorPair) { + return parentBucket.addChildBucket(parentBucket.createChild(childDescriptorPair).withGenericRelation()); + } + + private void assignBucketIdIfNecessary(Bucket bucket) { + if (idGenerationEnabled) { + bucket.setBucketId(++currentId); + } + } +} diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/init-collection-and-view.sql b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/init-collection-and-view.sql new file mode 100644 index 000000000..d23e72763 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/init-collection-and-view.sql @@ -0,0 +1,14 @@ +INSERT INTO collections (collection_id, name, timestamp_path, version_of_path, create_versions, is_closed) +VALUES (1, 'mobility-hindrances', 'http://www.w3.org/ns/prov#generatedAtTime', 'http://purl.org/dc/terms/isVersionOf', + false, false); + +INSERT INTO views (view_id, collection_id, name, fragmentations, retention_policies, page_size) +VALUES (1, 1, 'by-hour', '[ + { + "name": "HierarchicalTimeBasedFragmentation", + "config": { + "maxGranularity": "hour", + "fragmentationPath": "http://www.w3.org/ns/prov#generatedAtTime" + } + } +]', '', 250); diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/init-writer-test.sql b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/init-writer-test.sql similarity index 99% rename from ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/init-writer-test.sql rename to ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/init-writer-test.sql index 776517198..56591c9cb 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/init-writer-test.sql +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/init-writer-test.sql @@ -1,18 +1,3 @@ -INSERT INTO collections (collection_id, name, timestamp_path, version_of_path, create_versions, is_closed) -VALUES (1, 'mobility-hindrances', 'http://www.w3.org/ns/prov#generatedAtTime', 'http://purl.org/dc/terms/isVersionOf', - false, false); - -INSERT INTO views (view_id, collection_id, name, fragmentations, retention_policies, page_size) -VALUES (1, 1, 'by-hour', '[ - { - "name": "HierarchicalTimeBasedFragmentation", - "config": { - "maxGranularity": "hour", - "fragmentationPath": "http://www.w3.org/ns/prov#generatedAtTime" - } - } -]', '', 250); - INSERT INTO members (member_id, subject, old_id, collection_id, is_in_event_source, member_model, timestamp, transaction_id, version_of) VALUES (1, 'https://private-api.gipod.beta-vlaanderen.be/api/v1/mobility-hindrances/10810400/600000', @@ -40,4 +25,6 @@ VALUES (3, 'https://private-api.gipod.beta-vlaanderen.be/api/v1/mobility-hindran '2024-06-27 08:47:04.004000', '9f3a0219-e12d-4891-859f-6c9c5f967d48', 'https://private-api.gipod.beta-vlaanderen.be/api/v1/mobility-hindrances/10810400'); -INSERT INTO buckets (bucket_id, bucket, view_id) VALUES (1, 'year=2023&month=06', 1); \ No newline at end of file +INSERT INTO buckets (bucket_id, bucket, view_id) VALUES (1, '', 1); +INSERT INTO buckets (bucket_id, bucket, view_id) VALUES (2, 'year=2023', 1); +INSERT INTO buckets (bucket_id, bucket, view_id) VALUES (3, 'year=2023&month=06', 1); \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/insert-pages.sql b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/insert-pages.sql new file mode 100644 index 000000000..1ce826fed --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/insert-pages.sql @@ -0,0 +1,7 @@ +INSERT INTO buckets (bucket_id, bucket, view_id) VALUES (4, 'year=2023&month=07', 1); + +INSERT INTO pages (page_id, bucket_id, partial_url) +VALUES (1, 1, '/mobility-hindrances/by-hour'), + (2, 2, '/mobility-hindrances/by-hour?year=2023'), + (3, 3, '/mobility-hindrances/by-hour?year=2023&month=06'), + (4, 4, '/mobility-hindrances/by-hour?year=2023&month=07') \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/init-bucketReader.sql b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/init-bucketReader.sql index c0d965897..14e5a4423 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/init-bucketReader.sql +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/init-bucketReader.sql @@ -4,4 +4,5 @@ VALUES (1, 'collection', 'http://purl.org/dc/terms/created', 'http://purl.org/dc INSERT INTO views VALUES (1, 1, 'name', '[]', '', 150); -INSERT INTO buckets VALUES (1, 'key=value&k=v', 1) \ No newline at end of file +INSERT INTO buckets (bucket_id, bucket, view_id) VALUES (1, 'key=value&k=v', 1); +INSERT INTO buckets (bucket_id, bucket, view_id) VALUES (2, '', 1) \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/batch/MemberItemReader.java b/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/batch/MemberItemReader.java index e6d7b940e..a3fe12718 100644 --- a/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/batch/MemberItemReader.java +++ b/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/batch/MemberItemReader.java @@ -15,9 +15,11 @@ import java.util.HashMap; import java.util.Map; +import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.batch.BucketJobDefinitions.CHUNK_SIZE; + @Configuration public class MemberItemReader { - private static final int PAGE_SIZE = 500; + private static final int PAGE_SIZE = CHUNK_SIZE; @Bean @StepScope @@ -30,7 +32,7 @@ public JdbcPagingItemReader memberReader(@Value("#{jobParam .queryProvider(memberQuery()) .parameterValues(jobParameters) .pageSize(PAGE_SIZE) - .maxItemCount(20 * PAGE_SIZE) + .maxItemCount(40 * PAGE_SIZE) .build(); } @@ -38,19 +40,25 @@ private PostgresPagingQueryProvider memberQuery() { Map sortKeys = new HashMap<>(); sortKeys.put("member_id", Order.ASCENDING); PostgresPagingQueryProvider queryProvider = new PostgresPagingQueryProvider(); - queryProvider.setSelectClause("m.member_id, m.subject, m.version_of, m.timestamp, c.name, c.version_of_path, c.timestamp_path, c.create_versions, m.member_model"); + queryProvider.setSelectClause(""" + m.member_id, m.subject, m.version_of, m.timestamp, + c.name, c.version_of_path, c.timestamp_path, c.create_versions, + m.member_model + """); queryProvider.setFromClause(""" - collections c - join views v on v.collection_id = c.collection_id - join bucket_stats bs on bs.collection_id = c.collection_id and bs.view_id = v.view_id - join members m on m.collection_id = c.collection_id + collections c + join members m on m.collection_id = c.collection_id """); queryProvider.setWhereClause(""" - m.member_id > bs.last - AND v.name = :viewName AND c.name = :collectionName + c.name = :collectionName and + m.member_id > ( + COALESCE( + (SELECT max(pm.member_id) FROM page_members pm + WHERE pm.view_id IN (select v.view_id from views v where v.name = :viewName)), + (0)::bigint) + ) """); queryProvider.setSortKeys(sortKeys); return queryProvider; } - } diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/add-view-id-to-page-members.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/add-view-id-to-page-members.xml new file mode 100644 index 000000000..f7c5a9b58 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/add-view-id-to-page-members.xml @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/add_view_stats.sql b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/add_view_stats.sql new file mode 100644 index 000000000..22dbd4703 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/add_view_stats.sql @@ -0,0 +1,65 @@ +-- create view statistics table +CREATE TABLE view_stats +( + view_id bigint NOT NULL, + bucketized_count bigint NOT NULL default 0, + paginated_count bigint NOT NULL default 0 +); +ALTER TABLE "view_stats" ADD FOREIGN KEY ("view_id") REFERENCES "views" ("view_id") ON DELETE CASCADE; + +-- create function to insert a stats row for the new view +create function on_view_inserted() returns trigger language plpgsql as $$ +begin + insert into view_stats(view_id) values (NEW.view_id); + return null; +end +$$; + +-- add stats row after view inserted +create trigger views_ai after insert on views +for each row execute procedure on_view_inserted(); + +-- create function to delete stats row for a view +create function on_view_deleted() returns trigger language plpgsql as $$ +begin + delete from view_stats where view_id = OLD.view_id; + return null; +end +$$; + +-- remove stats row after view deleted +create trigger views_ad after delete on views +for each row execute procedure on_view_deleted(); + +-- initialize current counts +update view_stats vs +set + bucketized_count = (select count(*) from page_members pm WHERE pm.view_id = vs.view_id), + paginated_count = (select count(*) from page_members pm WHERE pm.view_id = vs.view_id and pm.page_id is not null) +; + +-- create function to increase view stats count when member bucketized +create function on_page_member_inserted() returns trigger language plpgsql as $$ +begin + update view_stats set bucketized_count = bucketized_count + 1 where view_id = NEW.view_id; + return null; +end +$$; + +-- update stats row after page_member inserted +create trigger page_member_ai after insert on page_members +for each row execute procedure on_page_member_inserted(); + +-- create function to increase view stats count when member paginated +create function on_page_member_updating() returns trigger language plpgsql as $$ +begin + if (OLD.page_id is null and NEW.page_id is not null) then + update view_stats set paginated_count = paginated_count + 1 where view_id = NEW.view_id; + end if; + return NEW; +end +$$; + +-- update stats row before page_member updating +create trigger page_member_bu before update on page_members +for each row execute procedure on_page_member_updating(); diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/define-bucket-last-page-view.sql b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/define-bucket-last-page-view.sql new file mode 100644 index 000000000..d0306ce3b --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/define-bucket-last-page-view.sql @@ -0,0 +1,5 @@ +create or replace view bucket_last_page as +select p.bucket_id, max(p.page_id) as last_page_id +from pages p +group by p.bucket_id +; \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/drop-on-bucket-insert-trigger.sql b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/drop-on-bucket-insert-trigger.sql new file mode 100644 index 000000000..009222142 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/drop-on-bucket-insert-trigger.sql @@ -0,0 +1,2 @@ +DROP TRIGGER insert_page_on_bucket_insertion ON buckets; +DROP FUNCTION on_bucket_insertion; \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/initialize-view-id.sql b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/initialize-view-id.sql new file mode 100644 index 000000000..4fb4b624a --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/initialize-view-id.sql @@ -0,0 +1,3 @@ +UPDATE page_members pm +SET view_id = (select b.view_id from buckets b where b.bucket_id = pm.bucket_id) +; diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/master.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/master.xml new file mode 100644 index 000000000..afb75413f --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/master.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/redefine-bucket-stats.sql b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/redefine-bucket-stats.sql new file mode 100644 index 000000000..4089f2434 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/redefine-bucket-stats.sql @@ -0,0 +1,6 @@ +CREATE OR REPLACE VIEW bucket_stats as +select c.collection_id, v.view_id, + coalesce((select max(pm.member_id) from page_members pm where v.view_id = pm.view_id),0) as last +from collections c +inner join views v on c.collection_id = v.collection_id +group by c.collection_id, v.view_id; diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/redefine-needs-bucketization.sql b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/redefine-needs-bucketization.sql new file mode 100644 index 000000000..ca25b91f0 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/redefine-needs-bucketization.sql @@ -0,0 +1,5 @@ +CREATE OR REPLACE VIEW "needs_bucketization" AS +SELECT ms.collection_id, ms.view_id, (pm.member_id is null) AS should_bucketize +FROM member_stats ms +left outer join page_members pm on pm.view_id = ms.view_id and pm.member_id = ms.last +group by ms.collection_id, ms.view_id, pm.member_id diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/redefine-needs-pagination.sql b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/redefine-needs-pagination.sql new file mode 100644 index 000000000..79990504b --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_0/redefine-needs-pagination.sql @@ -0,0 +1,6 @@ +CREATE OR REPLACE VIEW "needs_pagination" AS +SELECT c.collection_id, v.view_id, + (EXISTS (SELECT * FROM page_members pm WHERE pm.page_id IS NULL AND pm.view_id = v.view_id)) AS should_paginate +FROM collections c +JOIN views v ON c.collection_id = v.collection_id +GROUP BY c.collection_id, v.view_id; diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml index f054e4f7e..3061e20c4 100644 --- a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml @@ -7,5 +7,6 @@ + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PagePostgresRepository.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PagePostgresRepository.java index 4c61ef33b..56a2cd20b 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PagePostgresRepository.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PagePostgresRepository.java @@ -32,14 +32,15 @@ public PagePostgresRepository(JdbcTemplate jdbcTemplate, PageEntityRepository pa @Transactional(readOnly = true) public Page getOpenPage(long bucketId) { String sql = """ - select DISTINCT p.page_id, p.bucket_id, p.partial_url, v.page_size, COUNT(member_id) AS assigned_members - FROM pages p - LEFT JOIN page_members m ON p.page_id = m.page_id - JOIN buckets b ON p.bucket_id = b.bucket_id - JOIN views v ON v.view_id = b.view_id - where b.bucket_id = ? AND p.page_id NOT IN (SELECT from_page_id FROM page_relations) - group by p.page_id, v.page_size - order by page_id + select p.page_id, p.bucket_id, p.partial_url, v.page_size, COUNT(member_id) AS assigned_members + from pages p + left join page_members pm on pm.page_id = p.page_id + JOIN buckets b ON p.bucket_id = b.bucket_id + JOIN views v ON v.view_id = b.view_id + join bucket_last_page blp on blp.bucket_id = b.bucket_id AND blp.last_page_id = p.page_id + where b.bucket_id = ? + group by p.page_id, v.page_size + order by page_id """; return jdbcTemplate.query(sql, new PaginationRowMapper(), bucketId) .stream() @@ -60,12 +61,6 @@ public Page createNextPage(Page parentPage) { return new Page(newPage.getId(), parentPage.getBucketId(), partialUrl, parentPage.getPageSize()); } - @Override - @Transactional - public void setChildrenImmutableByBucketId(long bucketId) { - pageEntityRepository.setAllChildrenImmutableByBucketId(bucketId); - } - @Override @Transactional public void markAllPagesImmutableByCollectionName(String collectionName) { diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PageRelationPostgresRepository.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PageRelationPostgresRepository.java index e58ed1ec3..f894a9829 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PageRelationPostgresRepository.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PageRelationPostgresRepository.java @@ -1,7 +1,6 @@ package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres; import be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.RdfConstants; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.PageRelationEntityRepository; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRelationRepository; import org.springframework.stereotype.Repository; @@ -23,19 +22,6 @@ public void insertGenericBucketRelation(long fromPageId, long toPageId) { pageRelationEntityRepository.insertRelation(fromPageId, toPageId, RdfConstants.GENERIC_TREE_RELATION); } - @Override - @Transactional - public void insertBucketRelation(BucketRelation bucketRelation) { - pageRelationEntityRepository.insertRelation( - bucketRelation.fromBucket().createPartialUrl(), - bucketRelation.toBucket().createPartialUrl(), - bucketRelation.treeRelationType(), - bucketRelation.treeValue(), - bucketRelation.treeValueType(), - bucketRelation.treePath() - ); - } - @Override @Transactional public void updateCompactionBucketRelations(List compactedPageIds, long targetId) { diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/TreeNodePostgresRepository.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/TreeNodePostgresRepository.java index 8e7e71fc6..e3fb6d592 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/TreeNodePostgresRepository.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/TreeNodePostgresRepository.java @@ -3,7 +3,7 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.EventStreamCreatedEvent; import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.EventStreamDeletedEvent; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.EventStream; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentIdentifier; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.Member; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.TreeNode; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.repository.TreeNodeRepository; diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/BucketPartitioner.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/BucketPartitioner.java index 40bdd806c..a31f7f2d0 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/BucketPartitioner.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/batch/BucketPartitioner.java @@ -17,8 +17,7 @@ public class BucketPartitioner implements Partitioner{ static final String SQL = """ SELECT pm.bucket_id FROM page_members pm - JOIN buckets b on b.bucket_id = pm.bucket_id - JOIN views v on v.view_id = b.view_id + JOIN views v on v.view_id = pm.view_id JOIN collections c on c.collection_id = v.collection_id WHERE c.name = ? AND v.name = ? AND page_id IS NULL GROUP BY pm.bucket_id diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/mapper/TreeRelationMapper.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/mapper/TreeRelationMapper.java index d069f4dd9..3cec2777c 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/mapper/TreeRelationMapper.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/mapper/TreeRelationMapper.java @@ -1,23 +1,12 @@ package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.mapper; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.TreeRelation; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.PageRelationEntity; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.TreeRelation; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.projection.TreeRelationProjection; public class TreeRelationMapper { private TreeRelationMapper() {} - public static TreeRelation fromRelationEntity(PageRelationEntity pageRelationEntity) { - return new TreeRelation( - pageRelationEntity.getTreePath(), - LdesFragmentIdentifier.fromFragmentId(pageRelationEntity.getToPage().getPartialUrl()), - pageRelationEntity.getTreeValue(), - pageRelationEntity.getTreeValueType(), - pageRelationEntity.getTreeRelationType() - ); - } - public static TreeRelation fromProjection(TreeRelationProjection projection) { return new TreeRelation( projection.getTreePath(), diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageEntityRepository.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageEntityRepository.java index 4b3a974b8..289e4a77f 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageEntityRepository.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageEntityRepository.java @@ -21,18 +21,6 @@ public interface PageEntityRepository extends JpaRepository { @Query(value = "UPDATE pages SET immutable = true WHERE page_id = ?", nativeQuery = true) void setPageImmutable(long pageId); - @Modifying - @Query(value = """ - update pages set immutable = true - where page_id in ( - select distinct r.to_page_id from pages p - inner join buckets b on b.bucket_id = p.bucket_id - inner join page_relations r on r.from_page_id = p.page_id - where b.bucket_id = :bucketId - ) - """, nativeQuery = true) - void setAllChildrenImmutableByBucketId(long bucketId); - @Modifying @Query(value = """ UPDATE pages diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageMemberEntityRepository.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageMemberEntityRepository.java index d47f23e71..19214a1da 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageMemberEntityRepository.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageMemberEntityRepository.java @@ -21,25 +21,21 @@ public interface PageMemberEntityRepository extends JpaRepository pageIds); @Query(value = """ - select v.name, count(*) - from page_members - JOIN buckets b on b.bucket_id = page_members.bucket_id - JOIN views v on v.view_id = b.view_id - JOIN collections c on c.collection_id = v.collection_id - WHERE c.name = :collectionName - group by v.name + select v.name, vs.bucketized_count as count + from view_stats vs + JOIN views v on v.view_id = vs.view_id + JOIN collections c on c.collection_id = v.collection_id + WHERE c.name = :collectionName """, nativeQuery = true) List getBucketisedMemberCounts(String collectionName); @Query(value = """ - select v.name, count(*) - from page_members - JOIN buckets b on b.bucket_id = page_members.bucket_id - JOIN views v on v.view_id = b.view_id - JOIN collections c on c.collection_id = v.collection_id - WHERE page_id IS NOT NULL AND c.name = :collectionName - group by v.name - """, nativeQuery = true) + select v.name, vs.paginated_count as count + from view_stats vs + JOIN views v on v.view_id = vs.view_id + JOIN collections c on c.collection_id = v.collection_id + WHERE c.name = :collectionName + """, nativeQuery = true) List getPaginatedMemberCounts(String collectionName); @Modifying diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PaginationSteps.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PaginationSteps.java index a70e039fb..395bb21d3 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PaginationSteps.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PaginationSteps.java @@ -1,5 +1,6 @@ package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.entity.BucketEntity; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.entities.Page; import io.cucumber.java.Before; import io.cucumber.java.en.And; @@ -17,12 +18,15 @@ public class PaginationSteps extends PostgresPaginationIntegrationTest { private List memberIds; private int pageCount; private Long bucketId; + private Integer viewId; private Page openPage; @Before public void setUp() { pageCount = pageEntityRepository.findAll().size(); - bucketId = bucketEntityRepository.findAll().getFirst().getBucketId(); + final BucketEntity bucket = bucketEntityRepository.findAll().getFirst(); + bucketId = bucket.getBucketId(); + viewId = bucket.getView().getId(); openPage = pageRepository.getOpenPage(bucketId); } @@ -101,11 +105,11 @@ private void saveMembers(List memberIds) { private void saveMemberBuckets(List memberIds) { - String sql = "insert into page_members (bucket_id, member_id) VALUES (?, ?)"; + String sql = "insert into page_members (bucket_id, member_id, view_id) VALUES (?, ?, ?)"; final List batchArgs = memberIds.stream() .map(member -> new Object[]{ - bucketId, member + bucketId, member, viewId }) .toList(); diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/init-paged-test.sql b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/init-paged-test.sql index b75701c22..47619a32a 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/init-paged-test.sql +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/init-paged-test.sql @@ -1,6 +1,3 @@ ---TODO remove after VSDSPUB-1433 is done -ALTER TABLE buckets DISABLE TRIGGER ALL; - WITH new_collection AS ( INSERT INTO collections (name, timestamp_path, version_of_path, create_versions, is_closed, skolemization_domain) VALUES ('es', '', '', false, false, null) diff --git a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/CompactionServiceSteps.java b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/CompactionServiceSteps.java index 46daf65a3..46b872d42 100644 --- a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/CompactionServiceSteps.java +++ b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/CompactionServiceSteps.java @@ -1,5 +1,7 @@ package be.vlaanderen.informatievlaanderen.ldes.server; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.PageEntity; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.PageRelationEntity; import io.cucumber.java.en.And; import io.cucumber.java.en.Then; @@ -14,6 +16,7 @@ import java.time.temporal.ChronoUnit; import java.util.List; import java.util.Objects; +import java.util.UUID; import java.util.stream.Collectors; import static java.util.concurrent.TimeUnit.SECONDS; @@ -83,14 +86,18 @@ public void verifyRemovalOfPages(List ids) { }); } - @And("verify {long} pages have a relation pointing to the new page {long}") - public void verifyUpdateOfPredecessorRelations(long pointingCount, long id) { + @And("verify {long} pages have a relation pointing to a compacted page") + public void verifyUpdateOfPredecessorRelations(long pointingCount) { await().untilAsserted(() -> { var countNewPage = pageRelationEntityRepository.findAll() .stream() - .filter(relationEntity -> relationEntity.getToPage().getId().equals(id)) + .map(PageRelationEntity::getToPage) + .map(PageEntity::getPartialUrl) + .map(url -> url.split("pageNumber=")[1]) + .filter(this::isValidUuid) .count(); + assertThat(countNewPage).isEqualTo(pointingCount); }); } @@ -128,4 +135,13 @@ private String readMemberTemplate(String fileName) throws IOException, URISyntax private String getCurrentTimestamp() { return LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.[SSS]'Z'")); } + + private boolean isValidUuid(String pageNumber) { + try { + UUID.fromString(pageNumber); + return true; + } catch (IllegalArgumentException e) { + return false; + } + } } diff --git a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/resultactionsextensions/MemberCounter.java b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/resultactionsextensions/MemberCounter.java index be7be70a0..c10de9206 100644 --- a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/resultactionsextensions/MemberCounter.java +++ b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/resultactionsextensions/MemberCounter.java @@ -2,6 +2,8 @@ import org.apache.jena.rdf.model.Model; import org.mockito.ArgumentMatcher; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.test.util.AssertionErrors; import org.springframework.test.web.servlet.MvcResult; import org.springframework.test.web.servlet.ResultMatcher; @@ -9,6 +11,7 @@ import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.RdfConstants.TREE_MEMBER; public class MemberCounter implements ResultMatcher, ArgumentMatcher { + private static final Logger log = LoggerFactory.getLogger(MemberCounter.class); private final int expectedMemberCount; private MemberCounter(int expectedMemberCount) { @@ -28,6 +31,10 @@ public void match(MvcResult result) throws Exception { @Override public boolean matches(Model model) { - return expectedMemberCount == model.listObjectsOfProperty(TREE_MEMBER).toList().size(); + final int memberCount = model.listObjectsOfProperty(TREE_MEMBER).toList().size(); + if(memberCount != expectedMemberCount) { + log.atInfo().log("Expected {} members, but received {}", expectedMemberCount, memberCount); + } + return expectedMemberCount == memberCount; } } diff --git a/ldes-server-integration-test/src/test/resources/features/compaction/compaction.feature b/ldes-server-integration-test/src/test/resources/features/compaction/compaction.feature index b5cd64db7..c73719ec3 100644 --- a/ldes-server-integration-test/src/test/resources/features/compaction/compaction.feature +++ b/ldes-server-integration-test/src/test/resources/features/compaction/compaction.feature @@ -14,7 +14,7 @@ Feature: Execute CompactionService And verify the following pages have no relation pointing to them | 2 | | 3 | - And verify 3 pages have a relation pointing to the new page 5 + And verify 3 pages have a relation pointing to a compacted page And verify the following pages have no members | 2 | | 3 | diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PaginationJobDefinitions.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PaginationJobDefinitions.java index 0e76009b7..d353a64cd 100644 --- a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PaginationJobDefinitions.java +++ b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PaginationJobDefinitions.java @@ -36,8 +36,6 @@ public Step paginationStep(JobRepository jobRepository, PlatformTransactionManag @Bean("paginationTaskExecutor") public TaskExecutor paginationTaskExecutor() { - var taskExecutor = new SimpleAsyncTaskExecutor("spring_batch"); - taskExecutor.setConcurrencyLimit(50); - return taskExecutor; + return new SimpleAsyncTaskExecutor("spring_batch"); } } diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageRelationRepository.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageRelationRepository.java index 65cd97ac8..0339e1e3e 100644 --- a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageRelationRepository.java +++ b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageRelationRepository.java @@ -1,12 +1,9 @@ package be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelation; - import java.util.List; public interface PageRelationRepository { void insertGenericBucketRelation(long fromPageId, long toPageId); - void insertBucketRelation(BucketRelation bucketRelation); void updateCompactionBucketRelations(List compactedPageIds, long targetId); } diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageRepository.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageRepository.java index 7f0ab6728..032bd5d0e 100644 --- a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageRepository.java +++ b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageRepository.java @@ -11,7 +11,6 @@ public interface PageRepository { Page getOpenPage(long bucketId); Page createNextPage(Page parentPage); - void setChildrenImmutableByBucketId(long bucketId); void markAllPagesImmutableByCollectionName(String collectionName); Stream getPossibleCompactionCandidates(ViewName viewName, int capacityPerPage); void deleteOutdatedFragments(LocalDateTime deleteTime); diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PaginationSequenceRepository.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PaginationSequenceRepository.java deleted file mode 100644 index ce1754fe0..000000000 --- a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PaginationSequenceRepository.java +++ /dev/null @@ -1,16 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentSequence; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; - -import java.util.Optional; - -public interface PaginationSequenceRepository { - Optional findLastProcessedSequence(ViewName viewName); - - void saveLastProcessedSequence(FragmentSequence sequence); - - void deleteByViewName(ViewName viewName); - - void deleteByCollection(String collectionName); -} diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/BucketRelationsEventListener.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/BucketRelationsEventListener.java deleted file mode 100644 index bc090b333..000000000 --- a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/BucketRelationsEventListener.java +++ /dev/null @@ -1,20 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.pagination.services; - -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.valueobjects.BucketRelationCreatedEvent; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRelationRepository; -import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Component; - -@Component -public class BucketRelationsEventListener { - private final PageRelationRepository pageRelationRepository; - - public BucketRelationsEventListener(PageRelationRepository pageRelationRepository) { - this.pageRelationRepository = pageRelationRepository; - } - - @EventListener - public void onBucketRelationCreatedEvent(BucketRelationCreatedEvent event) { - pageRelationRepository.insertBucketRelation(event.bucketRelation()); - } -} diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/LinearCachingEventsListener.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/LinearCachingEventsListener.java deleted file mode 100644 index 8ceaaeb30..000000000 --- a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/LinearCachingEventsListener.java +++ /dev/null @@ -1,20 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.pagination.services; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.fragmentation.TimeBasedLinearCachingTriggered; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRepository; -import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Component; - -@Component -public class LinearCachingEventsListener { - private final PageRepository pageRepository; - - public LinearCachingEventsListener(PageRepository pageRepository) { - this.pageRepository = pageRepository; - } - - @EventListener - public void onLinearCachingTriggered(TimeBasedLinearCachingTriggered event) { - pageRepository.setChildrenImmutableByBucketId(event.bucketId()); - } -} diff --git a/ldes-server-port-fetch-rest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/caching/EtagCachingStrategy.java b/ldes-server-port-fetch-rest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/caching/EtagCachingStrategy.java index c2e1f862b..9a1902489 100644 --- a/ldes-server-port-fetch-rest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/caching/EtagCachingStrategy.java +++ b/ldes-server-port-fetch-rest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/caching/EtagCachingStrategy.java @@ -1,9 +1,9 @@ package be.vlaanderen.informatievlaanderen.ldes.server.rest.caching; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.TreeRelation; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.Member; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.TreeNode; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.TreeRelation; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; diff --git a/ldes-server-port-fetch-rest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/TreeNodeController.java b/ldes-server-port-fetch-rest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/TreeNodeController.java index baa9e18bf..3b8ca530f 100644 --- a/ldes-server-port-fetch-rest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/TreeNodeController.java +++ b/ldes-server-port-fetch-rest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/TreeNodeController.java @@ -1,8 +1,8 @@ package be.vlaanderen.informatievlaanderen.ldes.server.rest.treenode; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentPair; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentRequest; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.FragmentPair; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentRequest; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.TreeNode; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.services.StreamingTreeNodeFactory; diff --git a/ldes-server-port-fetch-rest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/services/TreeRelationResponse.java b/ldes-server-port-fetch-rest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/services/TreeRelationResponse.java index fd7343598..eb77b2249 100644 --- a/ldes-server-port-fetch-rest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/services/TreeRelationResponse.java +++ b/ldes-server-port-fetch-rest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/services/TreeRelationResponse.java @@ -1,6 +1,5 @@ package be.vlaanderen.informatievlaanderen.ldes.server.rest.treenode.services; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.TreeRelation; import org.apache.jena.datatypes.TypeMapper; import org.apache.jena.datatypes.xsd.XSDDatatype; import org.apache.jena.rdf.model.Property; @@ -60,9 +59,4 @@ private boolean hasMeaningfulValue(String objectContent) { return objectContent != null && !objectContent.isEmpty(); } - public static TreeRelationResponse fromRelation(TreeRelation relation, String prefix) { - return new TreeRelationResponse(relation.treePath(), - prefix + relation.treeNode().asEncodedFragmentId(), - relation.treeValue(), relation.treeValueType(), relation.relation()); - } } diff --git a/ldes-server-port-fetch-rest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/caching/EtagCachingStrategyTest.java b/ldes-server-port-fetch-rest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/caching/EtagCachingStrategyTest.java index 60d23a474..d918cc075 100644 --- a/ldes-server-port-fetch-rest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/caching/EtagCachingStrategyTest.java +++ b/ldes-server-port-fetch-rest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/caching/EtagCachingStrategyTest.java @@ -1,7 +1,7 @@ package be.vlaanderen.informatievlaanderen.ldes.server.rest.caching; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.TreeRelation; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.TreeRelation; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.Member; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.TreeNode; diff --git a/ldes-server-port-fetch-rest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/TreeNodeControllerTest.java b/ldes-server-port-fetch-rest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/TreeNodeControllerTest.java index 953d631d2..4746c30e5 100644 --- a/ldes-server-port-fetch-rest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/TreeNodeControllerTest.java +++ b/ldes-server-port-fetch-rest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/TreeNodeControllerTest.java @@ -11,6 +11,9 @@ import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.TreeNode; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.services.StreamingTreeNodeFactory; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.services.TreeNodeFetcher; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.FragmentPair; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentRequest; import be.vlaanderen.informatievlaanderen.ldes.server.rest.caching.CachingStrategy; import be.vlaanderen.informatievlaanderen.ldes.server.rest.caching.EtagCachingStrategy; import be.vlaanderen.informatievlaanderen.ldes.server.rest.config.RestConfig; diff --git a/ldes-server-port-fetch-rest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/services/TreeNodeConverterImplTest.java b/ldes-server-port-fetch-rest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/services/TreeNodeConverterImplTest.java index 1fea6a552..8796515ea 100644 --- a/ldes-server-port-fetch-rest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/services/TreeNodeConverterImplTest.java +++ b/ldes-server-port-fetch-rest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/services/TreeNodeConverterImplTest.java @@ -8,6 +8,8 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.rest.PrefixConstructor; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.Member; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.TreeNode; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.TreeRelation; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.RDFNode; import org.apache.jena.rdf.model.Resource; diff --git a/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/entities/TreeNode.java b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/entities/TreeNode.java index d4d76dd47..e810136b0 100644 --- a/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/entities/TreeNode.java +++ b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/entities/TreeNode.java @@ -1,6 +1,6 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.TreeRelation; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.TreeRelation; import org.jetbrains.annotations.Nullable; import java.time.LocalDateTime; diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/exceptions/LdesFragmentIdentifierParseException.java b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/exceptions/LdesFragmentIdentifierParseException.java similarity index 80% rename from ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/exceptions/LdesFragmentIdentifierParseException.java rename to ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/exceptions/LdesFragmentIdentifierParseException.java index 1946482fa..5aead1a7d 100644 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/exceptions/LdesFragmentIdentifierParseException.java +++ b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/exceptions/LdesFragmentIdentifierParseException.java @@ -1,4 +1,4 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.exceptions; +package be.vlaanderen.informatievlaanderen.ldes.server.fetching.exceptions; public class LdesFragmentIdentifierParseException extends RuntimeException { diff --git a/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/repository/TreeNodeRepository.java b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/repository/TreeNodeRepository.java index d6f32016b..7413027bd 100644 --- a/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/repository/TreeNodeRepository.java +++ b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/repository/TreeNodeRepository.java @@ -1,6 +1,6 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fetching.repository; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentIdentifier; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.TreeNode; import java.util.Optional; diff --git a/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/StreamingTreeNodeFactory.java b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/StreamingTreeNodeFactory.java index ce5bf42d0..fb3714560 100644 --- a/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/StreamingTreeNodeFactory.java +++ b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/StreamingTreeNodeFactory.java @@ -1,6 +1,6 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fetching.services; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentIdentifier; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.Member; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.TreeNode; diff --git a/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/StreamingTreeNodeFactoryImpl.java b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/StreamingTreeNodeFactoryImpl.java index 6da3e6146..19381dd57 100644 --- a/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/StreamingTreeNodeFactoryImpl.java +++ b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/StreamingTreeNodeFactoryImpl.java @@ -1,7 +1,7 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fetching.services; import be.vlaanderen.informatievlaanderen.ldes.server.domain.exceptions.MissingResourceException; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentIdentifier; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.Member; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.TreeNode; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.repository.TreeMemberRepository; diff --git a/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/TreeNodeFetcher.java b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/TreeNodeFetcher.java index eb63bec2b..0d9cd50ed 100644 --- a/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/TreeNodeFetcher.java +++ b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/TreeNodeFetcher.java @@ -1,6 +1,6 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fetching.services; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentRequest; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentRequest; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.TreeNode; public interface TreeNodeFetcher { diff --git a/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/TreeNodeFetcherImpl.java b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/TreeNodeFetcherImpl.java index 3b17a44ab..203d5a5cb 100644 --- a/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/TreeNodeFetcherImpl.java +++ b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/TreeNodeFetcherImpl.java @@ -1,8 +1,8 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fetching.services; import be.vlaanderen.informatievlaanderen.ldes.server.domain.exceptions.MissingResourceException; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentRequest; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentRequest; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.TreeNode; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.repository.TreeNodeRepository; diff --git a/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/FragmentPair.java b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/FragmentPair.java new file mode 100644 index 000000000..fb05b79d0 --- /dev/null +++ b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/FragmentPair.java @@ -0,0 +1,4 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects; + +public record FragmentPair(String fragmentKey, String fragmentValue) { +} diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/LdesFragmentIdentifier.java b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/LdesFragmentIdentifier.java similarity index 73% rename from ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/LdesFragmentIdentifier.java rename to ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/LdesFragmentIdentifier.java index ae515f729..d3d97928a 100644 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/LdesFragmentIdentifier.java +++ b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/LdesFragmentIdentifier.java @@ -1,9 +1,12 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.model; +package be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.exceptions.LdesFragmentIdentifierParseException; -import org.springframework.data.annotation.PersistenceCreator; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.exceptions.LdesFragmentIdentifierParseException; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import java.util.*; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; import static java.net.URLEncoder.encode; @@ -14,7 +17,6 @@ public class LdesFragmentIdentifier { private final ViewName viewName; private final List fragmentPairs; - @PersistenceCreator public LdesFragmentIdentifier(ViewName viewName, List fragmentPairs) { this.viewName = viewName; this.fragmentPairs = fragmentPairs; @@ -25,20 +27,6 @@ public LdesFragmentIdentifier(String viewName, List fragmentPairs) this.fragmentPairs = fragmentPairs; } - public ViewName getViewName() { - return viewName; - } - - public List getFragmentPairs() { - return fragmentPairs; - } - - public Optional getValueOfFragmentPairKey(String key) { - return fragmentPairs.stream().filter(pair -> pair.fragmentKey().equals(key)) - .map(FragmentPair::fragmentValue) - .findFirst(); - } - public static LdesFragmentIdentifier fromFragmentId(String fragmentId) { try { String[] splitString = fragmentId.substring(1).split("\\?"); @@ -83,17 +71,6 @@ private String getFragmentId(boolean encoded) { return stringBuilder.toString(); } - public Optional getParentId() { - - if (!this.fragmentPairs.isEmpty()) { - List parentPairs = new ArrayList<>(fragmentPairs); - parentPairs.remove(parentPairs.size() - 1); - - return Optional.of(new LdesFragmentIdentifier(viewName, parentPairs)); - } - return Optional.empty(); - } - @Override public boolean equals(Object o) { if (this == o) diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/LdesFragmentRequest.java b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/LdesFragmentRequest.java similarity index 73% rename from ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/LdesFragmentRequest.java rename to ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/LdesFragmentRequest.java index 62c4d1a84..0eff39b04 100644 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/LdesFragmentRequest.java +++ b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/LdesFragmentRequest.java @@ -1,14 +1,12 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.model; +package be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import java.util.List; import java.util.Objects; public record LdesFragmentRequest(ViewName viewName, List fragmentPairs) { - public static LdesFragmentRequest createViewRequest(ViewName viewName) { - return new LdesFragmentRequest(viewName, List.of()); - } - @Override public boolean equals(Object o) { if (this == o) { diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/TreeRelation.java b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/TreeRelation.java similarity index 90% rename from ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/TreeRelation.java rename to ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/TreeRelation.java index 0dbe63ec1..f9166f35b 100644 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/model/TreeRelation.java +++ b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/TreeRelation.java @@ -1,4 +1,4 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.model; +package be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects; import java.util.Objects; diff --git a/ldes-server-port-fetch/src/main/java/module-info.java b/ldes-server-port-fetch/src/main/java/module-info.java index a277efe4b..5d665cae1 100644 --- a/ldes-server-port-fetch/src/main/java/module-info.java +++ b/ldes-server-port-fetch/src/main/java/module-info.java @@ -3,8 +3,9 @@ exports be.vlaanderen.informatievlaanderen.ldes.server.fetching.services; exports be.vlaanderen.informatievlaanderen.ldes.server.fetching.repository; exports be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities; + exports be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects; - requires ldes.domain; + requires ldes.domain; requires ldes.ingest.domain; requires ldes.fragmentation.domain; diff --git a/ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/StreamingTreeNodeFactoryImplTest.java b/ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/StreamingTreeNodeFactoryImplTest.java index 823612901..bb657b134 100644 --- a/ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/StreamingTreeNodeFactoryImplTest.java +++ b/ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/StreamingTreeNodeFactoryImplTest.java @@ -1,14 +1,13 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fetching.services; import be.vlaanderen.informatievlaanderen.ldes.server.domain.exceptions.MissingResourceException; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentPair; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.FragmentPair; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentIdentifier; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.Member; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.TreeNode; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.repository.TreeMemberRepository; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.repository.TreeNodeRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Fragment; import org.apache.jena.rdf.model.ModelFactory; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -43,8 +42,7 @@ void setUp() { void when_NoFragmentExists_ThenMissingResourceExceptionIsThrown() { LdesFragmentIdentifier id = new LdesFragmentIdentifier(VIEW_NAME, List.of(new FragmentPair(GENERATED_AT_TIME, FRAGMENTATION_VALUE_1))); - Fragment fragment = new Fragment(id); - Mockito.when(treeNodeRepository.findTreeNodeWithoutMembers(fragment.getFragmentId())) + Mockito.when(treeNodeRepository.findTreeNodeWithoutMembers(id)) .thenReturn(Optional.empty()); assertThatThrownBy(() -> streamingTreeNodeFactory.getFragmentWithoutMemberData(id)) diff --git a/ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/TreeNodeFetcherImplTest.java b/ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/TreeNodeFetcherImplTest.java index 23e90457d..a252a5dac 100644 --- a/ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/TreeNodeFetcherImplTest.java +++ b/ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/services/TreeNodeFetcherImplTest.java @@ -1,16 +1,14 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fetching.services; import be.vlaanderen.informatievlaanderen.ldes.server.domain.exceptions.MissingResourceException; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentPair; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentRequest; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.FragmentPair; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects.LdesFragmentRequest; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.TreeNode; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.repository.TreeNodeRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.Fragment; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.mockito.Mockito; import java.util.List; import java.util.Optional; @@ -39,10 +37,10 @@ void setUp() { void when_getFragment_WhenNoFragmentExists_ThenMissingResourceExceptionIsThrown() { LdesFragmentRequest ldesFragmentRequest = new LdesFragmentRequest(VIEW_NAME, List.of(new FragmentPair(GENERATED_AT_TIME, FRAGMENTATION_VALUE_1))); - Fragment fragment = new Fragment(new LdesFragmentIdentifier(ldesFragmentRequest.viewName(), - ldesFragmentRequest.fragmentPairs())); - Mockito.when(treeNodeRepository.findByFragmentIdentifier(fragment.getFragmentId())) - .thenThrow(new MissingResourceException("TreeNode", fragment.getFragmentIdString())); + LdesFragmentIdentifier ldesFragmentIdentifier = new LdesFragmentIdentifier(ldesFragmentRequest.viewName(), + ldesFragmentRequest.fragmentPairs()); + when(treeNodeRepository.findByFragmentIdentifier(ldesFragmentIdentifier)) + .thenThrow(new MissingResourceException("TreeNode", ldesFragmentIdentifier.asDecodedFragmentId())); assertThatThrownBy(() -> treeNodeFetcher.getFragment(ldesFragmentRequest)) .isInstanceOf(MissingResourceException.class) @@ -53,12 +51,12 @@ void when_getFragment_WhenNoFragmentExists_ThenMissingResourceExceptionIsThrown( void when_getFragment_WhenExactFragmentExists_ThenReturnThatFragment() { LdesFragmentRequest ldesFragmentRequest = new LdesFragmentRequest(VIEW_NAME, List.of(new FragmentPair(GENERATED_AT_TIME, FRAGMENTATION_VALUE_1))); - Fragment fragment = new Fragment(new LdesFragmentIdentifier(ldesFragmentRequest.viewName(), - ldesFragmentRequest.fragmentPairs())); - TreeNode treeNode = new TreeNode(fragment.getFragmentIdString(), true, false, List.of(), + LdesFragmentIdentifier ldesFragmentIdentifier = new LdesFragmentIdentifier(ldesFragmentRequest.viewName(), + ldesFragmentRequest.fragmentPairs()); + TreeNode treeNode = new TreeNode(ldesFragmentIdentifier.asDecodedFragmentId(), true, false, List.of(), List.of(), "collectionName", null); - when(treeNodeRepository.findByFragmentIdentifier(fragment.getFragmentId())).thenReturn(Optional.of(treeNode)); + when(treeNodeRepository.findByFragmentIdentifier(ldesFragmentIdentifier)).thenReturn(Optional.of(treeNode)); TreeNode returnedTreeNode = treeNodeFetcher.getFragment(ldesFragmentRequest); diff --git a/ldes-server-domain/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/ldesfragment/valueobjects/LdesFragmentIdentifierTest.java b/ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/LdesFragmentIdentifierTest.java similarity index 81% rename from ldes-server-domain/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/ldesfragment/valueobjects/LdesFragmentIdentifierTest.java rename to ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/LdesFragmentIdentifierTest.java index 54f4efc3f..3ce7b5ba9 100644 --- a/ldes-server-domain/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/ldesfragment/valueobjects/LdesFragmentIdentifierTest.java +++ b/ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/LdesFragmentIdentifierTest.java @@ -1,17 +1,15 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.ldesfragment.valueobjects; +package be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.exceptions.LdesFragmentIdentifierParseException; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentPair; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.exceptions.LdesFragmentIdentifierParseException; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.List; -import java.util.Optional; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; class LdesFragmentIdentifierTest { @@ -91,14 +89,4 @@ void when_NonRootFragmentIdentifier_Then_CreateEncodedFragmentIdString_withOnlyT assertEquals(encodedFragmentIdString, fragmentId.asEncodedFragmentId()); } - @Test - void when_KeyPresent_Then_ReturnKey() { - assertEquals(fragmentPairValue1, fragmentId.getValueOfFragmentPairKey(fragmentPairKey1).get()); - } - - @Test - void when_KeyNotPresent_Then_ReturnEmptyOptional() { - assertEquals(Optional.empty(), fragmentId.getValueOfFragmentPairKey("NotPresent")); - } - } \ No newline at end of file diff --git a/ldes-server-domain/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/ldesfragmentrequest/valueobjects/LdesFragmentRequestTest.java b/ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/LdesFragmentRequestTest.java similarity index 75% rename from ldes-server-domain/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/ldesfragmentrequest/valueobjects/LdesFragmentRequestTest.java rename to ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/LdesFragmentRequestTest.java index 3eeab7b98..1611bdca8 100644 --- a/ldes-server-domain/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/ldesfragmentrequest/valueobjects/LdesFragmentRequestTest.java +++ b/ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/LdesFragmentRequestTest.java @@ -1,7 +1,5 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.ldesfragmentrequest.valueobjects; +package be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentPair; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentRequest; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -14,10 +12,8 @@ import java.util.List; import java.util.stream.Stream; -import static org.apache.commons.lang3.ObjectUtils.isEmpty; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; class LdesFragmentRequestTest { @@ -51,16 +47,7 @@ void test_InequalityOfLdesFragmentRequests(Object otherLdesFragmentRequest) { assertNotEquals(ldesFragmentRequest, otherLdesFragmentRequest); } - @Test - void when_ViewRequestIsCreated_RequestHasViewNameAndEmptyList() { - final ViewName viewName = new ViewName("collection_name", "view_name"); - LdesFragmentRequest request = LdesFragmentRequest.createViewRequest(viewName); - assertEquals(viewName, request.viewName()); - assertTrue(isEmpty(request.fragmentPairs())); - } - - static class LdesFragmentRequestArgumentsProvider implements - ArgumentsProvider { + static class LdesFragmentRequestArgumentsProvider implements ArgumentsProvider { @Override public Stream provideArguments(ExtensionContext context) { diff --git a/ldes-server-domain/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/ldesfragment/valueobjects/TreeRelationTest.java b/ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/TreeRelationTest.java similarity index 86% rename from ldes-server-domain/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/ldesfragment/valueobjects/TreeRelationTest.java rename to ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/TreeRelationTest.java index 996c4c69f..7207a7099 100644 --- a/ldes-server-domain/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/ldesfragment/valueobjects/TreeRelationTest.java +++ b/ldes-server-port-fetch/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/valueobjects/TreeRelationTest.java @@ -1,7 +1,5 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.domain.ldesfragment.valueobjects; +package be.vlaanderen.informatievlaanderen.ldes.server.fetching.valueobjects; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.LdesFragmentIdentifier; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.TreeRelation; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtensionContext; @@ -12,8 +10,7 @@ import java.util.stream.Stream; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.*; class TreeRelationTest { @@ -66,4 +63,4 @@ public Stream provideArguments(ExtensionContext context) { } } -} +} \ No newline at end of file diff --git a/ldes-server-port-ingest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/MemberIngesterImpl.java b/ldes-server-port-ingest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/MemberIngesterImpl.java index 8d47cfece..36e2a307a 100644 --- a/ldes-server-port-ingest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/MemberIngesterImpl.java +++ b/ldes-server-port-ingest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/MemberIngesterImpl.java @@ -1,6 +1,5 @@ package be.vlaanderen.informatievlaanderen.ldes.server.ingest; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.ingest.MembersIngestedEvent; import be.vlaanderen.informatievlaanderen.ldes.server.domain.exceptions.MissingResourceException; import be.vlaanderen.informatievlaanderen.ldes.server.domain.services.ServerMetrics; import be.vlaanderen.informatievlaanderen.ldes.server.ingest.collection.MemberExtractorCollection; @@ -11,11 +10,9 @@ import org.apache.jena.rdf.model.Model; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import java.util.List; -import java.util.concurrent.CompletableFuture; import static be.vlaanderen.informatievlaanderen.ldes.server.ingest.constants.IngestConstants.DUPLICATE_MEMBERS_DETECTED; import static be.vlaanderen.informatievlaanderen.ldes.server.ingest.constants.IngestConstants.MEMBER_WITH_ID_INGESTED; @@ -24,18 +21,16 @@ public class MemberIngesterImpl implements MemberIngester { private final MemberIngestValidator validator; private final MemberRepository memberRepository; - private final ApplicationEventPublisher eventPublisher; private final MemberExtractorCollection memberExtractorCollection; private final ServerMetrics serverMetrics; private static final Logger log = LoggerFactory.getLogger(MemberIngesterImpl.class); public MemberIngesterImpl(MemberIngestValidator validator, MemberRepository memberRepository, - ApplicationEventPublisher eventPublisher, MemberExtractorCollection memberExtractorCollection, + MemberExtractorCollection memberExtractorCollection, ServerMetrics serverMetrics) { this.validator = validator; this.memberRepository = memberRepository; - this.eventPublisher = eventPublisher; this.memberExtractorCollection = memberExtractorCollection; this.serverMetrics = serverMetrics; } @@ -53,7 +48,6 @@ public boolean ingest(String collectionName, Model ingestedModel) { log.warn(DUPLICATE_MEMBERS_DETECTED); return false; } - publishIngestedEvent(collectionName, members); serverMetrics.incrementIngestCount(collectionName, ingestedMembersCount); members.forEach(member -> logSuccessfulMemberIngestion(member.getSubject())); return true; @@ -66,15 +60,6 @@ private List extractMembersFromModel(String collectionName, Mode return memberExtractor.extractMembers(model); } - private void publishIngestedEvent(String collectionName, List members) { - CompletableFuture.runAsync(() -> { - final List memberProperties = members.stream() - .map(member -> new MembersIngestedEvent.MemberProperties(member.getCollectionName() + "/" + member.getSubject(), member.getVersionOf(), member.getTimestamp())) - .toList(); - eventPublisher.publishEvent(new MembersIngestedEvent(collectionName, memberProperties)); - }); - } - private void logSuccessfulMemberIngestion(String memberId) { final String loggableMemberId = memberId.replaceAll("[\n\r\t]", "_"); log.debug(MEMBER_WITH_ID_INGESTED, loggableMemberId); diff --git a/ldes-server-port-ingest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/MemberIngesterImplTest.java b/ldes-server-port-ingest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/MemberIngesterImplTest.java index 4241d0430..daa83ebe0 100644 --- a/ldes-server-port-ingest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/MemberIngesterImplTest.java +++ b/ldes-server-port-ingest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/MemberIngesterImplTest.java @@ -1,6 +1,5 @@ package be.vlaanderen.informatievlaanderen.ldes.server.ingest; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.ingest.MembersIngestedEvent; import be.vlaanderen.informatievlaanderen.ldes.server.domain.services.FragmentationMetricsRepository; import be.vlaanderen.informatievlaanderen.ldes.server.domain.services.MemberMetricsRepository; import be.vlaanderen.informatievlaanderen.ldes.server.domain.services.ServerMetrics; @@ -25,7 +24,6 @@ import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.context.ApplicationEventPublisher; import java.time.LocalDateTime; import java.time.ZonedDateTime; @@ -34,7 +32,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; @ExtendWith(MockitoExtension.class) @@ -47,8 +44,6 @@ class MemberIngesterImplTest { @Mock private MemberRepository memberRepository; @Mock - private ApplicationEventPublisher eventPublisher; - @Mock private MemberIngestValidator validator; private MeterRegistry meterRegistry; private MemberIngester memberIngestService; @@ -60,7 +55,7 @@ void setUp() { MemberExtractorCollection memberExtractorCollection = new MemberExtractorCollectionImpl(); ServerMetrics serverMetrics = new ServerMetrics(mock(FragmentationMetricsRepository.class), mock(MemberMetricsRepository.class)); - memberIngestService = new MemberIngesterImpl(validator, memberRepository, eventPublisher, memberExtractorCollection, serverMetrics); + memberIngestService = new MemberIngesterImpl(validator, memberRepository, memberExtractorCollection, serverMetrics); final MemberExtractor memberExtractor = new VersionObjectMemberExtractor(COLLECTION_NAME, "http://purl.org/dc/terms/isVersionOf", "http://www.w3.org/ns/prov#generatedAtTime"); memberExtractorCollection.addMemberExtractor(COLLECTION_NAME, memberExtractor); @@ -82,7 +77,6 @@ void whenValidatorThrowsAnException_thenTheIngestIsAborted_andTheExceptionIsThro var counter = meterRegistry.find(ServerMetrics.INGEST).counter(); assertThat(counter).isNull(); verifyNoInteractions(memberRepository); - verifyNoInteractions(eventPublisher); } @Test @@ -101,7 +95,6 @@ void when_TheMemberAlreadyExists_thenEmptyOptionalIsReturned() { var counter = meterRegistry.find(ServerMetrics.INGEST).counter(); assertThat(counter).isNull(); verify(memberRepository, times(1)).insertAll(List.of(member)); - verifyNoInteractions(eventPublisher); } @Test @@ -120,9 +113,8 @@ void when_TheMemberDoesNotAlreadyExists_thenMemberIsStored() { Gauge counter = meterRegistry.find(ServerMetrics.INGEST).gauge(); assertThat(counter).isNotNull(); assertThat(counter.value()).isEqualTo(1); - InOrder inOrder = inOrder(memberRepository, eventPublisher); + InOrder inOrder = inOrder(memberRepository); inOrder.verify(memberRepository, times(1)).insertAll(List.of(member)); - inOrder.verify(eventPublisher).publishEvent(any(MembersIngestedEvent.class)); inOrder.verifyNoMoreInteractions(); }