From e2a72b83f3708c044dfe2beb299c3e1debad165f Mon Sep 17 00:00:00 2001 From: Richard Treier Date: Mon, 28 Aug 2023 15:21:25 +0200 Subject: [PATCH 01/14] feat: E2E Tests in Actions, ConnectorRemote, TestBackend, launchers (#489) --------- Co-authored-by: Eric Fiege --- launchers/Dockerfile | 34 ++++++++++++++++++++ launchers/README.md | 62 ++++++++++++++++++++++++++++++++++++ launchers/logging.properties | 7 ++++ 3 files changed, 103 insertions(+) create mode 100644 launchers/Dockerfile create mode 100644 launchers/README.md create mode 100644 launchers/logging.properties diff --git a/launchers/Dockerfile b/launchers/Dockerfile new file mode 100644 index 000000000..cd5595898 --- /dev/null +++ b/launchers/Dockerfile @@ -0,0 +1,34 @@ +FROM eclipse-temurin:17-jre-alpine + +# Install curl for healthcheck, bash for entrypoint +RUN apk add --no-cache curl bash +SHELL ["/bin/bash", "-c"] + +# Use a non-root user +RUN adduser -D -H -s /sbin/nologin edc +USER edc:edc + +# Which app.jar to include +ARG CONNECTOR_NAME="sovity-ce" + +# For last-commit-info extension +ARG EDC_LAST_COMMIT_INFO_ARG="The docker container was built outside of github actions and you didn't provide the build arg EDC_LAST_COMMIT_INFO_ARG, so there's no last commit info." +ARG EDC_BUILD_DATE_ARG="The docker container was built outside of github actions and you didn't provide the build arg EDC_BUILD_DATE_ARG, so there's no build date." + +WORKDIR /app +COPY ./launchers/connectors/$CONNECTOR_NAME/build/libs/app.jar /app +COPY ./launchers/logging.properties /app +COPY ./launchers/.env /app/.env +RUN touch /app/emtpy-properties-file.properties + +# Replaces var statements so when they are sourced as bash they don't overwrite existing env vars +RUN sed -ri 's/^\s*(\S+)=(.*)$/\1=${\1:-"\2"}/' .env + +ENV EDC_LAST_COMMIT_INFO=$EDC_LAST_COMMIT_INFO_ARG +ENV EDC_BUILD_DATE=$EDC_BUILD_DATE_ARG +ENV JVM_ARGS="" + +ENTRYPOINT set -a && source /app/.env && set +a && exec java -Djava.util.logging.config.file=/app/logging.properties $JVM_ARGS -jar app.jar + +# health status is determined by the availability of the /health endpoint +HEALTHCHECK --interval=5s --timeout=5s --retries=10 CMD curl --fail http://localhost:11001/api/check/health diff --git a/launchers/README.md b/launchers/README.md new file mode 100644 index 000000000..10de01301 --- /dev/null +++ b/launchers/README.md @@ -0,0 +1,62 @@ + +
+
+ + Logo + + +

sovity Community Edition EDC:
Docker Images

+ +

+ Report Bug + · + Request Feature +

+
+ +## sovity Community Edition EDC Docker Images + +[Eclipse Dataspace Components](https://github.com/eclipse-edc) (EDC) is a framework +for building dataspaces, exchanging data securely with ensured data +sovereignty. + +[sovity](https://sovity.de/) extends the EDC Connector's functionality with extensions to offer +enterprise-ready managed services like "Connector-as-a-Service", out-of-the-box fully configured DAPS +and integrations to existing other dataspace technologies. + +We believe in open source and actively contribute to open source community. Our sovity Community Edition EDC packages +open source EDC Extensions and combines them with [our own open source EDC Extensions](../extensions) to build +ready-to-use EDC Docker Images. + +Together with our [EDC UI](https://github.com/sovity/EDC-UI) Docker Images, it offers several of our extended EDC +functionalities for self-hosting purposes. + +## Different Image Types + +Our sovity Community Edition EDC is built as several docker image variants in different configurations. + +| Docker Image | Type | Purpose | Features | +|----------------------------------------------------------------------------------|--------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [edc-dev](https://github.com/sovity/edc-extensions/pkgs/container/edc-dev) | Devevelopment | | | +| [edc-ce](https://github.com/sovity/edc-extensions/pkgs/container/edc-ce) | sovity Community Edition | | | +| [edc-ce-mds](https://github.com/sovity/edc-extensions/pkgs/container/edc-ce-mds) | MDS Community Edition | | | +| edc-ee | Commercial | | | + +## Image Tags + +| Tag | Description | +|---------|----------------------------------------------------| +| latest | latest version of our main branch | +| release | latest release of our sovity Community Edition EDC | + +## Configuration + +For available configurations please refer to our [Getting Started Guide](../docs/getting-started/README.md). + +## License + +Apache License 2.0 - see [LICENSE](../../LICENSE) + +## Contact + +sovity GmbH - contact@sovity.de diff --git a/launchers/logging.properties b/launchers/logging.properties new file mode 100644 index 000000000..b4d12f28f --- /dev/null +++ b/launchers/logging.properties @@ -0,0 +1,7 @@ +handlers = java.util.logging.ConsoleHandler +.level = INFO +java.util.logging.ConsoleHandler.level = ALL +java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter +java.util.logging.SimpleFormatter.format = %1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS %5$s %6$s%n +org.eclipse.dataspaceconnector.level = FINE +org.eclipse.dataspaceconnector.handler = java.util.logging.ConsoleHandler From 8b6f1152e8894da860658964b34b2d708b39ddfd Mon Sep 17 00:00:00 2001 From: Richard Treier Date: Thu, 7 Sep 2023 17:59:18 +0200 Subject: [PATCH 02/14] docs: reworked deployment documentation (#517) --- launchers/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launchers/README.md b/launchers/README.md index 10de01301..dd9a0862f 100644 --- a/launchers/README.md +++ b/launchers/README.md @@ -55,7 +55,7 @@ For available configurations please refer to our [Getting Started Guide](../docs ## License -Apache License 2.0 - see [LICENSE](../../LICENSE) +Apache License 2.0 - see [LICENSE](../LICENSE) ## Contact From fa5c38cd8a0f9c1729495e120aae8e56047860b4 Mon Sep 17 00:00:00 2001 From: Richard Treier Date: Fri, 13 Oct 2023 14:58:48 +0200 Subject: [PATCH 03/14] chore: prepare 5.0.0 release (#564) * chore: prepare release * fix: openapi.yaml, postman_collection.json Test Connector Remote for Eclipse EDC 0.2.1 * docs: fix API Wrapper documentation and client examples * docs: create deployment guides for 5.0.0 * chore: bump versions for local demo docker compose * chore: fix CHANGELOG.md structure * feat: add custom auth possibility to the java client * docs: fix asset creation and transfer initiation in docs * docs: migrate docs using the management api --- launchers/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/launchers/README.md b/launchers/README.md index dd9a0862f..cb6363027 100644 --- a/launchers/README.md +++ b/launchers/README.md @@ -37,9 +37,9 @@ Our sovity Community Edition EDC is built as several docker image variants in di | Docker Image | Type | Purpose | Features | |----------------------------------------------------------------------------------|--------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [edc-dev](https://github.com/sovity/edc-extensions/pkgs/container/edc-dev) | Devevelopment |
  • Lightweight local development
  • Used in EDC UI's Getting Started section
|
  • Control- and Data-Plane
  • sovity Community Edition EDC Extensions
  • Management API Auth via API Keys
  • Mock IAM
| -| [edc-ce](https://github.com/sovity/edc-extensions/pkgs/container/edc-ce) | sovity Community Edition |
  • Test sovity EDC features
  • Self-Deploy EDC
|
  • Control- and Data-Plane
  • sovity Community Edition EDC Extensions
  • Management API Auth via API Keys
  • DAPS Authentication
  • PostgreSQL Persistence & Flyway
| -| [edc-ce-mds](https://github.com/sovity/edc-extensions/pkgs/container/edc-ce-mds) | MDS Community Edition |
  • Demonstrate MDS EDC Features
  • Self-Deploy MDS EDC
|
  • Control- and Data-Plane
  • sovity Community Edition EDC Extensions
  • Management API Auth via API Keys
  • DAPS Authentication
  • PostgreSQL Persistence & Flyway
  • Broker Extension
  • Clearing House Extension
| +| [edc-dev](https://github.com/sovity/edc-extensions/pkgs/container/edc-dev) | Devevelopment |
  • Local manual testing
  • Local demos
|
  • Control- and Data-Plane
  • sovity Community Edition EDC Extensions
  • Management API Auth via API Keys
  • PostgreSQL Persistence & Flyway
  • Mock IAM
| +| [edc-ce](https://github.com/sovity/edc-extensions/pkgs/container/edc-ce) | sovity Community Edition |
  • Self-Deploy a productive sovity EDC
|
  • Control- and Data-Plane
  • sovity Community Edition EDC Extensions
  • Management API Auth via API Keys
  • PostgreSQL Persistence & Flyway
  • DAPS Authentication
| +| [edc-ce-mds](https://github.com/sovity/edc-extensions/pkgs/container/edc-ce-mds) | MDS Community Edition |
  • Self-Deploy a productive MDS EDC
|
  • Control- and Data-Plane
  • sovity Community Edition EDC Extensions
  • Management API Auth via API Keys
  • PostgreSQL Persistence & Flyway
  • DAPS Authentication
  • Broker Extension
  • Clearing House Extension
| | edc-ee | Commercial |
  • Productive use
  • Professional users
  • Our Connector-as-a-Service (CaaS) customers
  • [Request Demo](mailto:contact@sovity.de)
|
  • Managed Control- and Data Planes, individually scalable
  • Hosted on highly performant infrastructure
  • Management API Auth via Service Accounts
  • Managed User Auth via standalone IAM (SSO)
  • Automatic Dataspace Roll-In, for example to Data Spaces like Catena-X or Mobility Data Space
  • Managed DAPS Authentication
  • Support & Tutorials
  • Automatic updates to newest version and new features
  • Off-the-shelf extensions for use cases available
  • EDC available within minutes
  • Can be combined with Data Space as a Service (DSaaS)
| ## Image Tags From 267dd8c99d2716b07480afa61e428ed26436a41b Mon Sep 17 00:00:00 2001 From: Kamil Czaja <46053356+kamilczaja@users.noreply.github.com> Date: Tue, 26 Mar 2024 17:35:33 +0100 Subject: [PATCH 04/14] feat: debug logging and remote debugging (#790) --- launchers/Dockerfile | 5 +++- launchers/docker-entrypoint.sh | 43 ++++++++++++++++++++++++++++++++ launchers/logging.dev.properties | 7 ++++++ 3 files changed, 54 insertions(+), 1 deletion(-) create mode 100755 launchers/docker-entrypoint.sh create mode 100644 launchers/logging.dev.properties diff --git a/launchers/Dockerfile b/launchers/Dockerfile index cd5595898..a4dbda971 100644 --- a/launchers/Dockerfile +++ b/launchers/Dockerfile @@ -18,6 +18,7 @@ ARG EDC_BUILD_DATE_ARG="The docker container was built outside of github actions WORKDIR /app COPY ./launchers/connectors/$CONNECTOR_NAME/build/libs/app.jar /app COPY ./launchers/logging.properties /app +COPY ./launchers/logging.dev.properties /app COPY ./launchers/.env /app/.env RUN touch /app/emtpy-properties-file.properties @@ -28,7 +29,9 @@ ENV EDC_LAST_COMMIT_INFO=$EDC_LAST_COMMIT_INFO_ARG ENV EDC_BUILD_DATE=$EDC_BUILD_DATE_ARG ENV JVM_ARGS="" -ENTRYPOINT set -a && source /app/.env && set +a && exec java -Djava.util.logging.config.file=/app/logging.properties $JVM_ARGS -jar app.jar +COPY ./launchers/docker-entrypoint.sh /app/entrypoint.sh +ENTRYPOINT ["/app/entrypoint.sh"] +CMD ["start"] # health status is determined by the availability of the /health endpoint HEALTHCHECK --interval=5s --timeout=5s --retries=10 CMD curl --fail http://localhost:11001/api/check/health diff --git a/launchers/docker-entrypoint.sh b/launchers/docker-entrypoint.sh new file mode 100755 index 000000000..1f463be66 --- /dev/null +++ b/launchers/docker-entrypoint.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Use bash instead of sh, +# because sh in this image is provided by dash (https://git.kernel.org/pub/scm/utils/dash/dash.git/), +# which seems to eat environment variables containing dashes, +# which are required for some EDC configuration values. + +# Do not set -u to permit unset variables in .env +set -eo pipefail + +# Apply ENV Vars on JAR startup +set -a +source /app/.env +set +a + + +if [[ "x${1:-}" == "xstart" ]]; then + cmd=(java ${JAVA_ARGS:-}) + + if [ "${REMOTE_DEBUG:-n}" = "y" ] || [ "${REMOTE_DEBUG:-false}" = "true" ]; then + cmd+=( + "-agentlib:jdwp=transport=dt_socket,server=y,suspend=${REMOTE_DEBUG_SUSPEND:-n},address=${REMOTE_DEBUG_BIND:-127.0.0.1:5005}" + ) + fi + + logging_config='/app/logging.properties' + if [ "${DEBUG_LOGGING:-n}" = "y" ] || [ "${DEBUG_LOGGING:-false}" = "true" ]; then + logging_config='/app/logging.dev.properties' + fi + + cmd+=( + -Djava.util.logging.config.file=${logging_config} + -jar /app/app.jar + ) +else + cmd=("$@") +fi + +if [ "${REMOTE_DEBUG:-n}" = "y" ] || [ "${REMOTE_DEBUG:-false}" = "true" ]; then + echo "Jar CMD (printing, because REMOTE_DEBUG=y|true): ${cmd[@]}" +fi + +# Use "exec" for termination signals to reach JVM +exec "${cmd[@]}" diff --git a/launchers/logging.dev.properties b/launchers/logging.dev.properties new file mode 100644 index 000000000..3db949d79 --- /dev/null +++ b/launchers/logging.dev.properties @@ -0,0 +1,7 @@ +handlers = java.util.logging.ConsoleHandler +.level = FINE +java.util.logging.ConsoleHandler.level = ALL +java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter +java.util.logging.SimpleFormatter.format = %1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS %5$s %6$s%n +org.eclipse.dataspaceconnector.level = FINE +org.eclipse.dataspaceconnector.handler = java.util.logging.ConsoleHandler From 04e8711c2bd7c74065880d0ecf7f458329d1d0d0 Mon Sep 17 00:00:00 2001 From: Christophe Loiseau Date: Mon, 27 May 2024 15:33:10 +0200 Subject: [PATCH 05/14] Integrate the broker's extensions (#921) * Add accidentally deleted SUMMARY * chore: Apply new config style * Code review * Merge docs * Merge trivial templates changes * Merge more conflicting files * Merge more conflicting files * Merge more conflicting files and cleanup old versions * Add connector submodule * Merge more conflicting files * Move broker CHANGELOG * Move the broker's README * More cleaning * Fix links * Prepare docker build to merge the broker * Merge the docker build * Fix links and format * Add connector type info * self review * Convert MD table to html * Only use test containers for the Broker * Add broker NPM delivery * code review * Add broker local dev doc * Merge deployment READMEs and change healthcheck header * Fix links * Cleanup * Fix javadocs and add -sources and -javadocs artefacts * Rename .env.extensions to .env.connector * Rename IDS broker -> broker * Remove redundant .editorconfig setting * Change connector-type -> deployment-type in workflow config * NPM clean install * Removed mention of the broker in the main readme * Dry postgres image name * Remove test DB deprecation * Move broker group to toml * Fix missing broker's groups * More versions cleanup * Change MD for HTML * Merge the broker compose file * https://github.com/sovity/edc-extensions/pull/921#discussion_r1604695043 * Merge local quick start for broker * Align postgres versions * Stop annoying me with DB not ready messages on new startup * Wait for the DBs * Code review * remove docs task. The repo is now public * Avoid star imports * Remove reference to the merged doc --- launchers/Dockerfile | 8 +- launchers/README.md | 141 +++++++++++++++++++++++++++++++++-- launchers/logging.properties | 1 + 3 files changed, 141 insertions(+), 9 deletions(-) diff --git a/launchers/Dockerfile b/launchers/Dockerfile index a4dbda971..b1e492b9b 100644 --- a/launchers/Dockerfile +++ b/launchers/Dockerfile @@ -10,6 +10,7 @@ USER edc:edc # Which app.jar to include ARG CONNECTOR_NAME="sovity-ce" +ARG CONNECTOR_TYPE="connector" # For last-commit-info extension ARG EDC_LAST_COMMIT_INFO_ARG="The docker container was built outside of github actions and you didn't provide the build arg EDC_LAST_COMMIT_INFO_ARG, so there's no last commit info." @@ -19,8 +20,9 @@ WORKDIR /app COPY ./launchers/connectors/$CONNECTOR_NAME/build/libs/app.jar /app COPY ./launchers/logging.properties /app COPY ./launchers/logging.dev.properties /app -COPY ./launchers/.env /app/.env -RUN touch /app/emtpy-properties-file.properties +COPY ./launchers/.env.$CONNECTOR_TYPE /app/.env + +RUN touch /app/empty-properties-file.properties # Replaces var statements so when they are sourced as bash they don't overwrite existing env vars RUN sed -ri 's/^\s*(\S+)=(.*)$/\1=${\1:-"\2"}/' .env @@ -34,4 +36,4 @@ ENTRYPOINT ["/app/entrypoint.sh"] CMD ["start"] # health status is determined by the availability of the /health endpoint -HEALTHCHECK --interval=5s --timeout=5s --retries=10 CMD curl --fail http://localhost:11001/api/check/health +HEALTHCHECK --interval=5s --timeout=5s --retries=10 CMD curl -H "x-api-key: $EDC_API_AUTH_KEY" --fail http://localhost:11001/backend/api/check/health diff --git a/launchers/README.md b/launchers/README.md index cb6363027..02d235771 100644 --- a/launchers/README.md +++ b/launchers/README.md @@ -35,12 +35,141 @@ functionalities for self-hosting purposes. Our sovity Community Edition EDC is built as several docker image variants in different configurations. -| Docker Image | Type | Purpose | Features | -|----------------------------------------------------------------------------------|--------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [edc-dev](https://github.com/sovity/edc-extensions/pkgs/container/edc-dev) | Devevelopment |
  • Local manual testing
  • Local demos
|
  • Control- and Data-Plane
  • sovity Community Edition EDC Extensions
  • Management API Auth via API Keys
  • PostgreSQL Persistence & Flyway
  • Mock IAM
| -| [edc-ce](https://github.com/sovity/edc-extensions/pkgs/container/edc-ce) | sovity Community Edition |
  • Self-Deploy a productive sovity EDC
|
  • Control- and Data-Plane
  • sovity Community Edition EDC Extensions
  • Management API Auth via API Keys
  • PostgreSQL Persistence & Flyway
  • DAPS Authentication
| -| [edc-ce-mds](https://github.com/sovity/edc-extensions/pkgs/container/edc-ce-mds) | MDS Community Edition |
  • Self-Deploy a productive MDS EDC
|
  • Control- and Data-Plane
  • sovity Community Edition EDC Extensions
  • Management API Auth via API Keys
  • PostgreSQL Persistence & Flyway
  • DAPS Authentication
  • Broker Extension
  • Clearing House Extension
| -| edc-ee | Commercial |
  • Productive use
  • Professional users
  • Our Connector-as-a-Service (CaaS) customers
  • [Request Demo](mailto:contact@sovity.de)
|
  • Managed Control- and Data Planes, individually scalable
  • Hosted on highly performant infrastructure
  • Management API Auth via Service Accounts
  • Managed User Auth via standalone IAM (SSO)
  • Automatic Dataspace Roll-In, for example to Data Spaces like Catena-X or Mobility Data Space
  • Managed DAPS Authentication
  • Support & Tutorials
  • Automatic updates to newest version and new features
  • Off-the-shelf extensions for use cases available
  • EDC available within minutes
  • Can be combined with Data Space as a Service (DSaaS)
| + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Docker ImageTypePurposeFeatures
+ edc-dev + Development +
    +
  • Local manual testing
    • +
  • Local demos
    • +
+ 		+
    +
  • Control- and Data-Plane
    • +
  • sovity Community Edition EDC Extensions
    • +
  • Management API Auth via API Keys
    • +
  • PostgreSQL Persistence & Flyway
    • +
  • Mock IAM
    • +
+
+ edc-ce + sovity Community Edition +
    +
  • Self-Deploy a productive sovity EDC
    • +
+ 		+
    +
  • Control- and Data-Plane
    • +
  • sovity Community Edition EDC Extensions
    • +
  • Management API Auth via API Keys
    • +
  • PostgreSQL Persistence & Flyway
    • +
  • DAPS Authentication
    • +
+
+ edc-ce-mds + MDS Community Edition +
    +
  • Self-Deploy a productive MDS EDC
    • +
+ 		+
    +
  • Control- and Data-Plane
    • +
  • sovity Community Edition EDC Extensions
    • +
  • Management API Auth via API Keys
    • +
  • PostgreSQL Persistence & Flyway
    • +
  • DAPS Authentication
    • +
  • Broker Extension
    • +
  • Clearing House Extension
    • +
+
edc-eeCommercial +
    +
  • Productive use
    • +
  • Professional users
    • +
  • Our Connector-as-a-Service (CaaS) customers
    • +
  • Request Demo +
+ 		+
    +
  • Managed Control- and Data Planes, individually scalable
    • +
  • Hosted on highly performant infrastructure
    • +
  • Management API Auth via Service Accounts
    • +
  • Managed User Auth via standalone IAM (SSO)
    • +
  • Automatic Dataspace Roll-In, for example to Data Spaces like Catena-X or Mobility Data Space
    • +
  • Managed DAPS Authentication
    • +
  • Support & Tutorials
    • +
  • Automatic updates to newest version and new features
    • +
  • Off-the-shelf extensions for use cases available
    • +
  • EDC available within minutes
    • +
  • Can be combined with Data Space as a Service (DSaaS)
    • +
+
+ broker-dev + Development +
    +
  • Local Demo via our + docker-compose.yaml +
    • +
  • E2E Testing
    • +
+ 		+
    +
  • Broker Server Extension(s)
    • +
  • PostgreSQL Persistence & Flyway
    • +
  • Mock IAM
    • +
+
broker-ceCommunity Edition +
    +
  • Productive Deployment
    • +
+ 		+
    +
  • Broker Server Extension(s)
    • +
  • PostgreSQL Persistence & Flyway
    • +
  • DAPS Authentication
    • +
+
## Image Tags diff --git a/launchers/logging.properties b/launchers/logging.properties index b4d12f28f..17dfd8a75 100644 --- a/launchers/logging.properties +++ b/launchers/logging.properties @@ -5,3 +5,4 @@ java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter java.util.logging.SimpleFormatter.format = %1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS %5$s %6$s%n org.eclipse.dataspaceconnector.level = FINE org.eclipse.dataspaceconnector.handler = java.util.logging.ConsoleHandler +org.eclipse.edc.api.observability.ObservabilityApiController.level = ERROR From 6c379ac47fedebe5d91115b52f7f10821044fcb2 Mon Sep 17 00:00:00 2001 From: Christophe Loiseau Date: Thu, 6 Jun 2024 09:42:17 +0200 Subject: [PATCH 06/14] fix: health indicator and version (#962) * change the health indicator to be compatible with both the broker and the extensions * Use the latest EDC version --- launchers/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launchers/Dockerfile b/launchers/Dockerfile index b1e492b9b..06d01939c 100644 --- a/launchers/Dockerfile +++ b/launchers/Dockerfile @@ -36,4 +36,4 @@ ENTRYPOINT ["/app/entrypoint.sh"] CMD ["start"] # health status is determined by the availability of the /health endpoint -HEALTHCHECK --interval=5s --timeout=5s --retries=10 CMD curl -H "x-api-key: $EDC_API_AUTH_KEY" --fail http://localhost:11001/backend/api/check/health +HEALTHCHECK --interval=5s --timeout=5s --retries=10 CMD curl -H "x-api-key: $EDC_API_AUTH_KEY" --fail http://localhost:11001/api/check/health || curl -H "x-api-key: $EDC_API_AUTH_KEY" --fail http://localhost:11001/backend/api/check/health From ee6cd13402f69bd0fc0f6dfde9fc648b8983ce07 Mon Sep 17 00:00:00 2001 From: Richard Treier Date: Wed, 26 Jun 2024 16:41:47 +0200 Subject: [PATCH 07/14] feat: on request data offers, structured HttpData data sources via API (#983) --- launchers/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/launchers/README.md b/launchers/README.md index 02d235771..9a1f60f3d 100644 --- a/launchers/README.md +++ b/launchers/README.md @@ -1,16 +1,16 @@
- + Logo

sovity Community Edition EDC:
Docker Images

- Report Bug + Report Bug · - Request Feature + Request Feature

@@ -44,7 +44,7 @@ Our sovity Community Edition EDC is built as several docker image variants in di - edc-dev + edc-dev Development @@ -65,7 +65,7 @@ Our sovity Community Edition EDC is built as several docker image variants in di - edc-ce + edc-ce sovity Community Edition @@ -85,7 +85,7 @@ Our sovity Community Edition EDC is built as several docker image variants in di - edc-ce-mds + edc-ce-mds MDS Community Edition From f71962ff74835d24d6afcf07c1907a2e1c067405 Mon Sep 17 00:00:00 2001 From: Kamil Czaja <46053356+kamilczaja@users.noreply.github.com> Date: Tue, 9 Jul 2024 13:25:27 +0200 Subject: [PATCH 08/14] feat: authority portal catalog crawler (#985) Co-authored-by: Richard Treier --- extensions/catalog-crawler/README.md | 39 +++ .../build.gradle.kts | 19 ++ .../catalog-crawler/build.gradle.kts | 49 +++ .../ext/catalog/crawler/CrawlerExtension.java | 147 +++++++++ .../crawler/CrawlerExtensionContext.java | 42 +++ .../CrawlerExtensionContextBuilder.java | 312 ++++++++++++++++++ .../catalog/crawler/CrawlerInitializer.java | 27 ++ .../crawler/crawling/ConnectorCrawler.java | 90 +++++ .../crawling/OfflineConnectorCleaner.java | 42 +++ .../fetching/FetchedCatalogBuilder.java | 102 ++++++ .../fetching/FetchedCatalogMappingUtils.java | 56 ++++ .../fetching/FetchedCatalogService.java | 41 +++ .../fetching/model/FetchedCatalog.java | 34 ++ .../fetching/model/FetchedContractOffer.java | 28 ++ .../fetching/model/FetchedDataOffer.java | 36 ++ .../logging/ConnectorChangeTracker.java | 69 ++++ .../logging/CrawlerEventErrorMessage.java | 43 +++ .../crawling/logging/CrawlerEventLogger.java | 127 +++++++ .../logging/CrawlerExecutionTimeLogger.java | 43 +++ .../crawling/writing/CatalogPatchBuilder.java | 139 ++++++++ .../writing/ConnectorUpdateCatalogWriter.java | 53 +++ .../writing/ConnectorUpdateFailureWriter.java | 53 +++ .../writing/ConnectorUpdateSuccessWriter.java | 71 ++++ .../writing/DataOfferLimitsEnforcer.java | 102 ++++++ .../crawling/writing/utils/ChangeTracker.java | 36 ++ .../crawling/writing/utils/DiffUtils.java | 96 ++++++ .../catalog/crawler/dao/CatalogCleaner.java | 39 +++ .../ext/catalog/crawler/dao/CatalogPatch.java | 42 +++ .../crawler/dao/CatalogPatchApplier.java | 46 +++ .../crawler/dao/config/DataSourceFactory.java | 63 ++++ .../crawler/dao/config/DslContextFactory.java | 67 ++++ .../crawler/dao/config/FlywayService.java | 54 +++ .../dao/connectors/ConnectorQueries.java | 73 ++++ .../crawler/dao/connectors/ConnectorRef.java | 32 ++ .../connectors/ConnectorStatusUpdater.java | 34 ++ .../contract_offers/ContractOfferQueries.java | 29 ++ .../ContractOfferRecordUpdater.java | 83 +++++ .../dao/data_offers/DataOfferQueries.java | 31 ++ .../data_offers/DataOfferRecordUpdater.java | 157 +++++++++ .../catalog/crawler/dao/utils/JsonbUtils.java | 36 ++ .../crawler/dao/utils/PostgresqlUtils.java | 41 +++ .../crawler/dao/utils/RecordPatch.java | 46 +++ .../orchestration/config/CrawlerConfig.java | 32 ++ .../config/CrawlerConfigFactory.java | 54 +++ .../config/EdcConfigPropertyUtils.java | 39 +++ .../orchestration/queue/ConnectorQueue.java | 48 +++ .../queue/ConnectorQueueFiller.java | 31 ++ .../queue/ConnectorRefreshPriority.java | 24 ++ .../orchestration/queue/ThreadPool.java | 68 ++++ .../orchestration/queue/ThreadPoolTask.java | 45 +++ .../queue/ThreadPoolTaskQueue.java | 50 +++ .../schedules/DeadConnectorRefreshJob.java | 35 ++ .../schedules/OfflineConnectorCleanerJob.java | 32 ++ .../schedules/OfflineConnectorRefreshJob.java | 35 ++ .../schedules/OnlineConnectorRefreshJob.java | 35 ++ .../schedules/QuartzScheduleInitializer.java | 71 ++++ .../schedules/utils/CronJobRef.java | 39 +++ .../schedules/utils/JobFactoryImpl.java | 50 +++ .../crawler/utils/CollectionUtils2.java | 52 +++ .../ext/catalog/crawler/utils/JsonUtils2.java | 32 ++ .../ext/catalog/crawler/utils/MapUtils.java | 32 ++ .../catalog/crawler/utils/StringUtils2.java | 37 +++ ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + .../ext/catalog/crawler/AssertionUtils.java | 33 ++ .../ext/catalog/crawler/CrawlerTestDb.java | 63 ++++ .../ext/catalog/crawler/JsonTestUtils.java | 23 ++ .../edc/ext/catalog/crawler/TestData.java | 66 ++++ .../logging/CrawlerEventLoggerTest.java | 58 ++++ .../ConnectorUpdateCatalogWriterTest.java | 154 +++++++++ .../writing/DataOfferLimitsEnforcerTest.java | 218 ++++++++++++ .../DataOfferWriterTestDataHelper.java | 142 ++++++++ .../DataOfferWriterTestDataModels.java | 62 ++++ .../writing/DataOfferWriterTestDydi.java | 48 +++ .../DataOfferWriterTestResultHelper.java | 56 ++++ .../crawling/writing/DiffUtilsTest.java | 41 +++ .../dao/connectors/ConnectorRefTest.java | 67 ++++ .../queue/ThreadPoolQueueTest.java | 55 +++ .../OfflineConnectorRemovalJobTest.java | 101 ++++++ .../crawler/utils/CollectionUtils2Test.java | 44 +++ .../catalog/crawler/utils/JsonUtils2Test.java | 34 ++ .../crawler/utils/StringUtils2Test.java | 44 +++ .../src/test/resources/logging.properties | 6 + launchers/.env.catalog-crawler | 88 +++++ launchers/README.md | 15 +- .../catalog-crawler-ce/build.gradle.kts | 23 ++ .../catalog-crawler-dev/build.gradle.kts | 23 ++ 86 files changed, 5065 insertions(+), 10 deletions(-) create mode 100644 extensions/catalog-crawler/README.md create mode 100644 extensions/catalog-crawler/catalog-crawler-launcher-base/build.gradle.kts create mode 100644 extensions/catalog-crawler/catalog-crawler/build.gradle.kts create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContext.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerInitializer.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/ConnectorCrawler.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/OfflineConnectorCleaner.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogBuilder.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogMappingUtils.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogService.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedCatalog.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedContractOffer.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedDataOffer.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/ConnectorChangeTracker.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventErrorMessage.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventLogger.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerExecutionTimeLogger.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/CatalogPatchBuilder.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateCatalogWriter.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateFailureWriter.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateSuccessWriter.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferLimitsEnforcer.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/utils/ChangeTracker.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/utils/DiffUtils.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogCleaner.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogPatch.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogPatchApplier.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/DataSourceFactory.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/DslContextFactory.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/FlywayService.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorQueries.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorRef.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorStatusUpdater.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/contract_offers/ContractOfferQueries.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/contract_offers/ContractOfferRecordUpdater.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/data_offers/DataOfferQueries.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/data_offers/DataOfferRecordUpdater.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/utils/JsonbUtils.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/utils/PostgresqlUtils.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/utils/RecordPatch.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/CrawlerConfig.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/CrawlerConfigFactory.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/EdcConfigPropertyUtils.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorQueue.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorQueueFiller.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorRefreshPriority.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPool.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPoolTask.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPoolTaskQueue.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/DeadConnectorRefreshJob.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorCleanerJob.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorRefreshJob.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OnlineConnectorRefreshJob.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/QuartzScheduleInitializer.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/utils/CronJobRef.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/utils/JobFactoryImpl.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/CollectionUtils2.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/JsonUtils2.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/MapUtils.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/StringUtils2.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension create mode 100644 extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/AssertionUtils.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerTestDb.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/JsonTestUtils.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/TestData.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventLoggerTest.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateCatalogWriterTest.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferLimitsEnforcerTest.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDataHelper.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDataModels.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDydi.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestResultHelper.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DiffUtilsTest.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorRefTest.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPoolQueueTest.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorRemovalJobTest.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/CollectionUtils2Test.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/JsonUtils2Test.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/StringUtils2Test.java create mode 100644 extensions/catalog-crawler/catalog-crawler/src/test/resources/logging.properties create mode 100644 launchers/.env.catalog-crawler create mode 100644 launchers/connectors/catalog-crawler-ce/build.gradle.kts create mode 100644 launchers/connectors/catalog-crawler-dev/build.gradle.kts diff --git a/extensions/catalog-crawler/README.md b/extensions/catalog-crawler/README.md new file mode 100644 index 000000000..28de136de --- /dev/null +++ b/extensions/catalog-crawler/README.md @@ -0,0 +1,39 @@ + +
+
+ + Logo + + +

EDC-Connector Extension:
Catalog Crawler

+ +

+ Report Bug + · + Request Feature +

+
+ +## About this Extension + +The catalog crawler is a deployment unit depending on an existing Authority Portal's database: + +- It periodically checks the Authority Portal's connector list for its environment. +- It crawls the given connectors in regular intervals. +- It writes the data offers and connector statuses back into the Authority Portal DB. +- Each Environment configured in the Authority Portal requires its own Catalog Crawler with credentials for that environment's DAPS. + +## Why does this component exist? + +The Authority Portal uses a non-EDC stack, and the EDC stack cannot handle multiple sources of authority at once. + +With the `DB -> UI` part of the broker having been moved to the Authority Portal, only the `Catalog -> DB` part remains as the Catalog Crawler, +as it requires Connector-to-Connector IAM within the given Dataspace. + +## License + +Apache License 2.0 - see [LICENSE](../../LICENSE) + +## Contact + +sovity GmbH - contact@sovity.de diff --git a/extensions/catalog-crawler/catalog-crawler-launcher-base/build.gradle.kts b/extensions/catalog-crawler/catalog-crawler-launcher-base/build.gradle.kts new file mode 100644 index 000000000..2c376a085 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler-launcher-base/build.gradle.kts @@ -0,0 +1,19 @@ +plugins { + `java-library` +} + +dependencies { + // A minimal EDC that can request catalogs + api(libs.edc.controlPlaneCore) + api(libs.edc.dataPlaneSelectorCore) + api(libs.edc.configurationFilesystem) + api(libs.edc.controlPlaneAggregateServices) + api(libs.edc.http) + api(libs.edc.dsp) + api(libs.edc.jsonLd) + + // Data Catalog Crawler + api(project(":extensions:catalog-crawler:catalog-crawler")) +} + +group = libs.versions.sovityEdcGroup.get() diff --git a/extensions/catalog-crawler/catalog-crawler/build.gradle.kts b/extensions/catalog-crawler/catalog-crawler/build.gradle.kts new file mode 100644 index 000000000..b63b7004f --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/build.gradle.kts @@ -0,0 +1,49 @@ +plugins { + `java-library` +} + +dependencies { + annotationProcessor(libs.lombok) + compileOnly(libs.lombok) + + implementation(libs.edc.controlPlaneSpi) + implementation(libs.edc.managementApiConfiguration) + + implementation(libs.quartz.quartz) + implementation(libs.apache.commonsLang) + implementation(project(":utils:versions")) + + api(project(":utils:catalog-parser")) + api(project(":utils:json-and-jsonld-utils")) + api(project(":extensions:wrapper:wrapper-common-mappers")) + api(project(":extensions:catalog-crawler:catalog-crawler-db")) + api(project(":extensions:postgres-flyway-core")) + + testAnnotationProcessor(libs.lombok) + testCompileOnly(libs.lombok) + testImplementation(project(":utils:test-connector-remote")) + testImplementation(libs.assertj.core) + testImplementation(libs.mockito.core) + testImplementation(libs.mockito.inline) + testImplementation(libs.restAssured.restAssured) + testImplementation(libs.testcontainers.testcontainers) + testImplementation(libs.flyway.core) + testImplementation(libs.testcontainers.junitJupiter) + testImplementation(libs.testcontainers.postgresql) + testImplementation(libs.junit.api) + testImplementation(libs.jsonAssert) + testRuntimeOnly(libs.junit.engine) +} + +tasks.getByName("test") { + useJUnitPlatform() + maxParallelForks = 1 +} + +publishing { + publications { + create(project.name) { + from(components["java"]) + } + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java new file mode 100644 index 000000000..290c5c2c3 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler; + +import org.eclipse.edc.connector.api.management.configuration.transform.ManagementApiTypeTransformerRegistry; +import org.eclipse.edc.connector.spi.catalog.CatalogService; +import org.eclipse.edc.jsonld.spi.JsonLd; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.runtime.metamodel.annotation.Provides; +import org.eclipse.edc.runtime.metamodel.annotation.Setting; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; +import org.eclipse.edc.spi.types.TypeManager; + +import static de.sovity.edc.ext.catalog.crawler.orchestration.config.EdcConfigPropertyUtils.toEdcProp; + +@Provides({CrawlerExtensionContext.class}) +public class CrawlerExtension implements ServiceExtension { + + public static final String EXTENSION_NAME = "Authority Portal Data Catalog Crawler"; + + @Setting(required = true) + public static final String EXTENSION_ENABLED = toEdcProp("CRAWLER_EXTENSION_ENABLED"); + + @Setting(required = true) + public static final String ENVIRONMENT_ID = toEdcProp("CRAWLER_ENVIRONMENT_ID"); + + @Setting(required = true) + public static final String JDBC_URL = toEdcProp("CRAWLER_DB_JDBC_URL"); + + @Setting(required = true) + public static final String JDBC_USER = toEdcProp("CRAWLER_DB_JDBC_USER"); + + @Setting(required = true) + public static final String JDBC_PASSWORD = toEdcProp("CRAWLER_DB_JDBC_PASSWORD"); + + @Setting + public static final String DB_CONNECTION_POOL_SIZE = toEdcProp("CRAWLER_DB_CONNECTION_POOL_SIZE"); + + @Setting + public static final String DB_CONNECTION_TIMEOUT_IN_MS = toEdcProp("CRAWLER_DB_CONNECTION_TIMEOUT_IN_MS"); + + @Setting + public static final String DB_MIGRATE = toEdcProp("CRAWLER_DB_MIGRATE"); + + @Setting + public static final String DB_CLEAN = toEdcProp("CRAWLER_DB_CLEAN"); + + @Setting + public static final String DB_CLEAN_ENABLED = toEdcProp("CRAWLER_DB_CLEAN_ENABLED"); + + @Setting + public static final String DB_ADDITIONAL_FLYWAY_MIGRATION_LOCATIONS = toEdcProp("CRAWLER_DB_ADDITIONAL_FLYWAY_LOCATIONS"); + + @Setting + public static final String NUM_THREADS = toEdcProp("CRAWLER_NUM_THREADS"); + + @Setting + public static final String MAX_DATA_OFFERS_PER_CONNECTOR = toEdcProp("CRAWLER_MAX_DATA_OFFERS_PER_CONNECTOR"); + + @Setting + public static final String MAX_CONTRACT_OFFERS_PER_DATA_OFFER = toEdcProp("CRAWLER_MAX_CONTRACT_OFFERS_PER_DATA_OFFER"); + + @Setting + public static final String CRON_ONLINE_CONNECTOR_REFRESH = toEdcProp("CRAWLER_CRON_ONLINE_CONNECTOR_REFRESH"); + + @Setting + public static final String CRON_OFFLINE_CONNECTOR_REFRESH = toEdcProp("CRAWLER_CRON_OFFLINE_CONNECTOR_REFRESH"); + + @Setting + public static final String CRON_DEAD_CONNECTOR_REFRESH = toEdcProp("CRAWLER_CRON_DEAD_CONNECTOR_REFRESH"); + + @Setting + public static final String SCHEDULED_KILL_OFFLINE_CONNECTORS = toEdcProp("CRAWLER_SCHEDULED_KILL_OFFLINE_CONNECTORS"); + @Setting + public static final String KILL_OFFLINE_CONNECTORS_AFTER = toEdcProp("CRAWLER_KILL_OFFLINE_CONNECTORS_AFTER"); + + @Inject + private TypeManager typeManager; + + @Inject + private ManagementApiTypeTransformerRegistry typeTransformerRegistry; + + @Inject + private JsonLd jsonLd; + + @Inject + private CatalogService catalogService; + + /** + * Manual Dependency Injection Result + */ + private CrawlerExtensionContext services; + + @Override + public String name() { + return EXTENSION_NAME; + } + + @Override + public void initialize(ServiceExtensionContext context) { + if (!Boolean.TRUE.equals(context.getConfig().getBoolean(EXTENSION_ENABLED, false))) { + context.getMonitor().info("Crawler extension is disabled."); + return; + } + + services = CrawlerExtensionContextBuilder.buildContext( + context.getConfig(), + context.getMonitor(), + typeManager, + typeTransformerRegistry, + jsonLd, + catalogService + ); + + // Provide access for the tests + context.registerService(CrawlerExtensionContext.class, services); + } + + @Override + public void start() { + if (services == null) { + return; + } + services.crawlerInitializer().onStartup(); + } + + @Override + public void shutdown() { + if (services == null) { + return; + } + services.dataSource().close(); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContext.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContext.java new file mode 100644 index 000000000..264831fe8 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContext.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler; + +import com.zaxxer.hikari.HikariDataSource; +import de.sovity.edc.ext.catalog.crawler.crawling.ConnectorCrawler; +import de.sovity.edc.ext.catalog.crawler.crawling.fetching.FetchedCatalogBuilder; +import de.sovity.edc.ext.catalog.crawler.dao.config.DslContextFactory; +import de.sovity.edc.ext.catalog.crawler.dao.data_offers.DataOfferRecordUpdater; +import de.sovity.edc.ext.wrapper.api.common.mappers.PolicyMapper; + + +/** + * Manual Dependency Injection result + * + * @param crawlerInitializer Startup Logic + */ +public record CrawlerExtensionContext( + CrawlerInitializer crawlerInitializer, + // Required for stopping connections on closing + HikariDataSource dataSource, + DslContextFactory dslContextFactory, + + // Required for Integration Tests + ConnectorCrawler connectorCrawler, + PolicyMapper policyMapper, + FetchedCatalogBuilder catalogPatchBuilder, + DataOfferRecordUpdater dataOfferRecordUpdater +) { +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java new file mode 100644 index 000000000..cb49d40a5 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java @@ -0,0 +1,312 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import de.sovity.edc.ext.catalog.crawler.crawling.ConnectorCrawler; +import de.sovity.edc.ext.catalog.crawler.crawling.OfflineConnectorCleaner; +import de.sovity.edc.ext.catalog.crawler.crawling.fetching.FetchedCatalogBuilder; +import de.sovity.edc.ext.catalog.crawler.crawling.fetching.FetchedCatalogMappingUtils; +import de.sovity.edc.ext.catalog.crawler.crawling.fetching.FetchedCatalogService; +import de.sovity.edc.ext.catalog.crawler.crawling.logging.CrawlerEventLogger; +import de.sovity.edc.ext.catalog.crawler.crawling.logging.CrawlerExecutionTimeLogger; +import de.sovity.edc.ext.catalog.crawler.crawling.writing.CatalogPatchBuilder; +import de.sovity.edc.ext.catalog.crawler.crawling.writing.ConnectorUpdateCatalogWriter; +import de.sovity.edc.ext.catalog.crawler.crawling.writing.ConnectorUpdateFailureWriter; +import de.sovity.edc.ext.catalog.crawler.crawling.writing.ConnectorUpdateSuccessWriter; +import de.sovity.edc.ext.catalog.crawler.crawling.writing.DataOfferLimitsEnforcer; +import de.sovity.edc.ext.catalog.crawler.dao.CatalogCleaner; +import de.sovity.edc.ext.catalog.crawler.dao.CatalogPatchApplier; +import de.sovity.edc.ext.catalog.crawler.dao.config.DataSourceFactory; +import de.sovity.edc.ext.catalog.crawler.dao.config.DslContextFactory; +import de.sovity.edc.ext.catalog.crawler.dao.config.FlywayService; +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorQueries; +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorStatusUpdater; +import de.sovity.edc.ext.catalog.crawler.dao.contract_offers.ContractOfferQueries; +import de.sovity.edc.ext.catalog.crawler.dao.contract_offers.ContractOfferRecordUpdater; +import de.sovity.edc.ext.catalog.crawler.dao.data_offers.DataOfferQueries; +import de.sovity.edc.ext.catalog.crawler.dao.data_offers.DataOfferRecordUpdater; +import de.sovity.edc.ext.catalog.crawler.orchestration.config.CrawlerConfigFactory; +import de.sovity.edc.ext.catalog.crawler.orchestration.queue.ConnectorQueue; +import de.sovity.edc.ext.catalog.crawler.orchestration.queue.ConnectorQueueFiller; +import de.sovity.edc.ext.catalog.crawler.orchestration.queue.ThreadPool; +import de.sovity.edc.ext.catalog.crawler.orchestration.queue.ThreadPoolTaskQueue; +import de.sovity.edc.ext.catalog.crawler.orchestration.schedules.DeadConnectorRefreshJob; +import de.sovity.edc.ext.catalog.crawler.orchestration.schedules.OfflineConnectorCleanerJob; +import de.sovity.edc.ext.catalog.crawler.orchestration.schedules.OfflineConnectorRefreshJob; +import de.sovity.edc.ext.catalog.crawler.orchestration.schedules.OnlineConnectorRefreshJob; +import de.sovity.edc.ext.catalog.crawler.orchestration.schedules.QuartzScheduleInitializer; +import de.sovity.edc.ext.catalog.crawler.orchestration.schedules.utils.CronJobRef; +import de.sovity.edc.ext.wrapper.api.common.mappers.AssetMapper; +import de.sovity.edc.ext.wrapper.api.common.mappers.PolicyMapper; +import de.sovity.edc.ext.wrapper.api.common.mappers.asset.AssetEditRequestMapper; +import de.sovity.edc.ext.wrapper.api.common.mappers.asset.AssetJsonLdBuilder; +import de.sovity.edc.ext.wrapper.api.common.mappers.asset.AssetJsonLdParser; +import de.sovity.edc.ext.wrapper.api.common.mappers.asset.utils.AssetJsonLdUtils; +import de.sovity.edc.ext.wrapper.api.common.mappers.asset.utils.EdcPropertyUtils; +import de.sovity.edc.ext.wrapper.api.common.mappers.asset.utils.ShortDescriptionBuilder; +import de.sovity.edc.ext.wrapper.api.common.mappers.dataaddress.DataSourceMapper; +import de.sovity.edc.ext.wrapper.api.common.mappers.dataaddress.http.HttpDataSourceMapper; +import de.sovity.edc.ext.wrapper.api.common.mappers.dataaddress.http.HttpHeaderMapper; +import de.sovity.edc.ext.wrapper.api.common.mappers.policy.AtomicConstraintMapper; +import de.sovity.edc.ext.wrapper.api.common.mappers.policy.ConstraintExtractor; +import de.sovity.edc.ext.wrapper.api.common.mappers.policy.LiteralMapper; +import de.sovity.edc.ext.wrapper.api.common.mappers.policy.OperatorMapper; +import de.sovity.edc.ext.wrapper.api.common.mappers.policy.PolicyValidator; +import de.sovity.edc.utils.catalog.DspCatalogService; +import de.sovity.edc.utils.catalog.mapper.DspDataOfferBuilder; +import lombok.NoArgsConstructor; +import org.eclipse.edc.connector.spi.catalog.CatalogService; +import org.eclipse.edc.jsonld.spi.JsonLd; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.spi.CoreConstants; +import org.eclipse.edc.spi.monitor.Monitor; +import org.eclipse.edc.spi.system.configuration.Config; +import org.eclipse.edc.spi.types.TypeManager; +import org.eclipse.edc.transform.spi.TypeTransformerRegistry; +import org.jetbrains.annotations.NotNull; + +import java.util.List; + + +/** + * Manual Dependency Injection (DYDI). + *

+ * We want to develop as Java Backend Development is done, but we have + * no CDI / DI Framework to rely on. + *

+ * EDC {@link Inject} only works in {@link CrawlerExtension}. + */ +@NoArgsConstructor(access = lombok.AccessLevel.PRIVATE) +public class CrawlerExtensionContextBuilder { + + public static CrawlerExtensionContext buildContext( + Config config, + Monitor monitor, + TypeManager typeManager, + TypeTransformerRegistry typeTransformerRegistry, + JsonLd jsonLd, + CatalogService catalogService + ) { + // Config + var crawlerConfigFactory = new CrawlerConfigFactory(config); + var crawlerConfig = crawlerConfigFactory.buildCrawlerConfig(); + + // DB + var dataSourceFactory = new DataSourceFactory(config); + var dataSource = dataSourceFactory.newDataSource(); + var flywayService = new FlywayService(config, monitor, dataSource); + flywayService.validateOrMigrateInTests(); + + // Dao + var dataOfferQueries = new DataOfferQueries(); + var dslContextFactory = new DslContextFactory(dataSource); + var connectorQueries = new ConnectorQueries(crawlerConfig); + + // Services + var objectMapperJsonLd = getJsonLdObjectMapper(typeManager); + var assetMapper = newAssetMapper(typeTransformerRegistry, jsonLd); + var policyMapper = newPolicyMapper(typeTransformerRegistry, objectMapperJsonLd); + var crawlerEventLogger = new CrawlerEventLogger(); + var crawlerExecutionTimeLogger = new CrawlerExecutionTimeLogger(); + var dataOfferMappingUtils = new FetchedCatalogMappingUtils( + policyMapper, + assetMapper, + objectMapperJsonLd + ); + var contractOfferRecordUpdater = new ContractOfferRecordUpdater(); + var dataOfferRecordUpdater = new DataOfferRecordUpdater(connectorQueries); + var contractOfferQueries = new ContractOfferQueries(); + var dataOfferLimitsEnforcer = new DataOfferLimitsEnforcer(crawlerConfig, crawlerEventLogger); + var dataOfferPatchBuilder = new CatalogPatchBuilder( + contractOfferQueries, + dataOfferQueries, + dataOfferRecordUpdater, + contractOfferRecordUpdater + ); + var dataOfferPatchApplier = new CatalogPatchApplier(); + var dataOfferWriter = new ConnectorUpdateCatalogWriter(dataOfferPatchBuilder, dataOfferPatchApplier); + var connectorUpdateSuccessWriter = new ConnectorUpdateSuccessWriter( + crawlerEventLogger, + dataOfferWriter, + dataOfferLimitsEnforcer + ); + var fetchedDataOfferBuilder = new FetchedCatalogBuilder(dataOfferMappingUtils); + var dspDataOfferBuilder = new DspDataOfferBuilder(jsonLd); + var dspCatalogService = new DspCatalogService( + catalogService, + dspDataOfferBuilder + ); + var dataOfferFetcher = new FetchedCatalogService(dspCatalogService, fetchedDataOfferBuilder); + var connectorUpdateFailureWriter = new ConnectorUpdateFailureWriter(crawlerEventLogger, monitor); + var connectorUpdater = new ConnectorCrawler( + dataOfferFetcher, + connectorUpdateSuccessWriter, + connectorUpdateFailureWriter, + connectorQueries, + dslContextFactory, + monitor, + crawlerExecutionTimeLogger + ); + + var threadPoolTaskQueue = new ThreadPoolTaskQueue(); + var threadPool = new ThreadPool(threadPoolTaskQueue, crawlerConfig, monitor); + var connectorQueue = new ConnectorQueue(connectorUpdater, threadPool); + var connectorQueueFiller = new ConnectorQueueFiller(connectorQueue, connectorQueries); + var connectorStatusUpdater = new ConnectorStatusUpdater(); + var catalogCleaner = new CatalogCleaner(); + var offlineConnectorCleaner = new OfflineConnectorCleaner( + crawlerConfig, + connectorQueries, + crawlerEventLogger, + connectorStatusUpdater, + catalogCleaner + ); + + // Schedules + List> jobs = List.of( + getOnlineConnectorRefreshCronJob(dslContextFactory, connectorQueueFiller), + getOfflineConnectorRefreshCronJob(dslContextFactory, connectorQueueFiller), + getDeadConnectorRefreshCronJob(dslContextFactory, connectorQueueFiller), + getOfflineConnectorCleanerCronJob(dslContextFactory, offlineConnectorCleaner) + ); + + // Startup + var quartzScheduleInitializer = new QuartzScheduleInitializer(config, monitor, jobs); + var crawlerInitializer = new CrawlerInitializer(quartzScheduleInitializer); + + return new CrawlerExtensionContext( + crawlerInitializer, + dataSource, + dslContextFactory, + connectorUpdater, + policyMapper, + fetchedDataOfferBuilder, + dataOfferRecordUpdater + ); + } + + @NotNull + private static PolicyMapper newPolicyMapper(TypeTransformerRegistry typeTransformerRegistry, ObjectMapper objectMapperJsonLd) { + var operatorMapper = new OperatorMapper(); + var literalMapper = new LiteralMapper( + objectMapperJsonLd + ); + var atomicConstraintMapper = new AtomicConstraintMapper( + literalMapper, + operatorMapper + ); + var policyValidator = new PolicyValidator(); + var constraintExtractor = new ConstraintExtractor( + policyValidator, + atomicConstraintMapper + ); + return new PolicyMapper( + constraintExtractor, + atomicConstraintMapper, + typeTransformerRegistry + ); + } + + @NotNull + private static AssetMapper newAssetMapper( + TypeTransformerRegistry typeTransformerRegistry, + JsonLd jsonLd + ) { + var edcPropertyUtils = new EdcPropertyUtils(); + var assetJsonLdUtils = new AssetJsonLdUtils(); + var assetEditRequestMapper = new AssetEditRequestMapper(); + var shortDescriptionBuilder = new ShortDescriptionBuilder(); + var assetJsonLdParser = new AssetJsonLdParser( + assetJsonLdUtils, + shortDescriptionBuilder, + endpoint -> false + ); + var httpHeaderMapper = new HttpHeaderMapper(); + var httpDataSourceMapper = new HttpDataSourceMapper(httpHeaderMapper); + var dataSourceMapper = new DataSourceMapper( + edcPropertyUtils, + httpDataSourceMapper + ); + var assetJsonLdBuilder = new AssetJsonLdBuilder( + dataSourceMapper, + assetJsonLdParser, + assetEditRequestMapper + ); + return new AssetMapper( + typeTransformerRegistry, + assetJsonLdBuilder, + assetJsonLdParser, + jsonLd + ); + } + + @NotNull + private static CronJobRef getOfflineConnectorCleanerCronJob(DslContextFactory dslContextFactory, + OfflineConnectorCleaner offlineConnectorCleaner) { + return new CronJobRef<>( + CrawlerExtension.SCHEDULED_KILL_OFFLINE_CONNECTORS, + OfflineConnectorCleanerJob.class, + () -> new OfflineConnectorCleanerJob(dslContextFactory, offlineConnectorCleaner) + ); + } + + @NotNull + private static CronJobRef getOnlineConnectorRefreshCronJob( + DslContextFactory dslContextFactory, + ConnectorQueueFiller connectorQueueFiller + ) { + return new CronJobRef<>( + CrawlerExtension.CRON_ONLINE_CONNECTOR_REFRESH, + OnlineConnectorRefreshJob.class, + () -> new OnlineConnectorRefreshJob(dslContextFactory, connectorQueueFiller) + ); + } + + @NotNull + private static CronJobRef getOfflineConnectorRefreshCronJob( + DslContextFactory dslContextFactory, + ConnectorQueueFiller connectorQueueFiller + ) { + return new CronJobRef<>( + CrawlerExtension.CRON_OFFLINE_CONNECTOR_REFRESH, + OfflineConnectorRefreshJob.class, + () -> new OfflineConnectorRefreshJob(dslContextFactory, connectorQueueFiller) + ); + } + + @NotNull + private static CronJobRef getDeadConnectorRefreshCronJob(DslContextFactory dslContextFactory, + ConnectorQueueFiller connectorQueueFiller) { + return new CronJobRef<>( + CrawlerExtension.CRON_DEAD_CONNECTOR_REFRESH, + DeadConnectorRefreshJob.class, + () -> new DeadConnectorRefreshJob(dslContextFactory, connectorQueueFiller) + ); + } + + private static ObjectMapper getJsonLdObjectMapper(TypeManager typeManager) { + var objectMapper = typeManager.getMapper(CoreConstants.JSON_LD); + + // Fixes Dates in JSON-LD Object Mapper + // The Core EDC uses longs over OffsetDateTime, so they never fixed the date format + objectMapper.registerModule(new JavaTimeModule()); + objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + + return objectMapper; + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerInitializer.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerInitializer.java new file mode 100644 index 000000000..719ace1c3 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerInitializer.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler; + +import de.sovity.edc.ext.catalog.crawler.orchestration.schedules.QuartzScheduleInitializer; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class CrawlerInitializer { + private final QuartzScheduleInitializer quartzScheduleInitializer; + + public void onStartup() { + quartzScheduleInitializer.startSchedules(); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/ConnectorCrawler.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/ConnectorCrawler.java new file mode 100644 index 000000000..dca422308 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/ConnectorCrawler.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.crawling; + +import de.sovity.edc.ext.catalog.crawler.crawling.fetching.FetchedCatalogService; +import de.sovity.edc.ext.catalog.crawler.crawling.logging.CrawlerExecutionTimeLogger; +import de.sovity.edc.ext.catalog.crawler.crawling.writing.ConnectorUpdateFailureWriter; +import de.sovity.edc.ext.catalog.crawler.crawling.writing.ConnectorUpdateSuccessWriter; +import de.sovity.edc.ext.catalog.crawler.dao.config.DslContextFactory; +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorQueries; +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; +import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.MeasurementErrorStatus; +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.time.StopWatch; +import org.eclipse.edc.spi.monitor.Monitor; + +import java.util.concurrent.TimeUnit; + +/** + * Updates a single connector. + */ +@RequiredArgsConstructor +public class ConnectorCrawler { + private final FetchedCatalogService fetchedCatalogService; + private final ConnectorUpdateSuccessWriter connectorUpdateSuccessWriter; + private final ConnectorUpdateFailureWriter connectorUpdateFailureWriter; + private final ConnectorQueries connectorQueries; + private final DslContextFactory dslContextFactory; + private final Monitor monitor; + private final CrawlerExecutionTimeLogger crawlerExecutionTimeLogger; + + /** + * Updates single connector. + * + * @param connectorRef connector + */ + public void crawlConnector(ConnectorRef connectorRef) { + var executionTime = StopWatch.createStarted(); + var failed = false; + + try { + monitor.info("Updating connector: " + connectorRef); + + var catalog = fetchedCatalogService.fetchCatalog(connectorRef); + + // Update connector in a single transaction + dslContextFactory.transaction(dsl -> { + var connectorRecord = connectorQueries.findByConnectorId(dsl, connectorRef.getConnectorId()); + connectorUpdateSuccessWriter.handleConnectorOnline(dsl, connectorRef, connectorRecord, catalog); + }); + } catch (Exception e) { + failed = true; + try { + // Update connector in a single transaction + dslContextFactory.transaction(dsl -> { + var connectorRecord = connectorQueries.findByConnectorId(dsl, connectorRef.getConnectorId()); + connectorUpdateFailureWriter.handleConnectorOffline(dsl, connectorRef, connectorRecord, e); + }); + } catch (Exception e1) { + e1.addSuppressed(e); + monitor.severe("Failed updating connector as failed.", e1); + } + } finally { + executionTime.stop(); + try { + var status = failed ? MeasurementErrorStatus.ERROR : MeasurementErrorStatus.OK; + dslContextFactory.transaction(dsl -> crawlerExecutionTimeLogger.logExecutionTime( + dsl, + connectorRef, + executionTime.getTime(TimeUnit.MILLISECONDS), + status + )); + } catch (Exception e) { + monitor.severe("Failed logging connector update execution time.", e); + } + } + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/OfflineConnectorCleaner.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/OfflineConnectorCleaner.java new file mode 100644 index 000000000..51f5c9bd4 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/OfflineConnectorCleaner.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.crawling; + +import de.sovity.edc.ext.catalog.crawler.crawling.logging.CrawlerEventLogger; +import de.sovity.edc.ext.catalog.crawler.dao.CatalogCleaner; +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorQueries; +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorStatusUpdater; +import de.sovity.edc.ext.catalog.crawler.orchestration.config.CrawlerConfig; +import lombok.RequiredArgsConstructor; +import org.jooq.DSLContext; + +@RequiredArgsConstructor +public class OfflineConnectorCleaner { + private final CrawlerConfig crawlerConfig; + private final ConnectorQueries connectorQueries; + private final CrawlerEventLogger crawlerEventLogger; + private final ConnectorStatusUpdater connectorStatusUpdater; + private final CatalogCleaner catalogCleaner; + + public void cleanConnectorsIfOfflineTooLong(DSLContext dsl) { + var killOfflineConnectorsAfter = crawlerConfig.getKillOfflineConnectorsAfter(); + var connectorsToKill = connectorQueries.findAllConnectorsForKilling(dsl, killOfflineConnectorsAfter); + + catalogCleaner.removeCatalogByConnectors(dsl, connectorsToKill); + connectorStatusUpdater.markAsDead(dsl, connectorsToKill); + + crawlerEventLogger.addKilledDueToOfflineTooLongMessages(dsl, connectorsToKill); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogBuilder.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogBuilder.java new file mode 100644 index 000000000..2ef73ffa2 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogBuilder.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.crawling.fetching; + +import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedCatalog; +import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedContractOffer; +import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedDataOffer; +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; +import de.sovity.edc.utils.catalog.model.DspCatalog; +import de.sovity.edc.utils.catalog.model.DspContractOffer; +import de.sovity.edc.utils.catalog.model.DspDataOffer; +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.Validate; +import org.jetbrains.annotations.NotNull; + +import java.util.List; + +@RequiredArgsConstructor +public class FetchedCatalogBuilder { + private final FetchedCatalogMappingUtils fetchedCatalogMappingUtils; + + public FetchedCatalog buildFetchedCatalog(DspCatalog catalog, ConnectorRef connectorRef) { + assertEqualEndpoint(catalog, connectorRef); + assertEqualParticipantId(catalog, connectorRef); + + var fetchedDataOffers = catalog.getDataOffers().stream() + .map(dspDataOffer -> buildFetchedDataOffer(dspDataOffer, connectorRef)) + .toList(); + + var fetchedCatalog = new FetchedCatalog(); + fetchedCatalog.setConnectorRef(connectorRef); + fetchedCatalog.setDataOffers(fetchedDataOffers); + return fetchedCatalog; + } + + private void assertEqualParticipantId(DspCatalog catalog, ConnectorRef connectorRef) { + Validate.isTrue( + connectorRef.getConnectorId().equals(catalog.getParticipantId()), + String.format( + "Connector connectorId does not match the participantId: connectorId %s, participantId %s", + connectorRef.getConnectorId(), + catalog.getParticipantId() + ) + ); + } + + private void assertEqualEndpoint(DspCatalog catalog, ConnectorRef connectorRef) { + Validate.isTrue( + connectorRef.getEndpoint().equals(catalog.getEndpoint()), + String.format( + "Connector endpoint mismatch: expected %s, got %s", + connectorRef.getEndpoint(), + catalog.getEndpoint() + ) + ); + } + + @NotNull + private FetchedDataOffer buildFetchedDataOffer( + DspDataOffer dspDataOffer, + ConnectorRef connectorRef + ) { + var uiAsset = fetchedCatalogMappingUtils.buildUiAsset(dspDataOffer, connectorRef); + var uiAssetJson = fetchedCatalogMappingUtils.buildUiAssetJson(uiAsset); + + var fetchedDataOffer = new FetchedDataOffer(); + fetchedDataOffer.setAssetId(uiAsset.getAssetId()); + fetchedDataOffer.setUiAsset(uiAsset); + fetchedDataOffer.setUiAssetJson(uiAssetJson); + fetchedDataOffer.setContractOffers(buildFetchedContractOffers(dspDataOffer.getContractOffers())); + return fetchedDataOffer; + } + + @NotNull + private List buildFetchedContractOffers(List offers) { + return offers.stream() + .map(this::buildFetchedContractOffer) + .toList(); + } + + @NotNull + private FetchedContractOffer buildFetchedContractOffer(DspContractOffer offer) { + var uiPolicyJson = fetchedCatalogMappingUtils.buildUiPolicyJson(offer.getPolicyJsonLd()); + var contractOffer = new FetchedContractOffer(); + contractOffer.setContractOfferId(offer.getContractOfferId()); + contractOffer.setUiPolicyJson(uiPolicyJson); + return contractOffer; + } + +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogMappingUtils.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogMappingUtils.java new file mode 100644 index 000000000..1ed02ef3e --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogMappingUtils.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.crawling.fetching; + +import com.fasterxml.jackson.databind.ObjectMapper; +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; +import de.sovity.edc.ext.wrapper.api.common.mappers.AssetMapper; +import de.sovity.edc.ext.wrapper.api.common.mappers.PolicyMapper; +import de.sovity.edc.ext.wrapper.api.common.model.UiAsset; +import de.sovity.edc.utils.catalog.model.DspDataOffer; +import jakarta.json.JsonObject; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; + +@RequiredArgsConstructor +public class FetchedCatalogMappingUtils { + private final PolicyMapper policyMapper; + private final AssetMapper assetMapper; + private final ObjectMapper objectMapper; + + public UiAsset buildUiAsset( + DspDataOffer dspDataOffer, + ConnectorRef connectorRef + ) { + var assetJsonLd = assetMapper.buildAssetJsonLdFromDatasetProperties(dspDataOffer.getAssetPropertiesJsonLd()); + var asset = assetMapper.buildAsset(assetJsonLd); + var uiAsset = assetMapper.buildUiAsset(asset, connectorRef.getEndpoint(), connectorRef.getConnectorId()); + uiAsset.setCreatorOrganizationName(connectorRef.getOrganizationLegalName()); + uiAsset.setParticipantId(connectorRef.getConnectorId()); + return uiAsset; + } + + @SneakyThrows + public String buildUiAssetJson(UiAsset uiAsset) { + return objectMapper.writeValueAsString(uiAsset); + } + + @SneakyThrows + public String buildUiPolicyJson(JsonObject policyJsonLd) { + var policy = policyMapper.buildPolicy(policyJsonLd); + var uiPolicy = policyMapper.buildUiPolicy(policy); + return objectMapper.writeValueAsString(uiPolicy); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogService.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogService.java new file mode 100644 index 000000000..c55973bbe --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogService.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.crawling.fetching; + +import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedCatalog; +import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedDataOffer; +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; +import de.sovity.edc.utils.catalog.DspCatalogService; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.eclipse.edc.connector.contract.spi.types.offer.ContractOffer; + +@RequiredArgsConstructor +public class FetchedCatalogService { + private final DspCatalogService dspCatalogService; + private final FetchedCatalogBuilder catalogPatchBuilder; + + /** + * Fetches {@link ContractOffer}s and de-duplicates them into {@link FetchedDataOffer}s. + * + * @param connectorRef connector + * @return updated connector db row + */ + @SneakyThrows + public FetchedCatalog fetchCatalog(ConnectorRef connectorRef) { + var dspCatalog = dspCatalogService.fetchDataOffers(connectorRef.getEndpoint()); + return catalogPatchBuilder.buildFetchedCatalog(dspCatalog, connectorRef); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedCatalog.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedCatalog.java new file mode 100644 index 000000000..d930eb223 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedCatalog.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.crawling.fetching.model; + +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.FieldDefaults; + +import java.util.List; + +/** + * Contains catalog response as required for writing into DB. + */ +@Getter +@Setter +@FieldDefaults(level = AccessLevel.PRIVATE) +public class FetchedCatalog { + ConnectorRef connectorRef; + List dataOffers; +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedContractOffer.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedContractOffer.java new file mode 100644 index 000000000..64317498e --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedContractOffer.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.crawling.fetching.model; + +import lombok.AccessLevel; +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.FieldDefaults; + +@Getter +@Setter +@FieldDefaults(level = AccessLevel.PRIVATE) +public class FetchedContractOffer { + String contractOfferId; + String uiPolicyJson; +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedDataOffer.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedDataOffer.java new file mode 100644 index 000000000..752fc98fd --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedDataOffer.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.crawling.fetching.model; + +import de.sovity.edc.ext.wrapper.api.common.model.UiAsset; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.FieldDefaults; + +import java.util.List; + +/** + * Contains data offer response as required for writing into DB. + */ +@Getter +@Setter +@FieldDefaults(level = AccessLevel.PRIVATE) +public class FetchedDataOffer { + String assetId; + UiAsset uiAsset; + String uiAssetJson; + List contractOffers; +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/ConnectorChangeTracker.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/ConnectorChangeTracker.java new file mode 100644 index 000000000..e87eabb01 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/ConnectorChangeTracker.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.crawling.logging; + +import lombok.Getter; +import lombok.Setter; + +import java.util.ArrayList; +import java.util.List; + +/** + * Utility for collecting the information required to build log messages about what was updated. + */ +@Getter +public class ConnectorChangeTracker { + @Setter + private int numOffersAdded = 0; + + @Setter + private int numOffersDeleted = 0; + + @Setter + private int numOffersUpdated = 0; + + @Setter + private String participantIdChanged = null; + + public boolean isEmpty() { + return numOffersAdded == 0 && numOffersDeleted == 0 && numOffersUpdated == 0 && participantIdChanged == null; + } + + @Override + public String toString() { + if (isEmpty()) { + return "Connector is up to date."; + } + + var msg = "Connector Updated."; + if (numOffersAdded > 0 || numOffersDeleted > 0 || numOffersUpdated > 0) { + List offersMsgs = new ArrayList<>(); + if (numOffersAdded > 0) { + offersMsgs.add("%d added".formatted(numOffersAdded)); + } + if (numOffersUpdated > 0) { + offersMsgs.add("%d updated".formatted(numOffersUpdated)); + } + if (numOffersDeleted > 0) { + offersMsgs.add("%d deleted".formatted(numOffersDeleted)); + } + msg += " Data Offers changed: %s.".formatted(String.join(", ", offersMsgs)); + } + if (participantIdChanged != null) { + msg += " Participant ID changed to %s.".formatted(participantIdChanged); + } + return msg; + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventErrorMessage.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventErrorMessage.java new file mode 100644 index 000000000..9d7ca039a --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventErrorMessage.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.crawling.logging; + +import de.sovity.edc.ext.catalog.crawler.utils.StringUtils2; +import lombok.NonNull; +import org.apache.commons.lang3.exception.ExceptionUtils; + +/** + * Helper Dto that contains User Message + Error Stack Trace to be written into + * {@link de.sovity.edc.ext.catalog.crawler.db.jooq.tables.CrawlerEventLog}. + *
+ * This class exists so that logging exceptions has a consistent format. + * + * @param message message + * @param stackTraceOrNull stack trace + */ +public record CrawlerEventErrorMessage(String message, String stackTraceOrNull) { + + public static CrawlerEventErrorMessage ofMessage(@NonNull String message) { + return new CrawlerEventErrorMessage(message, null); + } + + public static CrawlerEventErrorMessage ofStackTrace(@NonNull String baseMessage, @NonNull Throwable cause) { + var message = baseMessage; + message = StringUtils2.removeSuffix(message, "."); + message = StringUtils2.removeSuffix(message, ":"); + message = "%s: %s".formatted(message, cause.getClass().getName()); + return new CrawlerEventErrorMessage(message, ExceptionUtils.getStackTrace(cause)); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventLogger.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventLogger.java new file mode 100644 index 000000000..396e2095e --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventLogger.java @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.crawling.logging; + +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; +import de.sovity.edc.ext.catalog.crawler.db.jooq.Tables; +import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.CrawlerEventStatus; +import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.CrawlerEventType; +import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.CrawlerEventLogRecord; +import lombok.RequiredArgsConstructor; +import org.jooq.DSLContext; + +import java.time.OffsetDateTime; +import java.util.Collection; +import java.util.UUID; + +/** + * Updates a single connector. + */ +@RequiredArgsConstructor +public class CrawlerEventLogger { + + public void logConnectorUpdated(DSLContext dsl, ConnectorRef connectorRef, ConnectorChangeTracker changes) { + var logEntry = newLogEntry(dsl, connectorRef); + logEntry.setEvent(CrawlerEventType.CONNECTOR_UPDATED); + logEntry.setEventStatus(CrawlerEventStatus.OK); + logEntry.setUserMessage(changes.toString()); + logEntry.insert(); + } + + public void logConnectorOffline(DSLContext dsl, ConnectorRef connectorRef, CrawlerEventErrorMessage errorMessage) { + var logEntry = newLogEntry(dsl, connectorRef); + logEntry.setEvent(CrawlerEventType.CONNECTOR_STATUS_CHANGE_OFFLINE); + logEntry.setEventStatus(CrawlerEventStatus.ERROR); + logEntry.setUserMessage("Connector is offline."); + logEntry.setErrorStack(errorMessage.stackTraceOrNull()); + logEntry.insert(); + } + + public void logConnectorOnline(DSLContext dsl, ConnectorRef connectorRef) { + var logEntry = newLogEntry(dsl, connectorRef); + logEntry.setEvent(CrawlerEventType.CONNECTOR_STATUS_CHANGE_ONLINE); + logEntry.setEventStatus(CrawlerEventStatus.OK); + logEntry.setUserMessage("Connector is online."); + logEntry.insert(); + } + + public void logConnectorUpdateDataOfferLimitExceeded( + DSLContext dsl, + ConnectorRef connectorRef, + Integer maxDataOffersPerConnector + ) { + var logEntry = newLogEntry(dsl, connectorRef); + logEntry.setEvent(CrawlerEventType.CONNECTOR_DATA_OFFER_LIMIT_EXCEEDED); + logEntry.setEventStatus(CrawlerEventStatus.OK); + logEntry.setUserMessage( + "Connector has more than %d data offers. Exceeding data offers will be ignored.".formatted(maxDataOffersPerConnector)); + logEntry.insert(); + } + + public void logConnectorUpdateDataOfferLimitOk(DSLContext dsl, ConnectorRef connectorRef) { + var logEntry = newLogEntry(dsl, connectorRef); + logEntry.setEvent(CrawlerEventType.CONNECTOR_DATA_OFFER_LIMIT_OK); + logEntry.setEventStatus(CrawlerEventStatus.OK); + logEntry.setUserMessage("Connector is not exceeding the maximum number of data offers limit anymore."); + logEntry.insert(); + } + + public void logConnectorUpdateContractOfferLimitExceeded( + DSLContext dsl, + ConnectorRef connectorRef, + Integer maxContractOffersPerConnector + ) { + var logEntry = newLogEntry(dsl, connectorRef); + logEntry.setEvent(CrawlerEventType.CONNECTOR_CONTRACT_OFFER_LIMIT_EXCEEDED); + logEntry.setEventStatus(CrawlerEventStatus.OK); + logEntry.setUserMessage(String.format( + "Some data offers have more than %d contract offers. Exceeding contract offers will be ignored.: ", + maxContractOffersPerConnector + )); + logEntry.insert(); + } + + public void logConnectorUpdateContractOfferLimitOk(DSLContext dsl, ConnectorRef connectorRef) { + var logEntry = newLogEntry(dsl, connectorRef); + logEntry.setEvent(CrawlerEventType.CONNECTOR_CONTRACT_OFFER_LIMIT_OK); + logEntry.setEventStatus(CrawlerEventStatus.OK); + logEntry.setUserMessage("Connector is not exceeding the maximum number of contract offers per data offer limit anymore."); + logEntry.insert(); + } + + public void addKilledDueToOfflineTooLongMessages(DSLContext dsl, Collection connectorRefs) { + var logEntries = connectorRefs.stream() + .map(connectorRef -> buildKilledDueToOfflineTooLongMessage(dsl, connectorRef)) + .toList(); + dsl.batchInsert(logEntries).execute(); + } + + private CrawlerEventLogRecord buildKilledDueToOfflineTooLongMessage(DSLContext dsl, ConnectorRef connectorRef) { + var logEntry = newLogEntry(dsl, connectorRef); + logEntry.setEvent(CrawlerEventType.CONNECTOR_KILLED_DUE_TO_OFFLINE_FOR_TOO_LONG); + logEntry.setEventStatus(CrawlerEventStatus.OK); + logEntry.setUserMessage("Connector was marked as dead for being offline too long."); + return logEntry; + } + + private CrawlerEventLogRecord newLogEntry(DSLContext dsl, ConnectorRef connectorRef) { + var logEntry = dsl.newRecord(Tables.CRAWLER_EVENT_LOG); + logEntry.setId(UUID.randomUUID()); + logEntry.setEnvironment(connectorRef.getEnvironmentId()); + logEntry.setConnectorId(connectorRef.getConnectorId()); + logEntry.setCreatedAt(OffsetDateTime.now()); + return logEntry; + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerExecutionTimeLogger.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerExecutionTimeLogger.java new file mode 100644 index 000000000..b04e757f4 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerExecutionTimeLogger.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.crawling.logging; + +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; +import de.sovity.edc.ext.catalog.crawler.db.jooq.Tables; +import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.MeasurementErrorStatus; +import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.MeasurementType; +import lombok.RequiredArgsConstructor; +import org.jooq.DSLContext; + +import java.time.OffsetDateTime; +import java.util.UUID; + +/** + * Updates a single connector. + */ +@RequiredArgsConstructor +public class CrawlerExecutionTimeLogger { + public void logExecutionTime(DSLContext dsl, ConnectorRef connectorRef, long executionTimeInMs, MeasurementErrorStatus errorStatus) { + var logEntry = dsl.newRecord(Tables.CRAWLER_EXECUTION_TIME_MEASUREMENT); + logEntry.setId(UUID.randomUUID()); + logEntry.setEnvironment(connectorRef.getEnvironmentId()); + logEntry.setConnectorId(connectorRef.getConnectorId()); + logEntry.setDurationInMs(executionTimeInMs); + logEntry.setType(MeasurementType.CONNECTOR_REFRESH); + logEntry.setErrorStatus(errorStatus); + logEntry.setCreatedAt(OffsetDateTime.now()); + logEntry.insert(); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/CatalogPatchBuilder.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/CatalogPatchBuilder.java new file mode 100644 index 000000000..309e8d8fc --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/CatalogPatchBuilder.java @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.crawling.writing; + +import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedContractOffer; +import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedDataOffer; +import de.sovity.edc.ext.catalog.crawler.crawling.writing.utils.DiffUtils; +import de.sovity.edc.ext.catalog.crawler.dao.CatalogPatch; +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; +import de.sovity.edc.ext.catalog.crawler.dao.contract_offers.ContractOfferQueries; +import de.sovity.edc.ext.catalog.crawler.dao.contract_offers.ContractOfferRecordUpdater; +import de.sovity.edc.ext.catalog.crawler.dao.data_offers.DataOfferQueries; +import de.sovity.edc.ext.catalog.crawler.dao.data_offers.DataOfferRecordUpdater; +import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ContractOfferRecord; +import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.DataOfferRecord; +import lombok.RequiredArgsConstructor; +import org.jooq.DSLContext; + +import java.util.Collection; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import static java.util.stream.Collectors.groupingBy; + +@RequiredArgsConstructor +public class CatalogPatchBuilder { + private final ContractOfferQueries contractOfferQueries; + private final DataOfferQueries dataOfferQueries; + private final DataOfferRecordUpdater dataOfferRecordUpdater; + private final ContractOfferRecordUpdater contractOfferRecordUpdater; + + /** + * Fetches existing data offers of given connector endpoint and compares them with fetched data offers. + * + * @param dsl dsl + * @param connectorRef connector + * @param fetchedDataOffers fetched data offers + * @return change list / patch + */ + public CatalogPatch buildDataOfferPatch( + DSLContext dsl, + ConnectorRef connectorRef, + Collection fetchedDataOffers + ) { + var patch = new CatalogPatch(); + var dataOffers = dataOfferQueries.findByConnectorId(dsl, connectorRef.getConnectorId()); + var contractOffersByAssetId = contractOfferQueries.findByConnectorId(dsl, connectorRef.getConnectorId()) + .stream() + .collect(groupingBy(ContractOfferRecord::getAssetId)); + + var diff = DiffUtils.compareLists( + dataOffers, + DataOfferRecord::getAssetId, + fetchedDataOffers, + FetchedDataOffer::getAssetId + ); + + diff.added().forEach(fetched -> { + var newRecord = dataOfferRecordUpdater.newDataOffer(connectorRef, fetched); + patch.dataOffers().insert(newRecord); + patchContractOffers(patch, newRecord, List.of(), fetched.getContractOffers()); + }); + + diff.updated().forEach(match -> { + var existing = match.existing(); + var fetched = match.fetched(); + + // Update Contract Offers + var contractOffers = contractOffersByAssetId.getOrDefault(existing.getAssetId(), List.of()); + var changed = patchContractOffers(patch, existing, contractOffers, fetched.getContractOffers()); + + // Update Data Offer (and update updatedAt if contractOffers changed) + changed = dataOfferRecordUpdater.updateDataOffer(existing, fetched, changed); + + if (changed) { + patch.dataOffers().update(existing); + } + }); + + diff.removed().forEach(dataOffer -> { + patch.dataOffers().delete(dataOffer); + var contractOffers = contractOffersByAssetId.getOrDefault(dataOffer.getAssetId(), List.of()); + contractOffers.forEach(it -> patch.contractOffers().delete(it)); + }); + + return patch; + } + + private boolean patchContractOffers( + CatalogPatch patch, + DataOfferRecord dataOffer, + Collection contractOffers, + Collection fetchedContractOffers + ) { + var hasUpdates = new AtomicBoolean(false); + + var diff = DiffUtils.compareLists( + contractOffers, + ContractOfferRecord::getContractOfferId, + fetchedContractOffers, + FetchedContractOffer::getContractOfferId + ); + + diff.added().forEach(fetched -> { + var newRecord = contractOfferRecordUpdater.newContractOffer(dataOffer, fetched); + patch.contractOffers().insert(newRecord); + hasUpdates.set(true); + }); + + diff.updated().forEach(match -> { + var existing = match.existing(); + var fetched = match.fetched(); + + if (contractOfferRecordUpdater.updateContractOffer(existing, fetched)) { + patch.contractOffers().update(existing); + hasUpdates.set(true); + } + }); + + diff.removed().forEach(existing -> { + patch.contractOffers().delete(existing); + hasUpdates.set(true); + }); + + return hasUpdates.get(); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateCatalogWriter.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateCatalogWriter.java new file mode 100644 index 000000000..ec67dd2a2 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateCatalogWriter.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.crawling.writing; + +import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedDataOffer; +import de.sovity.edc.ext.catalog.crawler.crawling.logging.ConnectorChangeTracker; +import de.sovity.edc.ext.catalog.crawler.dao.CatalogPatchApplier; +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.jooq.DSLContext; + +import java.util.Collection; + +@RequiredArgsConstructor +public class ConnectorUpdateCatalogWriter { + private final CatalogPatchBuilder catalogPatchBuilder; + private final CatalogPatchApplier catalogPatchApplier; + + /** + * Updates a connector's data offers with given {@link FetchedDataOffer}s. + * + * @param dsl dsl + * @param connectorRef connector + * @param fetchedDataOffers fetched data offers + * @param changes change tracker for log message + */ + @SneakyThrows + public void updateDataOffers( + DSLContext dsl, + ConnectorRef connectorRef, + Collection fetchedDataOffers, + ConnectorChangeTracker changes + ) { + var patch = catalogPatchBuilder.buildDataOfferPatch(dsl, connectorRef, fetchedDataOffers); + changes.setNumOffersAdded(patch.dataOffers().getInsertions().size()); + changes.setNumOffersUpdated(patch.dataOffers().getUpdates().size()); + changes.setNumOffersDeleted(patch.dataOffers().getDeletions().size()); + catalogPatchApplier.applyDbUpdatesBatched(dsl, patch); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateFailureWriter.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateFailureWriter.java new file mode 100644 index 000000000..05b7f1bb1 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateFailureWriter.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.crawling.writing; + +import de.sovity.edc.ext.catalog.crawler.crawling.logging.CrawlerEventErrorMessage; +import de.sovity.edc.ext.catalog.crawler.crawling.logging.CrawlerEventLogger; +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; +import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorOnlineStatus; +import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ConnectorRecord; +import lombok.RequiredArgsConstructor; +import org.eclipse.edc.spi.monitor.Monitor; +import org.jooq.DSLContext; + +import java.time.OffsetDateTime; + +@RequiredArgsConstructor +public class ConnectorUpdateFailureWriter { + private final CrawlerEventLogger crawlerEventLogger; + private final Monitor monitor; + + public void handleConnectorOffline( + DSLContext dsl, + ConnectorRef connectorRef, + ConnectorRecord connector, + Throwable e + ) { + // Log Status Change and set status to offline if necessary + if (connector.getOnlineStatus() == ConnectorOnlineStatus.ONLINE || connector.getLastRefreshAttemptAt() == null) { + monitor.info("Connector is offline: " + connector.getEndpointUrl(), e); + crawlerEventLogger.logConnectorOffline(dsl, connectorRef, getFailureMessage(e)); + connector.setOnlineStatus(ConnectorOnlineStatus.OFFLINE); + } + + connector.setLastRefreshAttemptAt(OffsetDateTime.now()); + connector.update(); + } + + public CrawlerEventErrorMessage getFailureMessage(Throwable e) { + return CrawlerEventErrorMessage.ofStackTrace("Unexpected exception during connector update.", e); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateSuccessWriter.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateSuccessWriter.java new file mode 100644 index 000000000..32b5ebe5e --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateSuccessWriter.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.crawling.writing; + +import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedCatalog; +import de.sovity.edc.ext.catalog.crawler.crawling.logging.ConnectorChangeTracker; +import de.sovity.edc.ext.catalog.crawler.crawling.logging.CrawlerEventLogger; +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; +import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorOnlineStatus; +import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ConnectorRecord; +import lombok.RequiredArgsConstructor; +import org.jooq.DSLContext; + +import java.time.OffsetDateTime; + +@RequiredArgsConstructor +public class ConnectorUpdateSuccessWriter { + private final CrawlerEventLogger crawlerEventLogger; + private final ConnectorUpdateCatalogWriter connectorUpdateCatalogWriter; + private final DataOfferLimitsEnforcer dataOfferLimitsEnforcer; + + public void handleConnectorOnline( + DSLContext dsl, + ConnectorRef connectorRef, + ConnectorRecord connector, + FetchedCatalog catalog + ) { + // Limit data offers and log limitation if necessary + var limitedDataOffers = dataOfferLimitsEnforcer.enforceLimits(catalog.getDataOffers()); + dataOfferLimitsEnforcer.logEnforcedLimitsIfChanged(dsl, connectorRef, connector, limitedDataOffers); + + // Log Status Change and set status to online if necessary + if (connector.getOnlineStatus() != ConnectorOnlineStatus.ONLINE || connector.getLastRefreshAttemptAt() == null) { + crawlerEventLogger.logConnectorOnline(dsl, connectorRef); + connector.setOnlineStatus(ConnectorOnlineStatus.ONLINE); + } + + // Track changes for final log message + var changes = new ConnectorChangeTracker(); + var now = OffsetDateTime.now(); + connector.setLastSuccessfulRefreshAt(now); + connector.setLastRefreshAttemptAt(now); + connector.update(); + + // Update data offers + connectorUpdateCatalogWriter.updateDataOffers( + dsl, + connectorRef, + limitedDataOffers.abbreviatedDataOffers(), + changes + ); + + // Log event if changes are present + if (!changes.isEmpty()) { + crawlerEventLogger.logConnectorUpdated(dsl, connectorRef, changes); + } + } + +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferLimitsEnforcer.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferLimitsEnforcer.java new file mode 100644 index 000000000..0583b93b3 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferLimitsEnforcer.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.crawling.writing; + +import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedDataOffer; +import de.sovity.edc.ext.catalog.crawler.crawling.logging.CrawlerEventLogger; +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; +import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorContractOffersExceeded; +import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorDataOffersExceeded; +import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ConnectorRecord; +import de.sovity.edc.ext.catalog.crawler.orchestration.config.CrawlerConfig; +import lombok.RequiredArgsConstructor; +import org.jooq.DSLContext; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +@RequiredArgsConstructor +public class DataOfferLimitsEnforcer { + private final CrawlerConfig crawlerConfig; + private final CrawlerEventLogger crawlerEventLogger; + + public record DataOfferLimitsEnforced( + Collection abbreviatedDataOffers, + boolean dataOfferLimitsExceeded, + boolean contractOfferLimitsExceeded + ) { + } + + public DataOfferLimitsEnforced enforceLimits(Collection dataOffers) { + // Get limits from config + var maxDataOffers = crawlerConfig.getMaxDataOffersPerConnector(); + var maxContractOffers = crawlerConfig.getMaxContractOffersPerDataOffer(); + List offerList = new ArrayList<>(dataOffers); + + // No limits set + if (maxDataOffers == -1 && maxContractOffers == -1) { + return new DataOfferLimitsEnforced(dataOffers, false, false); + } + + // Validate if limits exceeded + var dataOfferLimitsExceeded = false; + if (maxDataOffers != -1 && offerList.size() > maxDataOffers) { + offerList = offerList.subList(0, maxDataOffers); + dataOfferLimitsExceeded = true; + } + + var contractOfferLimitsExceeded = false; + for (var dataOffer : offerList) { + var contractOffers = dataOffer.getContractOffers(); + if (contractOffers != null && maxContractOffers != -1 && contractOffers.size() > maxContractOffers) { + dataOffer.setContractOffers(contractOffers.subList(0, maxContractOffers)); + contractOfferLimitsExceeded = true; + } + } + + // Create new list with limited offers + return new DataOfferLimitsEnforced(offerList, dataOfferLimitsExceeded, contractOfferLimitsExceeded); + } + + public void logEnforcedLimitsIfChanged( + DSLContext dsl, + ConnectorRef connectorRef, + ConnectorRecord connector, + DataOfferLimitsEnforced enforcedLimits + ) { + + // DataOffer + if (enforcedLimits.dataOfferLimitsExceeded() && connector.getDataOffersExceeded() == ConnectorDataOffersExceeded.OK) { + var maxDataOffers = crawlerConfig.getMaxDataOffersPerConnector(); + crawlerEventLogger.logConnectorUpdateDataOfferLimitExceeded(dsl, connectorRef, maxDataOffers); + connector.setDataOffersExceeded(ConnectorDataOffersExceeded.EXCEEDED); + } else if (!enforcedLimits.dataOfferLimitsExceeded() && connector.getDataOffersExceeded() == ConnectorDataOffersExceeded.EXCEEDED) { + crawlerEventLogger.logConnectorUpdateDataOfferLimitOk(dsl, connectorRef); + connector.setDataOffersExceeded(ConnectorDataOffersExceeded.OK); + } + + // ContractOffer + if (enforcedLimits.contractOfferLimitsExceeded() && connector.getContractOffersExceeded() == ConnectorContractOffersExceeded.OK) { + var maxContractOffers = crawlerConfig.getMaxContractOffersPerDataOffer(); + crawlerEventLogger.logConnectorUpdateContractOfferLimitExceeded(dsl, connectorRef, maxContractOffers); + connector.setContractOffersExceeded(ConnectorContractOffersExceeded.EXCEEDED); + } else if (!enforcedLimits.contractOfferLimitsExceeded() && + connector.getContractOffersExceeded() == ConnectorContractOffersExceeded.EXCEEDED) { + crawlerEventLogger.logConnectorUpdateContractOfferLimitOk(dsl, connectorRef); + connector.setContractOffersExceeded(ConnectorContractOffersExceeded.OK); + } + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/utils/ChangeTracker.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/utils/ChangeTracker.java new file mode 100644 index 000000000..dad7f9cae --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/utils/ChangeTracker.java @@ -0,0 +1,36 @@ +package de.sovity.edc.ext.catalog.crawler.crawling.writing.utils; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; + +import java.util.Objects; +import java.util.function.BiPredicate; +import java.util.function.Consumer; + +@Getter +@NoArgsConstructor +@AllArgsConstructor +public class ChangeTracker { + private boolean changed = false; + + public void setIfChanged( + T existing, + T fetched, + Consumer setter + ) { + setIfChanged(existing, fetched, setter, Objects::equals); + } + + public void setIfChanged( + T existing, + T fetched, + Consumer setter, + BiPredicate equalityChecker + ) { + if (!equalityChecker.test(existing, fetched)) { + setter.accept(fetched); + changed = true; + } + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/utils/DiffUtils.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/utils/DiffUtils.java new file mode 100644 index 000000000..7db2dac4f --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/utils/DiffUtils.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.crawling.writing.utils; + +import de.sovity.edc.ext.catalog.crawler.utils.MapUtils; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.function.Function; + + +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class DiffUtils { + /** + * Tries to match two collections by a key, then collects planned change sets as {@link DiffResult}. + * + * @param existing list of existing elements + * @param existingKeyFn existing elements key extractor + * @param fetched list of fetched elements + * @param fetchedKeyFn fetched elements key extractor + * @param first collection type + * @param second collection type + * @param key type + */ + public static DiffResult compareLists( + Collection existing, + Function existingKeyFn, + Collection fetched, + Function fetchedKeyFn + ) { + var existingByKey = MapUtils.associateBy(existing, existingKeyFn); + var fetchedByKey = MapUtils.associateBy(fetched, fetchedKeyFn); + + var keys = new HashSet<>(existingByKey.keySet()); + keys.addAll(fetchedByKey.keySet()); + + var result = new DiffResult(); + + keys.forEach(key -> { + var existingItem = existingByKey.get(key); + var fetchedItem = fetchedByKey.get(key); + + if (existingItem == null) { + result.added.add(fetchedItem); + } else if (fetchedItem == null) { + result.removed.add(existingItem); + } else { + result.updated.add(new DiffResultMatch<>(existingItem, fetchedItem)); + } + }); + + return result; + } + + /** + * Result of comparing two collections by keys. + * + * @param added elements that are present in fetched collection but not in existing + * @param updated elements that are present in both collections + * @param removed elements that are present in existing collection but not in fetched + * @param existing item type + * @param fetched item type + */ + public record DiffResult(List added, List> updated, List removed) { + DiffResult() { + this(new ArrayList<>(), new ArrayList<>(), new ArrayList<>()); + } + } + + /** + * Pair of elements that are present in both collections. + * + * @param existing existing item + * @param fetched fetched item + * @param existing item type + * @param fetched item type + */ + public record DiffResultMatch(A existing, B fetched) { + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogCleaner.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogCleaner.java new file mode 100644 index 000000000..87b17d337 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogCleaner.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.dao; + +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; +import de.sovity.edc.ext.catalog.crawler.dao.utils.PostgresqlUtils; +import de.sovity.edc.ext.catalog.crawler.db.jooq.Tables; +import lombok.RequiredArgsConstructor; +import org.jooq.DSLContext; + +import java.util.Collection; + +import static java.util.stream.Collectors.toSet; + +@RequiredArgsConstructor +public class CatalogCleaner { + + public void removeCatalogByConnectors(DSLContext dsl, Collection connectorRefs) { + var co = Tables.CONTRACT_OFFER; + var d = Tables.DATA_OFFER; + + var connectorIds = connectorRefs.stream().map(ConnectorRef::getConnectorId).collect(toSet()); + + dsl.deleteFrom(co).where(PostgresqlUtils.in(co.CONNECTOR_ID, connectorIds)).execute(); + dsl.deleteFrom(d).where(PostgresqlUtils.in(d.CONNECTOR_ID, connectorIds)).execute(); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogPatch.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogPatch.java new file mode 100644 index 000000000..8ad4ebc1e --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogPatch.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.dao; + +import de.sovity.edc.ext.catalog.crawler.dao.utils.RecordPatch; +import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ContractOfferRecord; +import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.DataOfferRecord; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; +import lombok.experimental.FieldDefaults; + +import java.util.List; + +/** + * Contains planned DB Row changes to be applied as batch. + */ +@Getter +@Setter +@Accessors(fluent = true) +@FieldDefaults(level = AccessLevel.PRIVATE) +public class CatalogPatch { + RecordPatch dataOffers = new RecordPatch<>(); + RecordPatch contractOffers = new RecordPatch<>(); + + public List> insertionOrder() { + return List.of(dataOffers, contractOffers); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogPatchApplier.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogPatchApplier.java new file mode 100644 index 000000000..9dce2fc9d --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogPatchApplier.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.dao; + +import de.sovity.edc.ext.catalog.crawler.dao.utils.RecordPatch; +import de.sovity.edc.ext.catalog.crawler.utils.CollectionUtils2; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.jooq.DSLContext; + +@RequiredArgsConstructor +public class CatalogPatchApplier { + + @SneakyThrows + public void applyDbUpdatesBatched(DSLContext dsl, CatalogPatch catalogPatch) { + var insertionOrder = catalogPatch.insertionOrder(); + var deletionOrder = CollectionUtils2.reverse(insertionOrder); + + insertionOrder.stream() + .map(RecordPatch::getInsertions) + .filter(CollectionUtils2::isNotEmpty) + .forEach(it -> dsl.batchInsert(it).execute()); + + insertionOrder.stream() + .map(RecordPatch::getUpdates) + .filter(CollectionUtils2::isNotEmpty) + .forEach(it -> dsl.batchUpdate(it).execute()); + + deletionOrder.stream() + .map(RecordPatch::getDeletions) + .filter(CollectionUtils2::isNotEmpty) + .forEach(it -> dsl.batchDelete(it).execute()); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/DataSourceFactory.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/DataSourceFactory.java new file mode 100644 index 000000000..b66997470 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/DataSourceFactory.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.dao.config; + +import com.zaxxer.hikari.HikariDataSource; +import de.sovity.edc.ext.catalog.crawler.CrawlerExtension; +import de.sovity.edc.extension.postgresql.HikariDataSourceFactory; +import de.sovity.edc.extension.postgresql.JdbcCredentials; +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.Validate; +import org.eclipse.edc.spi.system.configuration.Config; + +import javax.sql.DataSource; + +@RequiredArgsConstructor +public class DataSourceFactory { + private final Config config; + + + /** + * Create a new {@link DataSource} from EDC Config. + * + * @return {@link DataSource}. + */ + public HikariDataSource newDataSource() { + var jdbcCredentials = getJdbcCredentials(); + int maxPoolSize = config.getInteger(CrawlerExtension.DB_CONNECTION_POOL_SIZE); + int connectionTimeoutInMs = config.getInteger(CrawlerExtension.DB_CONNECTION_TIMEOUT_IN_MS); + return HikariDataSourceFactory.newDataSource( + jdbcCredentials, + maxPoolSize, + connectionTimeoutInMs + ); + } + + + public JdbcCredentials getJdbcCredentials() { + return new JdbcCredentials( + getRequiredStringProperty(config, CrawlerExtension.JDBC_URL), + getRequiredStringProperty(config, CrawlerExtension.JDBC_USER), + getRequiredStringProperty(config, CrawlerExtension.JDBC_PASSWORD) + ); + } + + private String getRequiredStringProperty(Config config, String name) { + String value = config.getString(name, ""); + Validate.notBlank(value, "EDC Property '%s' is required".formatted(name)); + return value; + } + +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/DslContextFactory.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/DslContextFactory.java new file mode 100644 index 000000000..6f97871bb --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/DslContextFactory.java @@ -0,0 +1,67 @@ +package de.sovity.edc.ext.catalog.crawler.dao.config; + +import lombok.RequiredArgsConstructor; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.jooq.impl.DSL; + +import java.util.function.Consumer; +import java.util.function.Function; +import javax.sql.DataSource; + +/** + * Quickly launch {@link org.jooq.DSLContext}s from EDC configuration. + */ +@RequiredArgsConstructor +public class DslContextFactory { + private final DataSource dataSource; + + /** + * Create new {@link DSLContext} for querying DB. + * + * @return new {@link DSLContext} + */ + public DSLContext newDslContext() { + return DSL.using(dataSource, SQLDialect.POSTGRES); + } + + /** + * Utility method for when the {@link DSLContext} will be used only for a single transaction. + *
+ * An example would be a REST request. + * + * @param return type + * @return new {@link DSLContext} + opened transaction + */ + public R transactionResult(Function function) { + return newDslContext().transactionResult(transaction -> function.apply(transaction.dsl())); + } + + /** + * Utility method for when the {@link DSLContext} will be used only for a single transaction. + *
+ * An example would be a REST request. + */ + public void transaction(Consumer function) { + newDslContext().transaction(transaction -> function.accept(transaction.dsl())); + } + + /** + * Runs given code within a test transaction. + * + * @param code code to run within the test transaction + */ + public void testTransaction(Consumer code) { + try { + transaction(dsl -> { + code.accept(dsl); + throw new TestTransactionNoopException(); + }); + } catch (TestTransactionNoopException e) { + // Ignore + } + } + + private static class TestTransactionNoopException extends RuntimeException { + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/FlywayService.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/FlywayService.java new file mode 100644 index 000000000..dee88e32b --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/FlywayService.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.dao.config; + +import de.sovity.edc.ext.catalog.crawler.CrawlerExtension; +import de.sovity.edc.extension.postgresql.FlywayExecutionParams; +import de.sovity.edc.extension.postgresql.FlywayUtils; +import lombok.RequiredArgsConstructor; +import org.eclipse.edc.spi.monitor.Monitor; +import org.eclipse.edc.spi.system.configuration.Config; + +import javax.sql.DataSource; + +@RequiredArgsConstructor +public class FlywayService { + private final Config config; + private final Monitor monitor; + private final DataSource dataSource; + + public void validateOrMigrateInTests() { + var additionalLocations = config.getString(CrawlerExtension.DB_ADDITIONAL_FLYWAY_MIGRATION_LOCATIONS, ""); + + var params = baseConfig(additionalLocations) + .clean(config.getBoolean(CrawlerExtension.DB_CLEAN, false)) + .cleanEnabled(config.getBoolean(CrawlerExtension.DB_CLEAN_ENABLED, false)) + .migrate(config.getBoolean(CrawlerExtension.DB_MIGRATE, false)) + .infoLogger(monitor::info) + .build(); + + FlywayUtils.cleanAndMigrate(params, dataSource); + } + + public static FlywayExecutionParams.FlywayExecutionParamsBuilder baseConfig(String additionalMigrationLocations) { + var migrationLocations = FlywayUtils.parseFlywayLocations( + "classpath:db-crawler/migration,%s".formatted(additionalMigrationLocations) + ); + + return FlywayExecutionParams.builder() + .migrationLocations(migrationLocations) + .table("flyway_schema_history"); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorQueries.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorQueries.java new file mode 100644 index 000000000..4c3e3843b --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorQueries.java @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.dao.connectors; + +import de.sovity.edc.ext.catalog.crawler.db.jooq.Tables; +import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorOnlineStatus; +import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.Connector; +import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.Organization; +import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ConnectorRecord; +import de.sovity.edc.ext.catalog.crawler.orchestration.config.CrawlerConfig; +import lombok.RequiredArgsConstructor; +import org.jetbrains.annotations.NotNull; +import org.jooq.Condition; +import org.jooq.DSLContext; + +import java.time.Duration; +import java.time.OffsetDateTime; +import java.util.HashSet; +import java.util.Set; +import java.util.function.BiFunction; + +@RequiredArgsConstructor +public class ConnectorQueries { + private final CrawlerConfig crawlerConfig; + + public ConnectorRecord findByConnectorId(DSLContext dsl, String connectorId) { + var c = Tables.CONNECTOR; + return dsl.fetchOne(c, c.CONNECTOR_ID.eq(connectorId)); + } + + public Set findConnectorsForScheduledRefresh(DSLContext dsl, ConnectorOnlineStatus onlineStatus) { + return queryConnectorRefs(dsl, (c, o) -> c.ONLINE_STATUS.eq(onlineStatus)); + } + + public Set findAllConnectorsForKilling(DSLContext dsl, Duration deleteOfflineConnectorsAfter) { + var minLastRefresh = OffsetDateTime.now().minus(deleteOfflineConnectorsAfter); + return queryConnectorRefs(dsl, (c, o) -> c.LAST_SUCCESSFUL_REFRESH_AT.lt(minLastRefresh)); + } + + @NotNull + private Set queryConnectorRefs( + DSLContext dsl, + BiFunction condition + ) { + var c = Tables.CONNECTOR; + var o = Tables.ORGANIZATION; + var query = dsl.select( + c.CONNECTOR_ID.as("connectorId"), + c.ENVIRONMENT.as("environmentId"), + o.NAME.as("organizationLegalName"), + o.MDS_ID.as("organizationId"), + c.ENDPOINT_URL.as("endpoint") + ) + .from(c) + .leftJoin(o).on(c.MDS_ID.eq(o.MDS_ID)) + .where(condition.apply(c, o), c.ENVIRONMENT.eq(crawlerConfig.getEnvironmentId())) + .fetchInto(ConnectorRef.class); + + return new HashSet<>(query); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorRef.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorRef.java new file mode 100644 index 000000000..3d94fca72 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorRef.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.dao.connectors; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +@Getter +@RequiredArgsConstructor +@EqualsAndHashCode(of = "connectorId", callSuper = false) +@ToString(of = "connectorId") +public class ConnectorRef { + private final String connectorId; + private final String environmentId; + private final String organizationLegalName; + private final String organizationId; + private final String endpoint; +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorStatusUpdater.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorStatusUpdater.java new file mode 100644 index 000000000..6dfad8125 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorStatusUpdater.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.dao.connectors; + +import de.sovity.edc.ext.catalog.crawler.dao.utils.PostgresqlUtils; +import de.sovity.edc.ext.catalog.crawler.db.jooq.Tables; +import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorOnlineStatus; +import org.jooq.DSLContext; + +import java.util.Collection; +import java.util.stream.Collectors; + +public class ConnectorStatusUpdater { + public void markAsDead(DSLContext dsl, Collection connectorRefs) { + var connectorIds = connectorRefs.stream() + .map(ConnectorRef::getConnectorId) + .collect(Collectors.toSet()); + var c = Tables.CONNECTOR; + dsl.update(c).set(c.ONLINE_STATUS, ConnectorOnlineStatus.DEAD) + .where(PostgresqlUtils.in(c.CONNECTOR_ID, connectorIds)).execute(); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/contract_offers/ContractOfferQueries.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/contract_offers/ContractOfferQueries.java new file mode 100644 index 000000000..0d350e429 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/contract_offers/ContractOfferQueries.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.dao.contract_offers; + +import de.sovity.edc.ext.catalog.crawler.db.jooq.Tables; +import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ContractOfferRecord; +import org.jooq.DSLContext; + +import java.util.List; + +public class ContractOfferQueries { + + public List findByConnectorId(DSLContext dsl, String connectorId) { + var co = Tables.CONTRACT_OFFER; + return dsl.selectFrom(co).where(co.CONNECTOR_ID.eq(connectorId)).stream().toList(); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/contract_offers/ContractOfferRecordUpdater.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/contract_offers/ContractOfferRecordUpdater.java new file mode 100644 index 000000000..ed23d2f07 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/contract_offers/ContractOfferRecordUpdater.java @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.dao.contract_offers; + +import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedContractOffer; +import de.sovity.edc.ext.catalog.crawler.crawling.writing.utils.ChangeTracker; +import de.sovity.edc.ext.catalog.crawler.dao.utils.JsonbUtils; +import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ContractOfferRecord; +import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.DataOfferRecord; +import de.sovity.edc.ext.catalog.crawler.utils.JsonUtils2; +import lombok.RequiredArgsConstructor; +import org.jooq.JSONB; + +import java.time.OffsetDateTime; + +/** + * Creates or updates {@link ContractOfferRecord} DB Rows. + *

+ * (Or at least prepares them for batch inserts / updates) + */ +@RequiredArgsConstructor +public class ContractOfferRecordUpdater { + + /** + * Create new {@link ContractOfferRecord} from {@link FetchedContractOffer}. + * + * @param dataOffer parent data offer db row + * @param fetchedContractOffer fetched contract offer + * @return new db row + */ + public ContractOfferRecord newContractOffer( + DataOfferRecord dataOffer, + FetchedContractOffer fetchedContractOffer + ) { + var contractOffer = new ContractOfferRecord(); + + contractOffer.setConnectorId(dataOffer.getConnectorId()); + contractOffer.setContractOfferId(fetchedContractOffer.getContractOfferId()); + contractOffer.setAssetId(dataOffer.getAssetId()); + contractOffer.setCreatedAt(OffsetDateTime.now()); + updateContractOffer(contractOffer, fetchedContractOffer); + return contractOffer; + } + + /** + * Update existing {@link ContractOfferRecord} with changes from {@link FetchedContractOffer}. + * + * @param contractOffer existing row + * @param fetchedContractOffer changes to be integrated + * @return if anything was changed + */ + public boolean updateContractOffer( + ContractOfferRecord contractOffer, + FetchedContractOffer fetchedContractOffer + ) { + var changes = new ChangeTracker(); + + changes.setIfChanged( + JsonbUtils.getDataOrNull(contractOffer.getUiPolicyJson()), + fetchedContractOffer.getUiPolicyJson(), + it -> contractOffer.setUiPolicyJson(JSONB.jsonb(it)), + JsonUtils2::isEqualJson + ); + + if (changes.isChanged()) { + contractOffer.setUpdatedAt(OffsetDateTime.now()); + } + + return changes.isChanged(); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/data_offers/DataOfferQueries.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/data_offers/DataOfferQueries.java new file mode 100644 index 000000000..f4a82b14c --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/data_offers/DataOfferQueries.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.dao.data_offers; + +import de.sovity.edc.ext.catalog.crawler.db.jooq.Tables; +import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.DataOfferRecord; +import lombok.RequiredArgsConstructor; +import org.jooq.DSLContext; + +import java.util.List; + +@RequiredArgsConstructor +public class DataOfferQueries { + + public List findByConnectorId(DSLContext dsl, String connectorId) { + var d = Tables.DATA_OFFER; + return dsl.selectFrom(d).where(d.CONNECTOR_ID.eq(connectorId)).stream().toList(); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/data_offers/DataOfferRecordUpdater.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/data_offers/DataOfferRecordUpdater.java new file mode 100644 index 000000000..955c05271 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/data_offers/DataOfferRecordUpdater.java @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.dao.data_offers; + +import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedDataOffer; +import de.sovity.edc.ext.catalog.crawler.crawling.writing.utils.ChangeTracker; +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorQueries; +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; +import de.sovity.edc.ext.catalog.crawler.dao.utils.JsonbUtils; +import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.DataOfferRecord; +import de.sovity.edc.ext.catalog.crawler.utils.JsonUtils2; +import lombok.RequiredArgsConstructor; +import org.jooq.JSONB; + +import java.time.OffsetDateTime; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +/** + * Creates or updates {@link DataOfferRecord} DB Rows. + *

+ * (Or at least prepares them for batch inserts / updates) + */ +@RequiredArgsConstructor +public class DataOfferRecordUpdater { + + private final ConnectorQueries connectorQueries; + + /** + * Create a new {@link DataOfferRecord}. + * + * @param connectorRef connector + * @param fetchedDataOffer new db row data + * @return new db row + */ + public DataOfferRecord newDataOffer( + ConnectorRef connectorRef, + FetchedDataOffer fetchedDataOffer + ) { + var dataOffer = new DataOfferRecord(); + var connectorId = connectorRef.getConnectorId(); + + dataOffer.setConnectorId(connectorId); + dataOffer.setAssetId(fetchedDataOffer.getAssetId()); + dataOffer.setCreatedAt(OffsetDateTime.now()); + updateDataOffer(dataOffer, fetchedDataOffer, true); + return dataOffer; + } + + + /** + * Update existing {@link DataOfferRecord}. + * + * @param record existing row + * @param fetchedDataOffer changes to be incorporated + * @param changed whether the data offer should be marked as updated simply because the contract offers changed + * @return whether any fields were updated + */ + public boolean updateDataOffer( + DataOfferRecord record, + FetchedDataOffer fetchedDataOffer, + boolean changed + ) { + var asset = fetchedDataOffer.getUiAsset(); + var changes = new ChangeTracker(changed); + + changes.setIfChanged( + blankIfNull(record.getAssetTitle()), + blankIfNull(asset.getTitle()), + record::setAssetTitle + ); + + changes.setIfChanged( + blankIfNull(record.getDescription()), + blankIfNull(asset.getDescription()), + record::setDescription + ); + + changes.setIfChanged( + blankIfNull(record.getDataCategory()), + blankIfNull(asset.getDataCategory()), + record::setDataCategory + ); + + changes.setIfChanged( + blankIfNull(record.getDataSubcategory()), + blankIfNull(asset.getDataSubcategory()), + record::setDataSubcategory + ); + + changes.setIfChanged( + blankIfNull(record.getDataModel()), + blankIfNull(asset.getDataModel()), + record::setDataModel + ); + + changes.setIfChanged( + blankIfNull(record.getTransportMode()), + blankIfNull(asset.getTransportMode()), + record::setTransportMode + ); + + changes.setIfChanged( + blankIfNull(record.getGeoReferenceMethod()), + blankIfNull(asset.getGeoReferenceMethod()), + record::setGeoReferenceMethod + ); + + changes.setIfChanged( + emptyIfNull(record.getKeywords()), + emptyIfNull(asset.getKeywords()), + it -> { + record.setKeywords(it.toArray(new String[0])); + record.setKeywordsCommaJoined(String.join(", ", it)); + } + ); + + changes.setIfChanged( + JsonbUtils.getDataOrNull(record.getUiAssetJson()), + fetchedDataOffer.getUiAssetJson(), + it -> record.setUiAssetJson(JSONB.jsonb(it)), + JsonUtils2::isEqualJson + ); + + if (changes.isChanged()) { + record.setUpdatedAt(OffsetDateTime.now()); + } + + return changes.isChanged(); + } + + private String blankIfNull(String string) { + return string == null ? "" : string; + } + + private Collection emptyIfNull(Collection collection) { + return collection == null ? List.of() : collection; + } + + private Collection emptyIfNull(T[] array) { + return array == null ? List.of() : Arrays.asList(array); + } + +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/utils/JsonbUtils.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/utils/JsonbUtils.java new file mode 100644 index 000000000..c3afdbabd --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/utils/JsonbUtils.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.dao.utils; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import org.jooq.JSONB; + +/** + * Utilities for dealing with {@link org.jooq.JSONB} fields. + */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class JsonbUtils { + + /** + * Returns the data of the given {@link JSONB} or null. + * + * @param jsonb {@link org.jooq.JSON} + * @return data or null + */ + public static String getDataOrNull(JSONB jsonb) { + return jsonb == null ? null : jsonb.data(); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/utils/PostgresqlUtils.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/utils/PostgresqlUtils.java new file mode 100644 index 000000000..4c7cd3e41 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/utils/PostgresqlUtils.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.dao.utils; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.impl.DSL; + +import java.util.Collection; + +/** + * PostgreSQL + JooQ Utils + */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class PostgresqlUtils { + + /** + * Replaces the IN operation with "field = ANY(...)" + * + * @param field field + * @param values values + * @return condition + */ + public static Condition in(Field field, Collection values) { + return field.eq(DSL.any(values.toArray(String[]::new))); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/utils/RecordPatch.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/utils/RecordPatch.java new file mode 100644 index 000000000..99c6a025a --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/utils/RecordPatch.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.dao.utils; + +import lombok.AccessLevel; +import lombok.Getter; +import lombok.experimental.FieldDefaults; +import org.jooq.UpdatableRecord; + +import java.util.ArrayList; +import java.util.List; + +/** + * Contains planned DB Row changes to be applied as batch. + */ +@Getter +@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) +public class RecordPatch> { + List insertions = new ArrayList<>(); + List updates = new ArrayList<>(); + List deletions = new ArrayList<>(); + + public void insert(T record) { + insertions.add(record); + } + + public void update(T record) { + updates.add(record); + } + + public void delete(T record) { + deletions.add(record); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/CrawlerConfig.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/CrawlerConfig.java new file mode 100644 index 000000000..af3e2a0f4 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/CrawlerConfig.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.orchestration.config; + +import lombok.Builder; +import lombok.Value; + +import java.time.Duration; + +@Value +@Builder +public class CrawlerConfig { + String environmentId; + int numThreads; + + Duration killOfflineConnectorsAfter; + + int maxDataOffersPerConnector; + int maxContractOffersPerDataOffer; +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/CrawlerConfigFactory.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/CrawlerConfigFactory.java new file mode 100644 index 000000000..f40e6b6af --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/CrawlerConfigFactory.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.orchestration.config; + +import de.sovity.edc.ext.catalog.crawler.CrawlerExtension; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.eclipse.edc.spi.system.configuration.Config; + +import java.time.Duration; + +@RequiredArgsConstructor +public class CrawlerConfigFactory { + private final Config config; + + public CrawlerConfig buildCrawlerConfig() { + var environmentId = config.getString(CrawlerExtension.ENVIRONMENT_ID); + var numThreads = config.getInteger(CrawlerExtension.NUM_THREADS, 1); + var killOfflineConnectorsAfter = getDuration(CrawlerExtension.KILL_OFFLINE_CONNECTORS_AFTER, Duration.ofDays(5)); + var maxDataOffers = config.getInteger(CrawlerExtension.MAX_DATA_OFFERS_PER_CONNECTOR, -1); + var maxContractOffers = config.getInteger(CrawlerExtension.MAX_CONTRACT_OFFERS_PER_DATA_OFFER, -1); + + return CrawlerConfig.builder() + .environmentId(environmentId) + .numThreads(numThreads) + .killOfflineConnectorsAfter(killOfflineConnectorsAfter) + .maxDataOffersPerConnector(maxDataOffers) + .maxContractOffersPerDataOffer(maxContractOffers) + .build(); + } + + private Duration getDuration(@NonNull String configProperty, Duration defaultValue) { + var value = config.getString(configProperty, ""); + + if (StringUtils.isBlank(value)) { + return defaultValue; + } + + return Duration.parse(value); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/EdcConfigPropertyUtils.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/EdcConfigPropertyUtils.java new file mode 100644 index 000000000..c0d9bdb5f --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/EdcConfigPropertyUtils.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.orchestration.config; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + +import java.util.Arrays; + +import static java.util.stream.Collectors.joining; + +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class EdcConfigPropertyUtils { + /** + * For better refactoring it is better if the string constant is + * found in the code as it is used and documented. + * + * @param envVarName e.g. "MY_EDC_PROP" + * @return e.g. "my.edc.prop" + */ + public static String toEdcProp(String envVarName) { + return Arrays.stream(envVarName.split("_")) + .map(String::toLowerCase) + .collect(joining(".")); + } + +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorQueue.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorQueue.java new file mode 100644 index 000000000..e46808a3a --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorQueue.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.orchestration.queue; + +import de.sovity.edc.ext.catalog.crawler.crawling.ConnectorCrawler; +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; +import lombok.RequiredArgsConstructor; + +import java.util.ArrayList; +import java.util.Collection; + +@RequiredArgsConstructor +public class ConnectorQueue { + private final ConnectorCrawler connectorCrawler; + private final ThreadPool threadPool; + + /** + * Enqueues connectors for update. + * + * @param connectorRefs connectors + * @param priority priority from {@link ConnectorRefreshPriority} + */ + public void addAll(Collection connectorRefs, int priority) { + var queued = threadPool.getQueuedConnectorRefs(); + connectorRefs = new ArrayList<>(connectorRefs); + connectorRefs.removeIf(queued::contains); + + for (var connectorRef : connectorRefs) { + threadPool.enqueueConnectorRefreshTask( + priority, + () -> connectorCrawler.crawlConnector(connectorRef), + connectorRef + ); + } + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorQueueFiller.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorQueueFiller.java new file mode 100644 index 000000000..aaf2d7da9 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorQueueFiller.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.orchestration.queue; + +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorQueries; +import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorOnlineStatus; +import lombok.RequiredArgsConstructor; +import org.jooq.DSLContext; + +@RequiredArgsConstructor +public class ConnectorQueueFiller { + private final ConnectorQueue connectorQueue; + private final ConnectorQueries connectorQueries; + + public void enqueueConnectors(DSLContext dsl, ConnectorOnlineStatus status, int priority) { + var connectorRefs = connectorQueries.findConnectorsForScheduledRefresh(dsl, status); + connectorQueue.addAll(connectorRefs, priority); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorRefreshPriority.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorRefreshPriority.java new file mode 100644 index 000000000..541c8528b --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorRefreshPriority.java @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.orchestration.queue; + +import lombok.NoArgsConstructor; + +@NoArgsConstructor(access = lombok.AccessLevel.PRIVATE) +public class ConnectorRefreshPriority { + public static final int SCHEDULED_ONLINE_REFRESH = 100; + public static final int SCHEDULED_OFFLINE_REFRESH = 200; + public static final int SCHEDULED_DEAD_REFRESH = 300; +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPool.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPool.java new file mode 100644 index 000000000..857bf32ed --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPool.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.orchestration.queue; + +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; +import de.sovity.edc.ext.catalog.crawler.orchestration.config.CrawlerConfig; +import org.eclipse.edc.spi.monitor.Monitor; + +import java.util.Set; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +public class ThreadPool { + private final ThreadPoolTaskQueue queue; + + private final boolean enabled; + private final ThreadPoolExecutor threadPoolExecutor; + + public ThreadPool(ThreadPoolTaskQueue queue, CrawlerConfig crawlerConfig, Monitor monitor) { + this.queue = queue; + int numThreads = crawlerConfig.getNumThreads(); + enabled = numThreads > 0; + + if (enabled) { + monitor.info("Initializing ThreadPoolExecutor with %d threads.".formatted(numThreads)); + threadPoolExecutor = new ThreadPoolExecutor( + numThreads, + numThreads, + 60, + TimeUnit.SECONDS, + queue.getAsRunnableQueue() + ); + threadPoolExecutor.prestartAllCoreThreads(); + } else { + monitor.info("Skipped ThreadPoolExecutor initialization."); + threadPoolExecutor = null; + } + } + + public void enqueueConnectorRefreshTask(int priority, Runnable runnable, ConnectorRef connectorRef) { + enqueueTask(new ThreadPoolTask(priority, runnable, connectorRef)); + } + + public Set getQueuedConnectorRefs() { + return queue.getConnectorRefs(); + } + + private void enqueueTask(ThreadPoolTask task) { + if (enabled) { + threadPoolExecutor.execute(task); + } else { + // Only relevant for test environment, where execution is disabled + queue.add(task); + } + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPoolTask.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPoolTask.java new file mode 100644 index 000000000..66712b2d7 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPoolTask.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.orchestration.queue; + +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +import java.util.Comparator; +import java.util.concurrent.atomic.AtomicLong; + + +@Getter +@RequiredArgsConstructor +public class ThreadPoolTask implements Runnable { + + public static final Comparator COMPARATOR = Comparator.comparing(ThreadPoolTask::getPriority) + .thenComparing(ThreadPoolTask::getSequence); + + /** + * {@link java.util.concurrent.PriorityBlockingQueue} does not guarantee sequential execution, so we need to add this. + */ + private static final AtomicLong SEQ = new AtomicLong(0); + private final long sequence = SEQ.incrementAndGet(); + private final int priority; + private final Runnable task; + private final ConnectorRef connectorRef; + + @Override + public void run() { + this.task.run(); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPoolTaskQueue.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPoolTaskQueue.java new file mode 100644 index 000000000..dc2b5598b --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPoolTaskQueue.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.orchestration.queue; + +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +import java.util.ArrayList; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.PriorityBlockingQueue; +import java.util.stream.Collectors; + +@RequiredArgsConstructor +public class ThreadPoolTaskQueue { + + @Getter + private final PriorityBlockingQueue queue = new PriorityBlockingQueue<>(50, ThreadPoolTask.COMPARATOR); + + @SuppressWarnings("unchecked") + public PriorityBlockingQueue getAsRunnableQueue() { + return (PriorityBlockingQueue) (PriorityBlockingQueue) queue; + } + + public void add(ThreadPoolTask task) { + queue.add(task); + } + + public Set getConnectorRefs() { + var queuedRunnables = new ArrayList<>(queue); + + return queuedRunnables.stream() + .map(ThreadPoolTask::getConnectorRef) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/DeadConnectorRefreshJob.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/DeadConnectorRefreshJob.java new file mode 100644 index 000000000..4f229b7fc --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/DeadConnectorRefreshJob.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.orchestration.schedules; + +import de.sovity.edc.ext.catalog.crawler.dao.config.DslContextFactory; +import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorOnlineStatus; +import de.sovity.edc.ext.catalog.crawler.orchestration.queue.ConnectorQueueFiller; +import de.sovity.edc.ext.catalog.crawler.orchestration.queue.ConnectorRefreshPriority; +import lombok.RequiredArgsConstructor; +import org.quartz.Job; +import org.quartz.JobExecutionContext; + +@RequiredArgsConstructor +public class DeadConnectorRefreshJob implements Job { + private final DslContextFactory dslContextFactory; + private final ConnectorQueueFiller connectorQueueFiller; + + @Override + public void execute(JobExecutionContext context) { + dslContextFactory.transaction(dsl -> connectorQueueFiller.enqueueConnectors(dsl, + ConnectorOnlineStatus.DEAD, ConnectorRefreshPriority.SCHEDULED_DEAD_REFRESH)); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorCleanerJob.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorCleanerJob.java new file mode 100644 index 000000000..2c2e71853 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorCleanerJob.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.orchestration.schedules; + +import de.sovity.edc.ext.catalog.crawler.crawling.OfflineConnectorCleaner; +import de.sovity.edc.ext.catalog.crawler.dao.config.DslContextFactory; +import lombok.RequiredArgsConstructor; +import org.quartz.Job; +import org.quartz.JobExecutionContext; + +@RequiredArgsConstructor +public class OfflineConnectorCleanerJob implements Job { + private final DslContextFactory dslContextFactory; + private final OfflineConnectorCleaner offlineConnectorCleaner; + + @Override + public void execute(JobExecutionContext context) { + dslContextFactory.transaction(offlineConnectorCleaner::cleanConnectorsIfOfflineTooLong); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorRefreshJob.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorRefreshJob.java new file mode 100644 index 000000000..18965edf6 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorRefreshJob.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.orchestration.schedules; + +import de.sovity.edc.ext.catalog.crawler.dao.config.DslContextFactory; +import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorOnlineStatus; +import de.sovity.edc.ext.catalog.crawler.orchestration.queue.ConnectorQueueFiller; +import de.sovity.edc.ext.catalog.crawler.orchestration.queue.ConnectorRefreshPriority; +import lombok.RequiredArgsConstructor; +import org.quartz.Job; +import org.quartz.JobExecutionContext; + +@RequiredArgsConstructor +public class OfflineConnectorRefreshJob implements Job { + private final DslContextFactory dslContextFactory; + private final ConnectorQueueFiller connectorQueueFiller; + + @Override + public void execute(JobExecutionContext context) { + dslContextFactory.transaction(dsl -> connectorQueueFiller.enqueueConnectors(dsl, + ConnectorOnlineStatus.OFFLINE, ConnectorRefreshPriority.SCHEDULED_OFFLINE_REFRESH)); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OnlineConnectorRefreshJob.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OnlineConnectorRefreshJob.java new file mode 100644 index 000000000..6cad1899f --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OnlineConnectorRefreshJob.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.orchestration.schedules; + +import de.sovity.edc.ext.catalog.crawler.dao.config.DslContextFactory; +import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorOnlineStatus; +import de.sovity.edc.ext.catalog.crawler.orchestration.queue.ConnectorQueueFiller; +import de.sovity.edc.ext.catalog.crawler.orchestration.queue.ConnectorRefreshPriority; +import lombok.RequiredArgsConstructor; +import org.quartz.Job; +import org.quartz.JobExecutionContext; + +@RequiredArgsConstructor +public class OnlineConnectorRefreshJob implements Job { + private final DslContextFactory dslContextFactory; + private final ConnectorQueueFiller connectorQueueFiller; + + @Override + public void execute(JobExecutionContext context) { + dslContextFactory.transaction(dsl -> connectorQueueFiller.enqueueConnectors(dsl, + ConnectorOnlineStatus.ONLINE, ConnectorRefreshPriority.SCHEDULED_ONLINE_REFRESH)); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/QuartzScheduleInitializer.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/QuartzScheduleInitializer.java new file mode 100644 index 000000000..d4f4597e7 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/QuartzScheduleInitializer.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.orchestration.schedules; + +import de.sovity.edc.ext.catalog.crawler.orchestration.schedules.utils.CronJobRef; +import de.sovity.edc.ext.catalog.crawler.orchestration.schedules.utils.JobFactoryImpl; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.apache.commons.lang3.StringUtils; +import org.eclipse.edc.spi.monitor.Monitor; +import org.eclipse.edc.spi.system.configuration.Config; +import org.quartz.CronScheduleBuilder; +import org.quartz.JobBuilder; +import org.quartz.Scheduler; +import org.quartz.TriggerBuilder; +import org.quartz.impl.StdSchedulerFactory; + +import java.util.Collection; + +@RequiredArgsConstructor +public class QuartzScheduleInitializer { + private final Config config; + private final Monitor monitor; + private final Collection> jobs; + + @SneakyThrows + public void startSchedules() { + var jobFactory = new JobFactoryImpl(jobs); + var scheduler = StdSchedulerFactory.getDefaultScheduler(); + scheduler.setJobFactory(jobFactory); + + jobs.forEach(job -> scheduleCronJob(scheduler, job)); + scheduler.start(); + } + + @SneakyThrows + private void scheduleCronJob(Scheduler scheduler, CronJobRef cronJobRef) { + // CRON property name doubles as job name + var jobName = cronJobRef.configPropertyName(); + + // Skip scheduling if property not provided to ensure tests have no schedules running + var cronTrigger = config.getString(jobName, ""); + if (StringUtils.isBlank(cronTrigger)) { + monitor.info("No cron trigger configured for %s. Skipping.".formatted(jobName)); + return; + } + + monitor.info("Starting schedule %s=%s.".formatted(jobName, cronTrigger)); + var job = JobBuilder.newJob(cronJobRef.clazz()) + .withIdentity(jobName) + .build(); + var trigger = TriggerBuilder.newTrigger() + .withIdentity(jobName) + .withSchedule(CronScheduleBuilder.cronSchedule(cronTrigger)) + .build(); + + scheduler.scheduleJob(job, trigger); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/utils/CronJobRef.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/utils/CronJobRef.java new file mode 100644 index 000000000..8f435fd23 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/utils/CronJobRef.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.orchestration.schedules.utils; + +import org.quartz.Job; + +import java.util.function.Supplier; + +/** + * CRON Job. + * + * @param configPropertyName EDC Config property that decides cron expression + * @param clazz class of the job + * @param factory factory that initializes the task class + * @param job type + */ +public record CronJobRef( + String configPropertyName, + Class clazz, + Supplier factory +) { + + @SuppressWarnings("unchecked") + public Supplier asJobSupplier() { + return (Supplier) factory; + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/utils/JobFactoryImpl.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/utils/JobFactoryImpl.java new file mode 100644 index 000000000..003f0161d --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/utils/JobFactoryImpl.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.orchestration.schedules.utils; + +import lombok.NonNull; +import org.quartz.Job; +import org.quartz.Scheduler; +import org.quartz.spi.JobFactory; +import org.quartz.spi.TriggerFiredBundle; + +import java.util.Collection; +import java.util.Map; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +public class JobFactoryImpl implements JobFactory { + private final Map, Supplier> factories; + + public JobFactoryImpl(@NonNull Collection> jobs) { + factories = jobs.stream().collect(Collectors.toMap( + CronJobRef::clazz, + CronJobRef::asJobSupplier + )); + } + + @Override + public Job newJob(TriggerFiredBundle bundle, Scheduler scheduler) { + Class jobClazz = bundle.getJobDetail().getJobClass(); + Supplier factory = factories.get(jobClazz); + if (factory == null) { + throw new IllegalArgumentException("No factory for Job class %s. Supported Job classes are: %s.".formatted( + jobClazz.getName(), + factories.keySet().stream().map(Class::getName).collect(Collectors.joining(", ")) + )); + } + return factory.get(); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/CollectionUtils2.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/CollectionUtils2.java new file mode 100644 index 000000000..f113d8601 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/CollectionUtils2.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.utils; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.NonNull; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class CollectionUtils2 { + /** + * Set Difference + * + * @param a base set + * @param b remove these items + * @param set item type + * @return a difference b + */ + public static Set difference(@NonNull Collection a, @NonNull Collection b) { + var result = new HashSet<>(a); + result.removeAll(b); + return result; + } + + public static boolean isNotEmpty(Collection collection) { + return collection != null && !collection.isEmpty(); + } + + public static List reverse(List source) { + var result = new ArrayList<>(source); + java.util.Collections.reverse(result); + return result; + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/JsonUtils2.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/JsonUtils2.java new file mode 100644 index 000000000..a1d2d57bb --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/JsonUtils2.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.utils; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.SneakyThrows; + +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class JsonUtils2 { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + @SneakyThrows + public static boolean isEqualJson(String json, String otherJson) { + return (json == null && otherJson == null) || + (json != null && otherJson != null && + OBJECT_MAPPER.readTree(json).equals(OBJECT_MAPPER.readTree(otherJson))); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/MapUtils.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/MapUtils.java new file mode 100644 index 000000000..8c2cd3a1d --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/MapUtils.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.utils; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + +import java.util.Collection; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class MapUtils { + public static Map associateBy(Collection collection, Function keyExtractor) { + return collection.stream().collect(Collectors.toMap(keyExtractor, Function.identity(), (a, b) -> { + throw new IllegalStateException("Duplicate key %s.".formatted(keyExtractor.apply(a))); + })); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/StringUtils2.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/StringUtils2.java new file mode 100644 index 000000000..190f16ede --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/StringUtils2.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.utils; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.NonNull; + +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class StringUtils2 { + + /** + * Removes the suffix from the given string if it ends with it. + * + * @param string string + * @param suffix suffix to remove + * @return string without suffix + */ + public static String removeSuffix(@NonNull String string, @NonNull String suffix) { + if (string.endsWith(suffix)) { + return string.substring(0, string.length() - suffix.length()); + } + return string; + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/catalog-crawler/catalog-crawler/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 000000000..5a369119d --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +de.sovity.edc.ext.catalog.crawler.CrawlerExtension diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/AssertionUtils.java b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/AssertionUtils.java new file mode 100644 index 000000000..aef8012b7 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/AssertionUtils.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.SneakyThrows; +import org.skyscreamer.jsonassert.JSONAssert; +import org.skyscreamer.jsonassert.JSONCompareMode; + +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class AssertionUtils { + @SneakyThrows + public static void assertEqualJson(String expected, String actual) { + JSONAssert.assertEquals(expected, actual, JSONCompareMode.STRICT); + } + + public static void assertEqualUsingJson(Object expected, Object actual) { + assertEqualJson(JsonTestUtils.serialize(expected), JsonTestUtils.serialize(actual)); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerTestDb.java b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerTestDb.java new file mode 100644 index 000000000..7e7026048 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerTestDb.java @@ -0,0 +1,63 @@ +package de.sovity.edc.ext.catalog.crawler; + +import com.zaxxer.hikari.HikariDataSource; +import de.sovity.edc.ext.catalog.crawler.dao.config.DslContextFactory; +import de.sovity.edc.ext.catalog.crawler.dao.config.FlywayService; +import de.sovity.edc.extension.e2e.db.TestDatabaseViaTestcontainers; +import de.sovity.edc.extension.postgresql.FlywayUtils; +import de.sovity.edc.extension.postgresql.HikariDataSourceFactory; +import de.sovity.edc.extension.postgresql.JdbcCredentials; +import org.jooq.DSLContext; +import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.ExtensionContext; + +import java.util.function.Consumer; + +public class CrawlerTestDb implements BeforeAllCallback, AfterAllCallback { + private final TestDatabaseViaTestcontainers db = new TestDatabaseViaTestcontainers(); + + + private HikariDataSource dataSource = null; + private DslContextFactory dslContextFactory = null; + + public void testTransaction(Consumer code) { + dslContextFactory.testTransaction(code); + } + + @Override + public void beforeAll(ExtensionContext extensionContext) throws Exception { + // Init DB + db.beforeAll(extensionContext); + + // Init Data Source + var credentials = new JdbcCredentials( + db.getJdbcCredentials().jdbcUrl(), + db.getJdbcCredentials().jdbcUser(), + db.getJdbcCredentials().jdbcPassword() + ); + dataSource = HikariDataSourceFactory.newDataSource(credentials, 10, 1000); + dslContextFactory = new DslContextFactory(dataSource); + + // Migrate DB + var params = FlywayService.baseConfig("classpath:db-crawler/migration-test-utils") + .migrate(true) + .build(); + try { + FlywayUtils.cleanAndMigrate(params, dataSource); + } catch (Exception e) { + var paramsWithClean = params.withClean(true).withCleanEnabled(true); + FlywayUtils.cleanAndMigrate(paramsWithClean, dataSource); + } + } + + @Override + public void afterAll(ExtensionContext extensionContext) throws Exception { + if (dataSource != null) { + dataSource.close(); + } + + // Close DB + db.afterAll(extensionContext); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/JsonTestUtils.java b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/JsonTestUtils.java new file mode 100644 index 000000000..ec4f26b5e --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/JsonTestUtils.java @@ -0,0 +1,23 @@ +package de.sovity.edc.ext.catalog.crawler; + +import lombok.SneakyThrows; +import org.testcontainers.shaded.com.fasterxml.jackson.databind.ObjectMapper; + +public class JsonTestUtils { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + @SneakyThrows + public static String serialize(Object obj) { + return OBJECT_MAPPER.writeValueAsString(obj); + } + + @SneakyThrows + public static T deserialize(String json, Class clazz) { + return OBJECT_MAPPER.readValue(json, clazz); + } + + public static T jsonCast(Object obj, Class clazz) { + return deserialize(serialize(obj), clazz); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/TestData.java b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/TestData.java new file mode 100644 index 000000000..b522c8881 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/TestData.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler; + +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; +import de.sovity.edc.ext.catalog.crawler.db.jooq.Tables; +import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorContractOffersExceeded; +import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorDataOffersExceeded; +import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorOnlineStatus; +import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ConnectorRecord; +import lombok.experimental.UtilityClass; +import org.jooq.DSLContext; + +import java.time.OffsetDateTime; +import java.util.function.Consumer; + +@UtilityClass +public class TestData { + public static OffsetDateTime old = OffsetDateTime.now().withNano(0).withSecond(0).withMinute(0).withHour(0).minusDays(100); + + public static ConnectorRef connectorRef = new ConnectorRef( + "MDSL1234XX.C1234XX", + "test", + "My Org", + "MDSL1234XX", + "https://example.com/api/dsp" + ); + + public static void insertConnector( + DSLContext dsl, + ConnectorRef connectorRef, + Consumer applier + ) { + var organization = dsl.newRecord(Tables.ORGANIZATION); + organization.setMdsId(connectorRef.getOrganizationId()); + organization.setName(connectorRef.getOrganizationLegalName()); + organization.insert(); + + var connector = dsl.newRecord(Tables.CONNECTOR); + connector.setEnvironment(connectorRef.getEnvironmentId()); + connector.setMdsId(connectorRef.getOrganizationId()); + connector.setConnectorId(connectorRef.getConnectorId()); + connector.setName(connectorRef.getConnectorId() + " Name"); + connector.setEndpointUrl(connectorRef.getEndpoint()); + connector.setOnlineStatus(ConnectorOnlineStatus.OFFLINE); + connector.setLastRefreshAttemptAt(null); + connector.setLastSuccessfulRefreshAt(null); + connector.setCreatedAt(old); + connector.setDataOffersExceeded(ConnectorDataOffersExceeded.OK); + connector.setContractOffersExceeded(ConnectorContractOffersExceeded.OK); + applier.accept(connector); + connector.insert(); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventLoggerTest.java b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventLoggerTest.java new file mode 100644 index 000000000..f48ba78c8 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventLoggerTest.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.crawling.logging; + +import de.sovity.edc.ext.catalog.crawler.CrawlerTestDb; +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; +import de.sovity.edc.ext.catalog.crawler.db.jooq.Tables; +import org.jooq.DSLContext; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import static org.assertj.core.api.Assertions.assertThat; + +class CrawlerEventLoggerTest { + @RegisterExtension + private static final CrawlerTestDb TEST_DATABASE = new CrawlerTestDb(); + + @Test + void testDataOfferWriter_allSortsOfUpdates() { + TEST_DATABASE.testTransaction(dsl -> { + var crawlerEventLogger = new CrawlerEventLogger(); + + // Test that insertions insert required fields and don't cause DB errors + var connectorRef = new ConnectorRef( + "MDSL1234XX.C1234XX", + "test", + "My Org", + "MDSL1234XX", + "https://example.com/api/dsp" + ); + crawlerEventLogger.logConnectorUpdated(dsl, connectorRef, new ConnectorChangeTracker()); + crawlerEventLogger.logConnectorOnline(dsl, connectorRef); + crawlerEventLogger.logConnectorOffline(dsl, connectorRef, new CrawlerEventErrorMessage("Message", "Stacktrace")); + crawlerEventLogger.logConnectorUpdateContractOfferLimitExceeded(dsl, connectorRef, 10); + crawlerEventLogger.logConnectorUpdateContractOfferLimitOk(dsl, connectorRef); + crawlerEventLogger.logConnectorUpdateDataOfferLimitExceeded(dsl, connectorRef, 10); + crawlerEventLogger.logConnectorUpdateDataOfferLimitOk(dsl, connectorRef); + + assertThat(numLogEntries(dsl)).isEqualTo(7); + }); + } + + private Integer numLogEntries(DSLContext dsl) { + return dsl.selectCount().from(Tables.CRAWLER_EVENT_LOG).fetchOne().component1(); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateCatalogWriterTest.java b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateCatalogWriterTest.java new file mode 100644 index 000000000..249d36764 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateCatalogWriterTest.java @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.crawling.writing; + +import de.sovity.edc.ext.catalog.crawler.AssertionUtils; +import de.sovity.edc.ext.catalog.crawler.CrawlerTestDb; +import de.sovity.edc.ext.catalog.crawler.crawling.logging.ConnectorChangeTracker; +import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.DataOfferRecord; +import org.assertj.core.data.TemporalUnitLessThanOffset; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; +import java.util.List; + +import static de.sovity.edc.ext.catalog.crawler.crawling.writing.DataOfferWriterTestDataModels.Co; +import static de.sovity.edc.ext.catalog.crawler.crawling.writing.DataOfferWriterTestDataModels.Do; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +class ConnectorUpdateCatalogWriterTest { + @RegisterExtension + private static final CrawlerTestDb TEST_DATABASE = new CrawlerTestDb(); + + @Test + void testDataOfferWriter_allSortsOfUpdates() { + TEST_DATABASE.testTransaction(dsl -> { + var testDydi = new DataOfferWriterTestDydi(); + var testData = new DataOfferWriterTestDataHelper(); + var changes = new ConnectorChangeTracker(); + var dataOfferWriter = testDydi.getConnectorUpdateCatalogWriter(); + when(testDydi.getCrawlerConfig().getEnvironmentId()).thenReturn("test"); + + // arrange + var unchanged = Do.forName("unchanged"); + testData.existing(unchanged); + testData.fetched(unchanged); + + var fieldChangedExisting = Do.forName("fieldChanged"); + var fieldChangedFetched = fieldChangedExisting.withAssetTitle("changed"); + testData.existing(fieldChangedExisting); + testData.fetched(fieldChangedFetched); + + var added = Do.forName("added"); + testData.fetched(added); + + var removed = Do.forName("removed"); + testData.existing(removed); + + var changedCoExisting = Do.forName("contractOffer"); + var changedCoFetched = changedCoExisting.withContractOffers(List.of( + changedCoExisting.getContractOffers().get(0).withPolicyValue("changed") + )); + testData.existing(changedCoExisting); + testData.fetched(changedCoFetched); + + var addedCoExisting = Do.forName("contractOfferAdded"); + var addedCoFetched = addedCoExisting.withContractOffer(new Co("added co", "added co")); + testData.existing(addedCoExisting); + testData.fetched(addedCoFetched); + + var removedCoExisting = Do.forName("contractOfferRemoved") + .withContractOffer(new Co("removed co", "removed co")); + var removedCoFetched = Do.forName("contractOfferRemoved"); + testData.existing(removedCoExisting); + testData.fetched(removedCoFetched); + + // act + dsl.transaction(it -> testData.initialize(it.dsl())); + dsl.transaction(it -> dataOfferWriter.updateDataOffers( + it.dsl(), + testData.connectorRef, + testData.fetchedDataOffers, + changes + )); + var actual = dsl.transactionResult(it -> new DataOfferWriterTestResultHelper(it.dsl())); + + // assert + assertThat(actual.numDataOffers()).isEqualTo(6); + assertThat(changes.getNumOffersAdded()).isEqualTo(1); + assertThat(changes.getNumOffersUpdated()).isEqualTo(4); + assertThat(changes.getNumOffersDeleted()).isEqualTo(1); + + var now = OffsetDateTime.now(); + var minuteAccuracy = new TemporalUnitLessThanOffset(1, ChronoUnit.MINUTES); + var addedActual = actual.getDataOffer(added.getAssetId()); + assertAssetPropertiesEqual(testData, addedActual, added); + assertThat(addedActual.getCreatedAt()).isCloseTo(now, minuteAccuracy); + assertThat(addedActual.getUpdatedAt()).isCloseTo(now, minuteAccuracy); + assertThat(actual.numContractOffers(added.getAssetId())).isEqualTo(1); + assertPolicyEquals(actual, testData, added, added.getContractOffers().get(0)); + + var unchangedActual = actual.getDataOffer(unchanged.getAssetId()); + assertThat(unchangedActual.getUpdatedAt()).isEqualTo(testData.old); + assertThat(unchangedActual.getCreatedAt()).isEqualTo(testData.old); + + var fieldChangedActual = actual.getDataOffer(fieldChangedExisting.getAssetId()); + assertAssetPropertiesEqual(testData, fieldChangedActual, fieldChangedFetched); + assertThat(fieldChangedActual.getCreatedAt()).isEqualTo(testData.old); + assertThat(fieldChangedActual.getUpdatedAt()).isCloseTo(now, minuteAccuracy); + + var removedActual = actual.getDataOffer(removed.getAssetId()); + assertThat(removedActual).isNull(); + + var changedCoActual = actual.getDataOffer(changedCoExisting.getAssetId()); + assertThat(changedCoActual.getCreatedAt()).isEqualTo(testData.old); + assertThat(changedCoActual.getUpdatedAt()).isCloseTo(now, minuteAccuracy); + assertThat(actual.numContractOffers(changedCoExisting.getAssetId())).isEqualTo(1); + assertPolicyEquals(actual, testData, changedCoFetched, changedCoFetched.getContractOffers().get(0)); + + var addedCoActual = actual.getDataOffer(addedCoExisting.getAssetId()); + assertThat(addedCoActual.getCreatedAt()).isEqualTo(testData.old); + assertThat(addedCoActual.getUpdatedAt()).isCloseTo(now, minuteAccuracy); + assertThat(actual.numContractOffers(addedCoActual.getAssetId())).isEqualTo(2); + + var removedCoActual = actual.getDataOffer(removedCoExisting.getAssetId()); + assertThat(removedCoActual.getCreatedAt()).isEqualTo(testData.old); + assertThat(removedCoActual.getUpdatedAt()).isCloseTo(now, minuteAccuracy); + assertThat(actual.numContractOffers(removedCoActual.getAssetId())).isEqualTo(1); + }); + } + + private void assertAssetPropertiesEqual(DataOfferWriterTestDataHelper testData, DataOfferRecord actual, + Do expected) { + var actualUiAssetJson = actual.getUiAssetJson().data(); + var expectedUiAssetJson = testData.dummyAssetJson(expected); + AssertionUtils.assertEqualJson(actualUiAssetJson, expectedUiAssetJson); + } + + private void assertPolicyEquals( + DataOfferWriterTestResultHelper actual, + DataOfferWriterTestDataHelper scenario, + Do expectedDo, + Co expectedCo + ) { + var actualContractOffer = actual.getContractOffer(expectedDo.getAssetId(), expectedCo.getId()); + var actualUiPolicyJson = actualContractOffer.getUiPolicyJson().data(); + var expectedUiPolicyJson = scenario.dummyPolicyJson(expectedCo.getPolicyValue()); + assertThat(actualUiPolicyJson).isEqualTo(expectedUiPolicyJson); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferLimitsEnforcerTest.java b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferLimitsEnforcerTest.java new file mode 100644 index 000000000..cb0bafdb5 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferLimitsEnforcerTest.java @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.crawling.writing; + +import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedContractOffer; +import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedDataOffer; +import de.sovity.edc.ext.catalog.crawler.crawling.logging.CrawlerEventLogger; +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; +import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorContractOffersExceeded; +import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorDataOffersExceeded; +import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ConnectorRecord; +import de.sovity.edc.ext.catalog.crawler.orchestration.config.CrawlerConfig; +import org.assertj.core.api.Assertions; +import org.jooq.DSLContext; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class DataOfferLimitsEnforcerTest { + DataOfferLimitsEnforcer dataOfferLimitsEnforcer; + CrawlerConfig settings; + CrawlerEventLogger crawlerEventLogger; + DSLContext dsl; + + ConnectorRef connectorRef = new DataOfferWriterTestDataHelper().connectorRef; + + @BeforeEach + void setup() { + settings = mock(CrawlerConfig.class); + crawlerEventLogger = mock(CrawlerEventLogger.class); + dataOfferLimitsEnforcer = new DataOfferLimitsEnforcer(settings, crawlerEventLogger); + dsl = mock(DSLContext.class); + } + + @Test + void no_limit_and_two_dataofffers_and_contractoffer_should_not_limit() { + // arrange + int maxDataOffers = -1; + int maxContractOffers = -1; + when(settings.getMaxDataOffersPerConnector()).thenReturn(maxDataOffers); + when(settings.getMaxContractOffersPerDataOffer()).thenReturn(maxContractOffers); + + var myDataOffer = new FetchedDataOffer(); + myDataOffer.setContractOffers(List.of(new FetchedContractOffer(), new FetchedContractOffer())); + var dataOffers = List.of(myDataOffer, myDataOffer); + + // act + var enforcedLimits = dataOfferLimitsEnforcer.enforceLimits(dataOffers); + var actual = enforcedLimits.abbreviatedDataOffers(); + var contractOffersLimitExceeded = enforcedLimits.contractOfferLimitsExceeded(); + var dataOffersLimitExceeded = enforcedLimits.dataOfferLimitsExceeded(); + + // assert + Assertions.assertThat(actual).hasSize(2); + assertFalse(contractOffersLimitExceeded); + assertFalse(dataOffersLimitExceeded); + } + + @Test + void limit_zero_and_one_dataoffers_should_result_to_none() { + // arrange + int maxDataOffers = 0; + int maxContractOffers = 0; + when(settings.getMaxDataOffersPerConnector()).thenReturn(maxDataOffers); + when(settings.getMaxContractOffersPerDataOffer()).thenReturn(maxContractOffers); + + var dataOffers = List.of(new FetchedDataOffer()); + + // act + var enforcedLimits = dataOfferLimitsEnforcer.enforceLimits(dataOffers); + var actual = new ArrayList<>(enforcedLimits.abbreviatedDataOffers()); + var contractOffersLimitExceeded = enforcedLimits.contractOfferLimitsExceeded(); + var dataOffersLimitExceeded = enforcedLimits.dataOfferLimitsExceeded(); + + // assert + assertThat(actual).isEmpty(); + assertFalse(contractOffersLimitExceeded); + assertTrue(dataOffersLimitExceeded); + } + + @Test + void limit_one_and_two_dataoffers_should_result_to_one() { + // arrange + int maxDataOffers = 1; + int maxContractOffers = 1; + when(settings.getMaxDataOffersPerConnector()).thenReturn(maxDataOffers); + when(settings.getMaxContractOffersPerDataOffer()).thenReturn(maxContractOffers); + + var myDataOffer = new FetchedDataOffer(); + myDataOffer.setContractOffers(List.of(new FetchedContractOffer(), new FetchedContractOffer())); + var dataOffers = List.of(myDataOffer, myDataOffer); + + // act + var enforcedLimits = dataOfferLimitsEnforcer.enforceLimits(dataOffers); + var actual = new ArrayList<>(enforcedLimits.abbreviatedDataOffers()); + var contractOffersLimitExceeded = enforcedLimits.contractOfferLimitsExceeded(); + var dataOffersLimitExceeded = enforcedLimits.dataOfferLimitsExceeded(); + + // assert + assertThat(actual).hasSize(1); + Assertions.assertThat(actual.get(0).getContractOffers()).hasSize(1); + assertTrue(contractOffersLimitExceeded); + assertTrue(dataOffersLimitExceeded); + } + + @Test + void verify_logConnectorUpdateDataOfferLimitExceeded() { + // arrange + var connector = new ConnectorRecord(); + connector.setDataOffersExceeded(ConnectorDataOffersExceeded.OK); + + int maxDataOffers = 1; + int maxContractOffers = 1; + when(settings.getMaxDataOffersPerConnector()).thenReturn(maxDataOffers); + when(settings.getMaxContractOffersPerDataOffer()).thenReturn(maxContractOffers); + + var myDataOffer = new FetchedDataOffer(); + myDataOffer.setContractOffers(List.of(new FetchedContractOffer(), new FetchedContractOffer())); + var dataOffers = List.of(myDataOffer, myDataOffer); + + // act + var enforcedLimits = dataOfferLimitsEnforcer.enforceLimits(dataOffers); + dataOfferLimitsEnforcer.logEnforcedLimitsIfChanged(dsl, connectorRef, connector, enforcedLimits); + + // assert + verify(crawlerEventLogger).logConnectorUpdateDataOfferLimitExceeded(dsl, connectorRef, 1); + } + + @Test + void verify_logConnectorUpdateDataOfferLimitOk() { + // arrange + var connector = new ConnectorRecord(); + connector.setDataOffersExceeded(ConnectorDataOffersExceeded.EXCEEDED); + + int maxDataOffers = -1; + int maxContractOffers = -1; + when(settings.getMaxDataOffersPerConnector()).thenReturn(maxDataOffers); + when(settings.getMaxContractOffersPerDataOffer()).thenReturn(maxContractOffers); + + var myDataOffer = new FetchedDataOffer(); + myDataOffer.setContractOffers(List.of(new FetchedContractOffer(), new FetchedContractOffer())); + var dataOffers = List.of(myDataOffer, myDataOffer); + + // act + var enforcedLimits = dataOfferLimitsEnforcer.enforceLimits(dataOffers); + dataOfferLimitsEnforcer.logEnforcedLimitsIfChanged(dsl, connectorRef, connector, enforcedLimits); + + // assert + verify(crawlerEventLogger).logConnectorUpdateDataOfferLimitOk(dsl, connectorRef); + } + + @Test + void verify_logConnectorUpdateContractOfferLimitExceeded() { + // arrange + var connector = new ConnectorRecord(); + connector.setContractOffersExceeded(ConnectorContractOffersExceeded.OK); + + int maxDataOffers = 1; + int maxContractOffers = 1; + when(settings.getMaxDataOffersPerConnector()).thenReturn(maxDataOffers); + when(settings.getMaxContractOffersPerDataOffer()).thenReturn(maxContractOffers); + + var myDataOffer = new FetchedDataOffer(); + myDataOffer.setContractOffers(List.of(new FetchedContractOffer(), new FetchedContractOffer())); + var dataOffers = List.of(myDataOffer, myDataOffer); + + // act + var enforcedLimits = dataOfferLimitsEnforcer.enforceLimits(dataOffers); + dataOfferLimitsEnforcer.logEnforcedLimitsIfChanged(dsl, connectorRef, connector, enforcedLimits); + + // assert + verify(crawlerEventLogger).logConnectorUpdateContractOfferLimitExceeded(dsl, connectorRef, 1); + } + + @Test + void verify_logConnectorUpdateContractOfferLimitOk() { + // arrange + var connector = new ConnectorRecord(); + connector.setContractOffersExceeded(ConnectorContractOffersExceeded.EXCEEDED); + + int maxDataOffers = -1; + int maxContractOffers = -1; + when(settings.getMaxDataOffersPerConnector()).thenReturn(maxDataOffers); + when(settings.getMaxContractOffersPerDataOffer()).thenReturn(maxContractOffers); + + var myDataOffer = new FetchedDataOffer(); + myDataOffer.setContractOffers(List.of(new FetchedContractOffer(), new FetchedContractOffer())); + var dataOffers = List.of(myDataOffer, myDataOffer); + + // act + var enforcedLimits = dataOfferLimitsEnforcer.enforceLimits(dataOffers); + dataOfferLimitsEnforcer.logEnforcedLimitsIfChanged(dsl, connectorRef, connector, enforcedLimits); + + // assert + verify(crawlerEventLogger).logConnectorUpdateContractOfferLimitOk(dsl, connectorRef); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDataHelper.java b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDataHelper.java new file mode 100644 index 000000000..997141ab5 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDataHelper.java @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.crawling.writing; + +import de.sovity.edc.ext.catalog.crawler.TestData; +import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedContractOffer; +import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedDataOffer; +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; +import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ContractOfferRecord; +import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.DataOfferRecord; +import de.sovity.edc.ext.wrapper.api.common.model.UiAsset; +import de.sovity.edc.utils.JsonUtils; +import jakarta.json.Json; +import org.apache.commons.lang3.Validate; +import org.jetbrains.annotations.NotNull; +import org.jooq.DSLContext; +import org.jooq.JSONB; + +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +class DataOfferWriterTestDataHelper { + OffsetDateTime old = TestData.old; + ConnectorRef connectorRef = TestData.connectorRef; + List existingContractOffers = new ArrayList<>(); + List existingDataOffers = new ArrayList<>(); + List fetchedDataOffers = new ArrayList<>(); + + + /** + * Adds fetched data offer + * + * @param dataOffer fetched data offer + */ + public void fetched(DataOfferWriterTestDataModels.Do dataOffer) { + Validate.notEmpty(dataOffer.getContractOffers()); + fetchedDataOffers.add(dummyFetchedDataOffer(dataOffer)); + } + + + /** + * Adds data offer directly to DB. + * + * @param dataOffer data offer + */ + public void existing(DataOfferWriterTestDataModels.Do dataOffer) { + Validate.notEmpty(dataOffer.getContractOffers()); + existingDataOffers.add(dummyDataOffer(dataOffer)); + dataOffer.getContractOffers().stream() + .map(contractOffer -> dummyContractOffer(dataOffer, contractOffer)) + .forEach(existingContractOffers::add); + } + + public void initialize(DSLContext dsl) { + TestData.insertConnector(dsl, connectorRef, record -> { + }); + dsl.batchInsert(existingDataOffers).execute(); + dsl.batchInsert(existingContractOffers).execute(); + } + + private ContractOfferRecord dummyContractOffer( + DataOfferWriterTestDataModels.Do dataOffer, + DataOfferWriterTestDataModels.Co contractOffer + ) { + var contractOfferRecord = new ContractOfferRecord(); + contractOfferRecord.setConnectorId(connectorRef.getConnectorId()); + contractOfferRecord.setAssetId(dataOffer.getAssetId()); + contractOfferRecord.setContractOfferId(contractOffer.getId()); + contractOfferRecord.setUiPolicyJson(JSONB.valueOf(dummyPolicyJson(contractOffer.getPolicyValue()))); + contractOfferRecord.setCreatedAt(old); + contractOfferRecord.setUpdatedAt(old); + return contractOfferRecord; + } + + private DataOfferRecord dummyDataOffer(DataOfferWriterTestDataModels.Do dataOffer) { + var assetName = Optional.of(dataOffer.getAssetTitle()).orElse(dataOffer.getAssetId()); + + var dataOfferRecord = new DataOfferRecord(); + dataOfferRecord.setConnectorId(connectorRef.getConnectorId()); + dataOfferRecord.setAssetId(dataOffer.getAssetId()); + dataOfferRecord.setAssetTitle(assetName); + dataOfferRecord.setUiAssetJson(JSONB.valueOf(dummyAssetJson(dataOffer))); + dataOfferRecord.setCreatedAt(old); + dataOfferRecord.setUpdatedAt(old); + return dataOfferRecord; + } + + private FetchedDataOffer dummyFetchedDataOffer(DataOfferWriterTestDataModels.Do dataOffer) { + var fetchedDataOffer = new FetchedDataOffer(); + fetchedDataOffer.setAssetId(dataOffer.getAssetId()); + fetchedDataOffer.setUiAsset( + UiAsset.builder() + .assetId(dataOffer.getAssetId()) + .title(dataOffer.getAssetTitle()) + .build() + ); + fetchedDataOffer.setUiAssetJson(dummyAssetJson(dataOffer)); + + var contractOffersMapped = dataOffer.getContractOffers().stream().map(this::dummyFetchedContractOffer).collect(Collectors.toList()); + fetchedDataOffer.setContractOffers(contractOffersMapped); + + return fetchedDataOffer; + } + + public String dummyAssetJson(DataOfferWriterTestDataModels.Do dataOffer) { + var dummyUiAssetJson = Json.createObjectBuilder() + .add("assetId", dataOffer.getAssetId()) + .add("title", dataOffer.getAssetTitle()) + .add("assetJsonLd", "{}") + .build(); + return JsonUtils.toJson(dummyUiAssetJson); + } + + public String dummyPolicyJson(String policyValue) { + return "{\"%s\": \"%s\"}".formatted( + "SomePolicyField", policyValue + ); + } + + @NotNull + private FetchedContractOffer dummyFetchedContractOffer(DataOfferWriterTestDataModels.Co it) { + var contractOffer = new FetchedContractOffer(); + contractOffer.setContractOfferId(it.getId()); + contractOffer.setUiPolicyJson(dummyPolicyJson(it.getPolicyValue())); + return contractOffer; + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDataModels.java b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDataModels.java new file mode 100644 index 000000000..7dd824d99 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDataModels.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.crawling.writing; + +import lombok.Value; +import lombok.With; + +import java.util.ArrayList; +import java.util.List; + +class DataOfferWriterTestDataModels { + /** + * Dummy Data Offer + */ + @Value + static class Do { + @With + String assetId; + @With + String assetTitle; + @With + List contractOffers; + + public Do withContractOffer(Co co) { + var list = new ArrayList<>(contractOffers); + list.add(co); + return this.withContractOffers(list); + } + + public static Do forName(String name) { + return new Do(name, name + " Title", List.of(new Co(name + " CO", name + " Policy"))); + } + } + + /** + * Dummy Contract Offer + */ + @Value + static class Co { + @With + String id; + @With + String policyValue; + } + + public static Co forName(String name) { + return new Co(name + " CO", name + " Policy"); + } + +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDydi.java b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDydi.java new file mode 100644 index 000000000..fb39bcbec --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDydi.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.crawling.writing; + +import de.sovity.edc.ext.catalog.crawler.dao.CatalogPatchApplier; +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorQueries; +import de.sovity.edc.ext.catalog.crawler.dao.contract_offers.ContractOfferQueries; +import de.sovity.edc.ext.catalog.crawler.dao.contract_offers.ContractOfferRecordUpdater; +import de.sovity.edc.ext.catalog.crawler.dao.data_offers.DataOfferQueries; +import de.sovity.edc.ext.catalog.crawler.dao.data_offers.DataOfferRecordUpdater; +import de.sovity.edc.ext.catalog.crawler.orchestration.config.CrawlerConfig; +import lombok.Value; +import org.eclipse.edc.spi.system.configuration.Config; + +import static org.mockito.Mockito.mock; + +@Value +class DataOfferWriterTestDydi { + Config config = mock(Config.class); + CrawlerConfig crawlerConfig = mock(CrawlerConfig.class); + DataOfferQueries dataOfferQueries = new DataOfferQueries(); + ContractOfferQueries contractOfferQueries = new ContractOfferQueries(); + ContractOfferRecordUpdater contractOfferRecordUpdater = new ContractOfferRecordUpdater(); + ConnectorQueries connectorQueries = new ConnectorQueries(crawlerConfig); + DataOfferRecordUpdater dataOfferRecordUpdater = new DataOfferRecordUpdater( + connectorQueries + ); + CatalogPatchBuilder catalogPatchBuilder = new CatalogPatchBuilder( + contractOfferQueries, + dataOfferQueries, + dataOfferRecordUpdater, + contractOfferRecordUpdater + ); + CatalogPatchApplier catalogPatchApplier = new CatalogPatchApplier(); + ConnectorUpdateCatalogWriter connectorUpdateCatalogWriter = new ConnectorUpdateCatalogWriter(catalogPatchBuilder, catalogPatchApplier); +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestResultHelper.java b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestResultHelper.java new file mode 100644 index 000000000..e5dfa39bb --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestResultHelper.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.crawling.writing; + +import de.sovity.edc.ext.catalog.crawler.db.jooq.Tables; +import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ContractOfferRecord; +import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.DataOfferRecord; +import org.jetbrains.annotations.NotNull; +import org.jooq.DSLContext; + +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static java.util.stream.Collectors.groupingBy; + +class DataOfferWriterTestResultHelper { + private final @NotNull Map dataOffers; + private final @NotNull Map> contractOffers; + + DataOfferWriterTestResultHelper(DSLContext dsl) { + this.dataOffers = dsl.selectFrom(Tables.DATA_OFFER).fetchMap(Tables.DATA_OFFER.ASSET_ID); + this.contractOffers = dsl.selectFrom(Tables.CONTRACT_OFFER).stream().collect(groupingBy( + ContractOfferRecord::getAssetId, + Collectors.toMap(ContractOfferRecord::getContractOfferId, Function.identity()) + )); + } + + public DataOfferRecord getDataOffer(String assetId) { + return dataOffers.get(assetId); + } + + public int numDataOffers() { + return dataOffers.size(); + } + + public int numContractOffers(String assetId) { + return contractOffers.getOrDefault(assetId, Map.of()).size(); + } + + public ContractOfferRecord getContractOffer(String assetId, String contractOfferId) { + return contractOffers.getOrDefault(assetId, Map.of()).get(contractOfferId); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DiffUtilsTest.java b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DiffUtilsTest.java new file mode 100644 index 000000000..79933dc2a --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DiffUtilsTest.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.crawling.writing; + +import de.sovity.edc.ext.catalog.crawler.crawling.writing.utils.DiffUtils; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.function.Function; + +import static org.assertj.core.api.Assertions.assertThat; + +class DiffUtilsTest { + + @Test + void testCompareLists() { + // arrange + List existing = List.of(1, 2); + List fetched = List.of("1", "3"); + + // act + var actual = DiffUtils.compareLists(existing, Function.identity(), fetched, Integer::parseInt); + + // assert + assertThat(actual.added()).containsExactly("3"); + assertThat(actual.updated()).containsExactly(new DiffUtils.DiffResultMatch<>(1, "1")); + assertThat(actual.removed()).containsExactly(2); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorRefTest.java b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorRefTest.java new file mode 100644 index 000000000..6593a5397 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorRefTest.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.dao.connectors; + + +import org.junit.jupiter.api.Test; + +import java.util.HashSet; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + + +class ConnectorRefTest { + + @Test + void testEqualsTrue() { + // arrange + var a = new ConnectorRef("a", "1", "1", "1", "1"); + var b = new ConnectorRef("a", "2", "2", "2", "2"); + + // act + var result = a.equals(b); + + // assert + assertThat(result).isTrue(); + } + + @Test + void testEqualsFalse() { + // arrange + var a = new ConnectorRef("a", "1", "1", "1", "1"); + var b = new ConnectorRef("b", "1", "1", "1", "1"); + + // act + var result = a.equals(b); + + // assert + assertThat(result).isFalse(); + } + + @Test + void testSet() { + // arrange + var a = new ConnectorRef("a", "1", "1", "1", "1"); + var a2 = new ConnectorRef("a", "2", "2", "2", "2"); + var b = new ConnectorRef("b", "1", "1", "1", "1"); + + // act + var result = new HashSet<>(List.of(a, a2, b)).stream().map(ConnectorRef::getConnectorId).toList(); + + // assert + assertThat(result).containsExactlyInAnyOrder("a", "b"); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPoolQueueTest.java b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPoolQueueTest.java new file mode 100644 index 000000000..e6b1ad053 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPoolQueueTest.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.orchestration.queue; + +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; + +import static org.mockito.Mockito.mock; + +class ThreadPoolQueueTest { + + + /** + * Regression against bug where the queue did not act like a queue. + */ + @Test + void testOrdering() { + Runnable noop = () -> { + }; + + var c10 = mock(ConnectorRef.class); + var c20 = mock(ConnectorRef.class); + var c11 = mock(ConnectorRef.class); + var c21 = mock(ConnectorRef.class); + var c00 = mock(ConnectorRef.class); + + var queue = new ThreadPoolTaskQueue(); + queue.add(new ThreadPoolTask(1, noop, c10)); + queue.add(new ThreadPoolTask(2, noop, c20)); + queue.add(new ThreadPoolTask(1, noop, c11)); + queue.add(new ThreadPoolTask(2, noop, c21)); + queue.add(new ThreadPoolTask(0, noop, c00)); + + var result = new ArrayList(); + queue.getQueue().drainTo(result); + + Assertions.assertThat(result).extracting(ThreadPoolTask::getConnectorRef) + .containsExactly(c00, c10, c11, c20, c21); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorRemovalJobTest.java b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorRemovalJobTest.java new file mode 100644 index 000000000..f054ff5b8 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorRemovalJobTest.java @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.orchestration.schedules; + +import de.sovity.edc.ext.catalog.crawler.CrawlerTestDb; +import de.sovity.edc.ext.catalog.crawler.TestData; +import de.sovity.edc.ext.catalog.crawler.crawling.OfflineConnectorCleaner; +import de.sovity.edc.ext.catalog.crawler.crawling.logging.CrawlerEventLogger; +import de.sovity.edc.ext.catalog.crawler.dao.CatalogCleaner; +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorQueries; +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; +import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorStatusUpdater; +import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorOnlineStatus; +import de.sovity.edc.ext.catalog.crawler.orchestration.config.CrawlerConfig; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.time.Duration; +import java.time.OffsetDateTime; + +import static de.sovity.edc.ext.catalog.crawler.db.jooq.tables.Connector.CONNECTOR; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class OfflineConnectorRemovalJobTest { + @RegisterExtension + private static final CrawlerTestDb TEST_DATABASE = new CrawlerTestDb(); + + ConnectorRef connectorRef = TestData.connectorRef; + + CrawlerConfig crawlerConfig; + OfflineConnectorCleaner offlineConnectorCleaner; + ConnectorQueries connectorQueries; + + @BeforeEach + void beforeEach() { + crawlerConfig = mock(CrawlerConfig.class); + connectorQueries = new ConnectorQueries(crawlerConfig); + offlineConnectorCleaner = new OfflineConnectorCleaner( + crawlerConfig, + new ConnectorQueries(crawlerConfig), + new CrawlerEventLogger(), + new ConnectorStatusUpdater(), + new CatalogCleaner() + ); + when(crawlerConfig.getEnvironmentId()).thenReturn(connectorRef.getEnvironmentId()); + } + + @Test + void test_offlineConnectorCleaner_should_be_dead() { + TEST_DATABASE.testTransaction(dsl -> { + // arrange + when(crawlerConfig.getKillOfflineConnectorsAfter()).thenReturn(Duration.ofDays(5)); + TestData.insertConnector(dsl, connectorRef, record -> { + record.setOnlineStatus(ConnectorOnlineStatus.OFFLINE); + record.setLastSuccessfulRefreshAt(OffsetDateTime.now().minusDays(6)); + }); + + // act + offlineConnectorCleaner.cleanConnectorsIfOfflineTooLong(dsl); + + // assert + var connector = dsl.fetchOne(CONNECTOR, CONNECTOR.CONNECTOR_ID.eq(connectorRef.getConnectorId())); + assertThat(connector.getOnlineStatus()).isEqualTo(ConnectorOnlineStatus.DEAD); + }); + } + + @Test + void test_offlineConnectorCleaner_should_not_be_dead() { + TEST_DATABASE.testTransaction(dsl -> { + // arrange + when(crawlerConfig.getKillOfflineConnectorsAfter()).thenReturn(Duration.ofDays(5)); + TestData.insertConnector(dsl, connectorRef, record -> { + record.setOnlineStatus(ConnectorOnlineStatus.OFFLINE); + record.setLastSuccessfulRefreshAt(OffsetDateTime.now().minusDays(2)); + }); + + // act + offlineConnectorCleaner.cleanConnectorsIfOfflineTooLong(dsl); + + // assert + var connector = dsl.fetchOne(CONNECTOR, CONNECTOR.CONNECTOR_ID.eq(connectorRef.getConnectorId())); + assertThat(connector.getOnlineStatus()).isEqualTo(ConnectorOnlineStatus.OFFLINE); + }); + } + +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/CollectionUtils2Test.java b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/CollectionUtils2Test.java new file mode 100644 index 000000000..87673188f --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/CollectionUtils2Test.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.utils; + +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + + +class CollectionUtils2Test { + @Test + void difference() { + assertThat(CollectionUtils2.difference(List.of(1, 2, 3), List.of(2, 3, 4))).containsExactly(1); + } + + @Test + void isNotEmpty_withEmptyList() { + assertThat(CollectionUtils2.isNotEmpty(List.of())).isFalse(); + } + + @Test + void isNotEmpty_withNull() { + assertThat(CollectionUtils2.isNotEmpty(null)).isFalse(); + } + + @Test + void isNotEmpty_withNonEmptyList() { + assertThat(CollectionUtils2.isNotEmpty(List.of(1))).isTrue(); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/JsonUtils2Test.java b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/JsonUtils2Test.java new file mode 100644 index 000000000..8843aebbb --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/JsonUtils2Test.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.utils; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class JsonUtils2Test { + @Test + void equalityTests() { + assertTrue(JsonUtils2.isEqualJson(null, null)); + assertTrue(JsonUtils2.isEqualJson("null", "null")); + assertTrue(JsonUtils2.isEqualJson("{}", "{}")); + assertTrue(JsonUtils2.isEqualJson("{\"a\": true, \"b\": \"hello\"}", "{\"a\": true,\"b\": \"hello\"}")); + assertTrue(JsonUtils2.isEqualJson("{\"a\": true, \"b\": \"hello\"}", "{\"b\": \"hello\", \"a\": true}")); + + assertFalse(JsonUtils2.isEqualJson(null, "1")); + assertFalse(JsonUtils2.isEqualJson("1", null)); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/StringUtils2Test.java b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/StringUtils2Test.java new file mode 100644 index 000000000..1cbd56dd5 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/StringUtils2Test.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.utils; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class StringUtils2Test { + @Test + void removeSuffix_emptyStrings() { + assertThat(StringUtils2.removeSuffix("", "")).isEmpty(); + } + + @Test + void removeSuffix_emptySuffix() { + assertThat(StringUtils2.removeSuffix("test", "")).isEqualTo("test"); + } + + + @Test + void removeSuffix_withSuffix() { + assertThat(StringUtils2.removeSuffix("testabc", "abc")).isEqualTo("test"); + } + + + @Test + void removeSuffix_withoutSuffix() { + assertThat(StringUtils2.removeSuffix("test", "abc")).isEqualTo("test"); + } + +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/resources/logging.properties b/extensions/catalog-crawler/catalog-crawler/src/test/resources/logging.properties new file mode 100644 index 000000000..471bd20d6 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/test/resources/logging.properties @@ -0,0 +1,6 @@ +.level=ALL +org.eclipse.edc.level=ALL +handlers=java.util.logging.ConsoleHandler +java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter +java.util.logging.ConsoleHandler.level=ALL +java.util.logging.SimpleFormatter.format=[%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS] [%4$-7s] %5$s%6$s%n diff --git a/launchers/.env.catalog-crawler b/launchers/.env.catalog-crawler new file mode 100644 index 000000000..77aa13f53 --- /dev/null +++ b/launchers/.env.catalog-crawler @@ -0,0 +1,88 @@ +# Default ENV Vars + +# This file will be sourced as bash script: +# - KEY=Value will become KEY=${KEY:-"Value"}, so that ENV Vars can be overwritten by parent docker-compose.yaml. +# - Watch out for escaping issues as values will be surrounded by quotes, and dollar signs must be escaped. + +# =========================================================== +# Available Catalog Crawler Config +# =========================================================== + +# Environment ID +CRAWLER_ENVIRONMENT_ID=missing-env-CRAWLER_ENVIRONMENT_ID + +# Fully Qualified Domain Name (e.g. example.com) +MY_EDC_FQDN=missing-env-MY_EDC_FQDN + +# Postgres Database Connection +CRAWLER_DB_JDBC_URL=jdbc:postgresql://missing-postgresql-url +CRAWLER_DB_JDBC_USER=missing-postgresql-user +CRAWLER_DB_JDBC_PASSWORD=missing-postgresql-password + +# Database Connection Pool Size +CRAWLER_DB_CONNECTION_POOL_SIZE=30 + +# Database Connection Timeout (in ms) +CRAWLER_DB_CONNECTION_TIMEOUT_IN_MS=30000 + +# CRON interval for crawling ONLINE connectors +CRAWLER_CRON_ONLINE_CONNECTOR_REFRESH=*/20 * * ? * * + +# CRON interval for crawling OFFLINE connectors +CRAWLER_CRON_OFFLINE_CONNECTOR_REFRESH=0 */5 * ? * * + +# CRON interval for crawling DEAD connectors +CRAWLER_CRON_DEAD_CONNECTOR_REFRESH=0 0 * ? * * + +# CRON interval for marking connectors as DEAD +CRAWLER_SCHEDULED_KILL_OFFLINE_CONNECTORS=0 0 2 ? * * + +# Delete data offers / mark as dead after connector has been offline for: +CRAWLER_KILL_OFFLINE_CONNECTORS_AFTER=P5D + +# Hide data offers after connector has been offline for: +CRAWLER_HIDE_OFFLINE_DATA_OFFERS_AFTER=P1D + +# Parallelization for Crawling +CRAWLER_NUM_THREADS=32 + +# Maximum number of Data Offers per Connector +CRAWLER_MAX_DATA_OFFERS_PER_CONNECTOR=50 + +# Maximum number of Contract Offers per Data Offer +CRAWLER_MAX_CONTRACT_OFFERS_PER_DATA_OFFER=10 + +# Enable the extension +CRAWLER_EXTENSION_ENABLED=true + +# =========================================================== +# Other EDC Config +# =========================================================== + +# Ports and Paths +MY_EDC_PARTICIPANT_ID=broker +EDC_CONNECTOR_NAME=${MY_EDC_PARTICIPANT_ID:-MY_EDC_NAME_KEBAB_CASE} +EDC_PARTICIPANT_ID=${MY_EDC_PARTICIPANT_ID:-MY_EDC_NAME_KEBAB_CASE} +MY_EDC_BASE_PATH= +MY_EDC_PROTOCOL=https:// +WEB_HTTP_PORT=11001 +WEB_HTTP_MANAGEMENT_PORT=11002 +WEB_HTTP_PROTOCOL_PORT=11003 +WEB_HTTP_CONTROL_PORT=11004 +WEB_HTTP_PATH=${MY_EDC_BASE_PATH}/api +WEB_HTTP_MANAGEMENT_PATH=${MY_EDC_BASE_PATH}/api/management +WEB_HTTP_PROTOCOL_PATH=${MY_EDC_BASE_PATH}/api/dsp +WEB_HTTP_CONTROL_PATH=${MY_EDC_BASE_PATH}/api/control + +EDC_HOSTNAME=${MY_EDC_FQDN} +EDC_DSP_CALLBACK_ADDRESS=${MY_EDC_PROTOCOL}${MY_EDC_FQDN}${WEB_HTTP_PROTOCOL_PATH} + +# Oauth default configurations for compatibility with sovity DAPS +EDC_OAUTH_PROVIDER_AUDIENCE=${EDC_OAUTH_TOKEN_URL} +EDC_OAUTH_ENDPOINT_AUDIENCE=idsc:IDS_CONNECTORS_ALL +EDC_AGENT_IDENTITY_KEY=referringConnector + +# This file could contain an entry replacing the EDC_KEYSTORE ENV var, +# but for some reason it is required, and EDC won't start up if it isn't configured. +# It will be created in the Dockerfile +EDC_VAULT=/app/empty-properties-file.properties diff --git a/launchers/README.md b/launchers/README.md index 9a1f60f3d..dd9d43c90 100644 --- a/launchers/README.md +++ b/launchers/README.md @@ -100,7 +100,6 @@ Our sovity Community Edition EDC is built as several docker image variants in di

  • Management API Auth via API Keys
  • PostgreSQL Persistence & Flyway
  • DAPS Authentication
    • -
  • Broker Extension
  • Clearing House Extension
    • @@ -134,27 +133,24 @@ Our sovity Community Edition EDC is built as several docker image variants in di -     broker-dev + catalog-crawler-dev Development
      -
    • Local Demo via our - docker-compose.yaml -
      • +
    • Local Demo
    • E2E Testing
      -
    • Broker Server Extension(s)
      • -
    • PostgreSQL Persistence & Flyway
      • +
    • Catalog Crawler for one environment
    • Mock IAM
    - broker-ce + catalog-crawler-ce Community Edition
      @@ -163,8 +159,7 @@ Our sovity Community Edition EDC is built as several docker image variants in di
        -
      • Broker Server Extension(s)
        • -
      • PostgreSQL Persistence & Flyway
        • +
      • Catalog Crawler for one environment
      • DAPS Authentication
      diff --git a/launchers/connectors/catalog-crawler-ce/build.gradle.kts b/launchers/connectors/catalog-crawler-ce/build.gradle.kts new file mode 100644 index 000000000..e72aa1f52 --- /dev/null +++ b/launchers/connectors/catalog-crawler-ce/build.gradle.kts @@ -0,0 +1,23 @@ +plugins { + `java-library` + id("application") + alias(libs.plugins.shadow) +} + +dependencies { + implementation(project(":extensions:catalog-crawler:catalog-crawler-launcher-base")) + + api(libs.edc.monitorJdkLogger) + api(libs.edc.apiObservability) + + implementation(project(":launchers:common:auth-daps")) +} + +application { + mainClass.set("org.eclipse.edc.boot.system.runtime.BaseRuntime") +} + +tasks.withType { + mergeServiceFiles() + archiveFileName.set("app.jar") +} diff --git a/launchers/connectors/catalog-crawler-dev/build.gradle.kts b/launchers/connectors/catalog-crawler-dev/build.gradle.kts new file mode 100644 index 000000000..c68026e6c --- /dev/null +++ b/launchers/connectors/catalog-crawler-dev/build.gradle.kts @@ -0,0 +1,23 @@ +plugins { + `java-library` + id("application") + alias(libs.plugins.shadow) +} + +dependencies { + implementation(project(":extensions:catalog-crawler:catalog-crawler-launcher-base")) + + api(libs.edc.monitorJdkLogger) + api(libs.edc.apiObservability) + + implementation(project(":launchers:common:auth-mock")) +} + +application { + mainClass.set("org.eclipse.edc.boot.system.runtime.BaseRuntime") +} + +tasks.withType { + mergeServiceFiles() + archiveFileName.set("app.jar") +} From 14019cc40cda163527837bf7a812dae79db9f8a4 Mon Sep 17 00:00:00 2001 From: Richard Treier Date: Wed, 10 Jul 2024 12:45:12 +0300 Subject: [PATCH 09/14] fix: crawler not handling descriptions properly (#993) --- .../CrawlerExtensionContextBuilder.java | 3 +- .../fetching/model/FetchedCatalog.java | 6 + .../fetching/model/FetchedContractOffer.java | 6 + .../fetching/model/FetchedDataOffer.java | 6 + .../data_offers/DataOfferRecordUpdater.java | 17 +- .../ext/catalog/crawler/CrawlerTestDb.java | 1 - .../writing/ConnectorSuccessWriterTest.java | 146 ++++++++++++++++++ .../writing/DataOfferWriterTestDydi.java | 19 ++- 8 files changed, 193 insertions(+), 11 deletions(-) create mode 100644 extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorSuccessWriterTest.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java index cb49d40a5..2437dd751 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java @@ -129,7 +129,8 @@ public static CrawlerExtensionContext buildContext( objectMapperJsonLd ); var contractOfferRecordUpdater = new ContractOfferRecordUpdater(); - var dataOfferRecordUpdater = new DataOfferRecordUpdater(connectorQueries); + var shortDescriptionBuilder = new ShortDescriptionBuilder(); + var dataOfferRecordUpdater = new DataOfferRecordUpdater(shortDescriptionBuilder); var contractOfferQueries = new ContractOfferQueries(); var dataOfferLimitsEnforcer = new DataOfferLimitsEnforcer(crawlerConfig, crawlerEventLogger); var dataOfferPatchBuilder = new CatalogPatchBuilder( diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedCatalog.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedCatalog.java index d930eb223..027d2c8b3 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedCatalog.java +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedCatalog.java @@ -16,7 +16,10 @@ import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.Setter; import lombok.experimental.FieldDefaults; @@ -27,6 +30,9 @@ */ @Getter @Setter +@Builder +@RequiredArgsConstructor +@AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) public class FetchedCatalog { ConnectorRef connectorRef; diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedContractOffer.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedContractOffer.java index 64317498e..2b1993749 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedContractOffer.java +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedContractOffer.java @@ -15,12 +15,18 @@ package de.sovity.edc.ext.catalog.crawler.crawling.fetching.model; import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.Setter; import lombok.experimental.FieldDefaults; @Getter @Setter +@Builder +@NoArgsConstructor +@AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) public class FetchedContractOffer { String contractOfferId; diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedDataOffer.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedDataOffer.java index 752fc98fd..a2e6fe99b 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedDataOffer.java +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedDataOffer.java @@ -16,7 +16,10 @@ import de.sovity.edc.ext.wrapper.api.common.model.UiAsset; import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.Setter; import lombok.experimental.FieldDefaults; @@ -27,6 +30,9 @@ */ @Getter @Setter +@Builder +@NoArgsConstructor +@AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) public class FetchedDataOffer { String assetId; diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/data_offers/DataOfferRecordUpdater.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/data_offers/DataOfferRecordUpdater.java index 955c05271..c09a3d0b8 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/data_offers/DataOfferRecordUpdater.java +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/data_offers/DataOfferRecordUpdater.java @@ -16,11 +16,11 @@ import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedDataOffer; import de.sovity.edc.ext.catalog.crawler.crawling.writing.utils.ChangeTracker; -import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorQueries; import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; import de.sovity.edc.ext.catalog.crawler.dao.utils.JsonbUtils; import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.DataOfferRecord; import de.sovity.edc.ext.catalog.crawler.utils.JsonUtils2; +import de.sovity.edc.ext.wrapper.api.common.mappers.asset.utils.ShortDescriptionBuilder; import lombok.RequiredArgsConstructor; import org.jooq.JSONB; @@ -36,8 +36,7 @@ */ @RequiredArgsConstructor public class DataOfferRecordUpdater { - - private final ConnectorQueries connectorQueries; + private final ShortDescriptionBuilder shortDescriptionBuilder; /** * Create a new {@link DataOfferRecord}. @@ -84,9 +83,15 @@ public boolean updateDataOffer( ); changes.setIfChanged( - blankIfNull(record.getDescription()), - blankIfNull(asset.getDescription()), - record::setDescription + blankIfNull(record.getDescriptionNoMarkdown()), + shortDescriptionBuilder.extractMarkdownText(blankIfNull(asset.getDescription())), + record::setDescriptionNoMarkdown + ); + + changes.setIfChanged( + blankIfNull(record.getShortDescriptionNoMarkdown()), + blankIfNull(asset.getDescriptionShortText()), + record::setShortDescriptionNoMarkdown ); changes.setIfChanged( diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerTestDb.java b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerTestDb.java index 7e7026048..a4057c57b 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerTestDb.java +++ b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerTestDb.java @@ -17,7 +17,6 @@ public class CrawlerTestDb implements BeforeAllCallback, AfterAllCallback { private final TestDatabaseViaTestcontainers db = new TestDatabaseViaTestcontainers(); - private HikariDataSource dataSource = null; private DslContextFactory dslContextFactory = null; diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorSuccessWriterTest.java b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorSuccessWriterTest.java new file mode 100644 index 000000000..4e7d6a603 --- /dev/null +++ b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorSuccessWriterTest.java @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler.crawling.writing; + +import de.sovity.edc.ext.catalog.crawler.CrawlerTestDb; +import de.sovity.edc.ext.catalog.crawler.TestData; +import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedCatalog; +import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedContractOffer; +import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedDataOffer; +import de.sovity.edc.ext.catalog.crawler.db.jooq.Tables; +import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorContractOffersExceeded; +import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorDataOffersExceeded; +import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorOnlineStatus; +import de.sovity.edc.ext.wrapper.api.common.model.UiAsset; +import org.assertj.core.data.TemporalUnitLessThanOffset; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +class ConnectorSuccessWriterTest { + @RegisterExtension + private static final CrawlerTestDb TEST_DATABASE = new CrawlerTestDb(); + + ConnectorUpdateSuccessWriter connectorUpdateSuccessWriter; + + @BeforeEach + void setup() { + var container = new DataOfferWriterTestDydi(); + connectorUpdateSuccessWriter = container.getConnectorUpdateSuccessWriter(); + when(container.getCrawlerConfig().getMaxContractOffersPerDataOffer()).thenReturn(1); + when(container.getCrawlerConfig().getMaxDataOffersPerConnector()).thenReturn(1); + } + + @Test + void testDataOfferWriter_fullSingleUpdate() { + TEST_DATABASE.testTransaction(dsl -> { + // arrange + var connectorRef = TestData.connectorRef; + TestData.insertConnector(dsl, connectorRef, unused -> { + }); + var uiAsset = UiAsset.builder() + .assetId("assetId") + .title("title") + .description("# Description\n\n**with Markdown**") + .descriptionShortText("descriptionShortText") + .dataCategory("dataCategory") + .dataSubcategory("dataSubCategory") + .dataModel("dataModel") + .transportMode("transportMode") + .geoReferenceMethod("geoReferenceMethod") + .keywords(List.of("a", "b")) + .build(); + var fetchedContractOffer = FetchedContractOffer.builder() + .contractOfferId("contractOfferId") + .uiPolicyJson("\"test-policy\"") + .build(); + var fetchedDataOffer = FetchedDataOffer.builder() + .assetId("assetId") + .uiAsset(uiAsset) + .uiAssetJson("\"test\"") + .contractOffers(List.of(fetchedContractOffer)) + .build(); + var fetchedCatalog = FetchedCatalog.builder() + .connectorRef(connectorRef) + .dataOffers(List.of(fetchedDataOffer)) + .build(); + + // act + connectorUpdateSuccessWriter.handleConnectorOnline( + dsl, + connectorRef, + dsl.fetchOne( + Tables.CONNECTOR, + Tables.CONNECTOR.CONNECTOR_ID.eq(connectorRef.getConnectorId()) + ), + fetchedCatalog + ); + + // assert + var connector = dsl.fetchOne( + Tables.CONNECTOR, + Tables.CONNECTOR.CONNECTOR_ID.eq(connectorRef.getConnectorId()) + ); + var dataOffer = dsl.fetchOne( + Tables.DATA_OFFER, + DSL.and( + Tables.DATA_OFFER.CONNECTOR_ID.eq(connectorRef.getConnectorId()), + Tables.DATA_OFFER.ASSET_ID.eq("assetId") + ) + ); + var contractOffer = dsl.fetchOne( + Tables.CONTRACT_OFFER, + DSL.and( + Tables.CONTRACT_OFFER.CONNECTOR_ID.eq(connectorRef.getConnectorId()), + Tables.CONTRACT_OFFER.ASSET_ID.eq("assetId"), + Tables.CONTRACT_OFFER.CONTRACT_OFFER_ID.eq("contractOfferId") + ) + ); + + var now = OffsetDateTime.now(); + var minuteAccuracy = new TemporalUnitLessThanOffset(1, ChronoUnit.MINUTES); + assertThat(connector).isNotNull(); + assertThat(connector.getOnlineStatus()).isEqualTo(ConnectorOnlineStatus.ONLINE); + assertThat(connector.getLastRefreshAttemptAt()).isCloseTo(now, minuteAccuracy); + assertThat(connector.getLastSuccessfulRefreshAt()).isCloseTo(now, minuteAccuracy); + assertThat(connector.getDataOffersExceeded()).isEqualTo(ConnectorDataOffersExceeded.OK); + assertThat(connector.getContractOffersExceeded()).isEqualTo(ConnectorContractOffersExceeded.OK); + + assertThat(dataOffer).isNotNull(); + assertThat(dataOffer.getAssetTitle()).isEqualTo("title"); + assertThat(dataOffer.getDescriptionNoMarkdown()).isEqualTo("Description with Markdown"); + assertThat(dataOffer.getShortDescriptionNoMarkdown()).isEqualTo("descriptionShortText"); + assertThat(dataOffer.getDataCategory()).isEqualTo("dataCategory"); + assertThat(dataOffer.getDataSubcategory()).isEqualTo("dataSubCategory"); + assertThat(dataOffer.getDataModel()).isEqualTo("dataModel"); + assertThat(dataOffer.getTransportMode()).isEqualTo("transportMode"); + assertThat(dataOffer.getGeoReferenceMethod()).isEqualTo("geoReferenceMethod"); + assertThat(dataOffer.getKeywords()).containsExactly("a", "b"); + assertThat(dataOffer.getKeywordsCommaJoined()).isEqualTo("a, b"); + assertThat(dataOffer.getUiAssetJson().data()).isEqualTo("\"test\""); + + assertThat(contractOffer).isNotNull(); + assertThat(contractOffer.getUiPolicyJson().data()).isEqualTo("\"test-policy\""); + }); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDydi.java b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDydi.java index fb39bcbec..a46cf1dc4 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDydi.java +++ b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDydi.java @@ -14,6 +14,7 @@ package de.sovity.edc.ext.catalog.crawler.crawling.writing; +import de.sovity.edc.ext.catalog.crawler.crawling.logging.CrawlerEventLogger; import de.sovity.edc.ext.catalog.crawler.dao.CatalogPatchApplier; import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorQueries; import de.sovity.edc.ext.catalog.crawler.dao.contract_offers.ContractOfferQueries; @@ -21,6 +22,7 @@ import de.sovity.edc.ext.catalog.crawler.dao.data_offers.DataOfferQueries; import de.sovity.edc.ext.catalog.crawler.dao.data_offers.DataOfferRecordUpdater; import de.sovity.edc.ext.catalog.crawler.orchestration.config.CrawlerConfig; +import de.sovity.edc.ext.wrapper.api.common.mappers.asset.utils.ShortDescriptionBuilder; import lombok.Value; import org.eclipse.edc.spi.system.configuration.Config; @@ -34,9 +36,8 @@ class DataOfferWriterTestDydi { ContractOfferQueries contractOfferQueries = new ContractOfferQueries(); ContractOfferRecordUpdater contractOfferRecordUpdater = new ContractOfferRecordUpdater(); ConnectorQueries connectorQueries = new ConnectorQueries(crawlerConfig); - DataOfferRecordUpdater dataOfferRecordUpdater = new DataOfferRecordUpdater( - connectorQueries - ); + ShortDescriptionBuilder shortDescriptionBuilder = new ShortDescriptionBuilder(); + DataOfferRecordUpdater dataOfferRecordUpdater = new DataOfferRecordUpdater(shortDescriptionBuilder); CatalogPatchBuilder catalogPatchBuilder = new CatalogPatchBuilder( contractOfferQueries, dataOfferQueries, @@ -45,4 +46,16 @@ class DataOfferWriterTestDydi { ); CatalogPatchApplier catalogPatchApplier = new CatalogPatchApplier(); ConnectorUpdateCatalogWriter connectorUpdateCatalogWriter = new ConnectorUpdateCatalogWriter(catalogPatchBuilder, catalogPatchApplier); + + // for the ConnectorUpdateSuccessWriterTest + CrawlerEventLogger crawlerEventLogger = new CrawlerEventLogger(); + DataOfferLimitsEnforcer dataOfferLimitsEnforcer = new DataOfferLimitsEnforcer( + crawlerConfig, + crawlerEventLogger + ); + ConnectorUpdateSuccessWriter connectorUpdateSuccessWriter = new ConnectorUpdateSuccessWriter( + crawlerEventLogger, + connectorUpdateCatalogWriter, + dataOfferLimitsEnforcer + ); } From 9edec17eff33a9346dc55857a56df0f9693d6f25 Mon Sep 17 00:00:00 2001 From: Richard Treier Date: Mon, 15 Jul 2024 10:01:46 +0300 Subject: [PATCH 10/14] chore: prepare release (#999) --- extensions/catalog-crawler/README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/extensions/catalog-crawler/README.md b/extensions/catalog-crawler/README.md index 28de136de..881dbbf40 100644 --- a/extensions/catalog-crawler/README.md +++ b/extensions/catalog-crawler/README.md @@ -18,6 +18,7 @@ The catalog crawler is a deployment unit depending on an existing Authority Portal's database: +- It is a modified EDC connector with the task to crawl the other connector's public data offers. - It periodically checks the Authority Portal's connector list for its environment. - It crawls the given connectors in regular intervals. - It writes the data offers and connector statuses back into the Authority Portal DB. @@ -27,9 +28,13 @@ The catalog crawler is a deployment unit depending on an existing Authority Port The Authority Portal uses a non-EDC stack, and the EDC stack cannot handle multiple sources of authority at once. -With the `DB -> UI` part of the broker having been moved to the Authority Portal, only the `Catalog -> DB` part remains as the Catalog Crawler, +With the `DB -> UI` part of the broker having been moved to the Authority Portal, only the `Catalog -> DB` part remains as the Catalog Crawler, as it requires Connector-to-Connector IAM within the given Dataspace. +## Deployment + +Please see the [Catalog Crawler Productive Deployment Guide](../../docs/deployment-guide/goals/catalog-crawler-production/README.md) for more information. + ## License Apache License 2.0 - see [LICENSE](../../LICENSE) From 51bd1a999266ac08c226e115d43ea64cdf7d9d35 Mon Sep 17 00:00:00 2001 From: Christophe Loiseau Date: Thu, 18 Jul 2024 12:08:03 +0200 Subject: [PATCH 11/14] feat: contract termination (#996) --- extensions/catalog-crawler/catalog-crawler/build.gradle.kts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/extensions/catalog-crawler/catalog-crawler/build.gradle.kts b/extensions/catalog-crawler/catalog-crawler/build.gradle.kts index b63b7004f..7c2b46f7e 100644 --- a/extensions/catalog-crawler/catalog-crawler/build.gradle.kts +++ b/extensions/catalog-crawler/catalog-crawler/build.gradle.kts @@ -21,10 +21,9 @@ dependencies { testAnnotationProcessor(libs.lombok) testCompileOnly(libs.lombok) - testImplementation(project(":utils:test-connector-remote")) + testImplementation(project(":utils:test-utils")) testImplementation(libs.assertj.core) testImplementation(libs.mockito.core) - testImplementation(libs.mockito.inline) testImplementation(libs.restAssured.restAssured) testImplementation(libs.testcontainers.testcontainers) testImplementation(libs.flyway.core) From 8c02af785c104e2a39541442313c2403f77a735b Mon Sep 17 00:00:00 2001 From: Ilia Orlov <66363651+illfixit@users.noreply.github.com> Date: Fri, 19 Jul 2024 11:18:11 +0200 Subject: [PATCH 12/14] feat: extend UiPolicy from only constraints to expressions (#988) Co-authored-by: Richard Treier --- .../CrawlerExtensionContextBuilder.java | 171 +++++++++--------- 1 file changed, 87 insertions(+), 84 deletions(-) diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java index 2437dd751..f95463a9f 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java @@ -63,7 +63,8 @@ import de.sovity.edc.ext.wrapper.api.common.mappers.dataaddress.http.HttpDataSourceMapper; import de.sovity.edc.ext.wrapper.api.common.mappers.dataaddress.http.HttpHeaderMapper; import de.sovity.edc.ext.wrapper.api.common.mappers.policy.AtomicConstraintMapper; -import de.sovity.edc.ext.wrapper.api.common.mappers.policy.ConstraintExtractor; +import de.sovity.edc.ext.wrapper.api.common.mappers.policy.ExpressionExtractor; +import de.sovity.edc.ext.wrapper.api.common.mappers.policy.ExpressionMapper; import de.sovity.edc.ext.wrapper.api.common.mappers.policy.LiteralMapper; import de.sovity.edc.ext.wrapper.api.common.mappers.policy.OperatorMapper; import de.sovity.edc.ext.wrapper.api.common.mappers.policy.PolicyValidator; @@ -95,12 +96,12 @@ public class CrawlerExtensionContextBuilder { public static CrawlerExtensionContext buildContext( - Config config, - Monitor monitor, - TypeManager typeManager, - TypeTransformerRegistry typeTransformerRegistry, - JsonLd jsonLd, - CatalogService catalogService + Config config, + Monitor monitor, + TypeManager typeManager, + TypeTransformerRegistry typeTransformerRegistry, + JsonLd jsonLd, + CatalogService catalogService ) { // Config var crawlerConfigFactory = new CrawlerConfigFactory(config); @@ -124,9 +125,9 @@ public static CrawlerExtensionContext buildContext( var crawlerEventLogger = new CrawlerEventLogger(); var crawlerExecutionTimeLogger = new CrawlerExecutionTimeLogger(); var dataOfferMappingUtils = new FetchedCatalogMappingUtils( - policyMapper, - assetMapper, - objectMapperJsonLd + policyMapper, + assetMapper, + objectMapperJsonLd ); var contractOfferRecordUpdater = new ContractOfferRecordUpdater(); var shortDescriptionBuilder = new ShortDescriptionBuilder(); @@ -134,34 +135,34 @@ public static CrawlerExtensionContext buildContext( var contractOfferQueries = new ContractOfferQueries(); var dataOfferLimitsEnforcer = new DataOfferLimitsEnforcer(crawlerConfig, crawlerEventLogger); var dataOfferPatchBuilder = new CatalogPatchBuilder( - contractOfferQueries, - dataOfferQueries, - dataOfferRecordUpdater, - contractOfferRecordUpdater + contractOfferQueries, + dataOfferQueries, + dataOfferRecordUpdater, + contractOfferRecordUpdater ); var dataOfferPatchApplier = new CatalogPatchApplier(); var dataOfferWriter = new ConnectorUpdateCatalogWriter(dataOfferPatchBuilder, dataOfferPatchApplier); var connectorUpdateSuccessWriter = new ConnectorUpdateSuccessWriter( - crawlerEventLogger, - dataOfferWriter, - dataOfferLimitsEnforcer + crawlerEventLogger, + dataOfferWriter, + dataOfferLimitsEnforcer ); var fetchedDataOfferBuilder = new FetchedCatalogBuilder(dataOfferMappingUtils); var dspDataOfferBuilder = new DspDataOfferBuilder(jsonLd); var dspCatalogService = new DspCatalogService( - catalogService, - dspDataOfferBuilder + catalogService, + dspDataOfferBuilder ); var dataOfferFetcher = new FetchedCatalogService(dspCatalogService, fetchedDataOfferBuilder); var connectorUpdateFailureWriter = new ConnectorUpdateFailureWriter(crawlerEventLogger, monitor); var connectorUpdater = new ConnectorCrawler( - dataOfferFetcher, - connectorUpdateSuccessWriter, - connectorUpdateFailureWriter, - connectorQueries, - dslContextFactory, - monitor, - crawlerExecutionTimeLogger + dataOfferFetcher, + connectorUpdateSuccessWriter, + connectorUpdateFailureWriter, + connectorQueries, + dslContextFactory, + monitor, + crawlerExecutionTimeLogger ); var threadPoolTaskQueue = new ThreadPoolTaskQueue(); @@ -171,19 +172,19 @@ public static CrawlerExtensionContext buildContext( var connectorStatusUpdater = new ConnectorStatusUpdater(); var catalogCleaner = new CatalogCleaner(); var offlineConnectorCleaner = new OfflineConnectorCleaner( - crawlerConfig, - connectorQueries, - crawlerEventLogger, - connectorStatusUpdater, - catalogCleaner + crawlerConfig, + connectorQueries, + crawlerEventLogger, + connectorStatusUpdater, + catalogCleaner ); // Schedules List> jobs = List.of( - getOnlineConnectorRefreshCronJob(dslContextFactory, connectorQueueFiller), - getOfflineConnectorRefreshCronJob(dslContextFactory, connectorQueueFiller), - getDeadConnectorRefreshCronJob(dslContextFactory, connectorQueueFiller), - getOfflineConnectorCleanerCronJob(dslContextFactory, offlineConnectorCleaner) + getOnlineConnectorRefreshCronJob(dslContextFactory, connectorQueueFiller), + getOfflineConnectorRefreshCronJob(dslContextFactory, connectorQueueFiller), + getDeadConnectorRefreshCronJob(dslContextFactory, connectorQueueFiller), + getOfflineConnectorCleanerCronJob(dslContextFactory, offlineConnectorCleaner) ); // Startup @@ -191,68 +192,70 @@ public static CrawlerExtensionContext buildContext( var crawlerInitializer = new CrawlerInitializer(quartzScheduleInitializer); return new CrawlerExtensionContext( - crawlerInitializer, - dataSource, - dslContextFactory, - connectorUpdater, - policyMapper, - fetchedDataOfferBuilder, - dataOfferRecordUpdater + crawlerInitializer, + dataSource, + dslContextFactory, + connectorUpdater, + policyMapper, + fetchedDataOfferBuilder, + dataOfferRecordUpdater ); } @NotNull - private static PolicyMapper newPolicyMapper(TypeTransformerRegistry typeTransformerRegistry, ObjectMapper objectMapperJsonLd) { + private static PolicyMapper newPolicyMapper( + TypeTransformerRegistry typeTransformerRegistry, + ObjectMapper objectMapperJsonLd + ) { var operatorMapper = new OperatorMapper(); - var literalMapper = new LiteralMapper( - objectMapperJsonLd - ); + var literalMapper = new LiteralMapper(objectMapperJsonLd); var atomicConstraintMapper = new AtomicConstraintMapper( - literalMapper, - operatorMapper + literalMapper, + operatorMapper ); var policyValidator = new PolicyValidator(); - var constraintExtractor = new ConstraintExtractor( - policyValidator, - atomicConstraintMapper + var expressionMapper = new ExpressionMapper(atomicConstraintMapper); + var constraintExtractor = new ExpressionExtractor( + policyValidator, + expressionMapper ); return new PolicyMapper( - constraintExtractor, - atomicConstraintMapper, - typeTransformerRegistry + constraintExtractor, + expressionMapper, + typeTransformerRegistry ); } @NotNull private static AssetMapper newAssetMapper( - TypeTransformerRegistry typeTransformerRegistry, - JsonLd jsonLd + TypeTransformerRegistry typeTransformerRegistry, + JsonLd jsonLd ) { var edcPropertyUtils = new EdcPropertyUtils(); var assetJsonLdUtils = new AssetJsonLdUtils(); var assetEditRequestMapper = new AssetEditRequestMapper(); var shortDescriptionBuilder = new ShortDescriptionBuilder(); var assetJsonLdParser = new AssetJsonLdParser( - assetJsonLdUtils, - shortDescriptionBuilder, - endpoint -> false + assetJsonLdUtils, + shortDescriptionBuilder, + endpoint -> false ); var httpHeaderMapper = new HttpHeaderMapper(); var httpDataSourceMapper = new HttpDataSourceMapper(httpHeaderMapper); var dataSourceMapper = new DataSourceMapper( - edcPropertyUtils, - httpDataSourceMapper + edcPropertyUtils, + httpDataSourceMapper ); var assetJsonLdBuilder = new AssetJsonLdBuilder( - dataSourceMapper, - assetJsonLdParser, - assetEditRequestMapper + dataSourceMapper, + assetJsonLdParser, + assetEditRequestMapper ); return new AssetMapper( - typeTransformerRegistry, - assetJsonLdBuilder, - assetJsonLdParser, - jsonLd + typeTransformerRegistry, + assetJsonLdBuilder, + assetJsonLdParser, + jsonLd ); } @@ -260,33 +263,33 @@ private static AssetMapper newAssetMapper( private static CronJobRef getOfflineConnectorCleanerCronJob(DslContextFactory dslContextFactory, OfflineConnectorCleaner offlineConnectorCleaner) { return new CronJobRef<>( - CrawlerExtension.SCHEDULED_KILL_OFFLINE_CONNECTORS, - OfflineConnectorCleanerJob.class, - () -> new OfflineConnectorCleanerJob(dslContextFactory, offlineConnectorCleaner) + CrawlerExtension.SCHEDULED_KILL_OFFLINE_CONNECTORS, + OfflineConnectorCleanerJob.class, + () -> new OfflineConnectorCleanerJob(dslContextFactory, offlineConnectorCleaner) ); } @NotNull private static CronJobRef getOnlineConnectorRefreshCronJob( - DslContextFactory dslContextFactory, - ConnectorQueueFiller connectorQueueFiller + DslContextFactory dslContextFactory, + ConnectorQueueFiller connectorQueueFiller ) { return new CronJobRef<>( - CrawlerExtension.CRON_ONLINE_CONNECTOR_REFRESH, - OnlineConnectorRefreshJob.class, - () -> new OnlineConnectorRefreshJob(dslContextFactory, connectorQueueFiller) + CrawlerExtension.CRON_ONLINE_CONNECTOR_REFRESH, + OnlineConnectorRefreshJob.class, + () -> new OnlineConnectorRefreshJob(dslContextFactory, connectorQueueFiller) ); } @NotNull private static CronJobRef getOfflineConnectorRefreshCronJob( - DslContextFactory dslContextFactory, - ConnectorQueueFiller connectorQueueFiller + DslContextFactory dslContextFactory, + ConnectorQueueFiller connectorQueueFiller ) { return new CronJobRef<>( - CrawlerExtension.CRON_OFFLINE_CONNECTOR_REFRESH, - OfflineConnectorRefreshJob.class, - () -> new OfflineConnectorRefreshJob(dslContextFactory, connectorQueueFiller) + CrawlerExtension.CRON_OFFLINE_CONNECTOR_REFRESH, + OfflineConnectorRefreshJob.class, + () -> new OfflineConnectorRefreshJob(dslContextFactory, connectorQueueFiller) ); } @@ -294,9 +297,9 @@ private static CronJobRef getOfflineConnectorRefresh private static CronJobRef getDeadConnectorRefreshCronJob(DslContextFactory dslContextFactory, ConnectorQueueFiller connectorQueueFiller) { return new CronJobRef<>( - CrawlerExtension.CRON_DEAD_CONNECTOR_REFRESH, - DeadConnectorRefreshJob.class, - () -> new DeadConnectorRefreshJob(dslContextFactory, connectorQueueFiller) + CrawlerExtension.CRON_DEAD_CONNECTOR_REFRESH, + DeadConnectorRefreshJob.class, + () -> new DeadConnectorRefreshJob(dslContextFactory, connectorQueueFiller) ); } From 71c10d5a26b12e29951f5ce63ff2d1514afb4a3a Mon Sep 17 00:00:00 2001 From: Christophe Loiseau Date: Wed, 24 Jul 2024 13:29:59 +0200 Subject: [PATCH 13/14] feat: on request assets now return placeholder data (#1004) Co-authored-by: Richard Treier --- .../edc/ext/catalog/crawler/CrawlerExtension.java | 14 ++++++++------ .../crawler/CrawlerExtensionContextBuilder.java | 11 +++++++---- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java index 290c5c2c3..ae1f0eca4 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java @@ -14,6 +14,7 @@ package de.sovity.edc.ext.catalog.crawler; +import de.sovity.edc.ext.wrapper.api.common.mappers.PlaceholderEndpointService; import org.eclipse.edc.connector.api.management.configuration.transform.ManagementApiTypeTransformerRegistry; import org.eclipse.edc.connector.spi.catalog.CatalogService; import org.eclipse.edc.jsonld.spi.JsonLd; @@ -117,12 +118,13 @@ public void initialize(ServiceExtensionContext context) { } services = CrawlerExtensionContextBuilder.buildContext( - context.getConfig(), - context.getMonitor(), - typeManager, - typeTransformerRegistry, - jsonLd, - catalogService + context.getConfig(), + context.getMonitor(), + typeManager, + typeTransformerRegistry, + jsonLd, + catalogService, + new PlaceholderEndpointService("http://0.0.0.0/") ); // Provide access for the tests diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java index f95463a9f..89aa9e491 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java @@ -52,6 +52,7 @@ import de.sovity.edc.ext.catalog.crawler.orchestration.schedules.QuartzScheduleInitializer; import de.sovity.edc.ext.catalog.crawler.orchestration.schedules.utils.CronJobRef; import de.sovity.edc.ext.wrapper.api.common.mappers.AssetMapper; +import de.sovity.edc.ext.wrapper.api.common.mappers.PlaceholderEndpointService; import de.sovity.edc.ext.wrapper.api.common.mappers.PolicyMapper; import de.sovity.edc.ext.wrapper.api.common.mappers.asset.AssetEditRequestMapper; import de.sovity.edc.ext.wrapper.api.common.mappers.asset.AssetJsonLdBuilder; @@ -101,7 +102,8 @@ public static CrawlerExtensionContext buildContext( TypeManager typeManager, TypeTransformerRegistry typeTransformerRegistry, JsonLd jsonLd, - CatalogService catalogService + CatalogService catalogService, + PlaceholderEndpointService placeholderEndpointService ) { // Config var crawlerConfigFactory = new CrawlerConfigFactory(config); @@ -120,7 +122,7 @@ public static CrawlerExtensionContext buildContext( // Services var objectMapperJsonLd = getJsonLdObjectMapper(typeManager); - var assetMapper = newAssetMapper(typeTransformerRegistry, jsonLd); + var assetMapper = newAssetMapper(typeTransformerRegistry, jsonLd, placeholderEndpointService); var policyMapper = newPolicyMapper(typeTransformerRegistry, objectMapperJsonLd); var crawlerEventLogger = new CrawlerEventLogger(); var crawlerExecutionTimeLogger = new CrawlerExecutionTimeLogger(); @@ -229,7 +231,8 @@ private static PolicyMapper newPolicyMapper( @NotNull private static AssetMapper newAssetMapper( TypeTransformerRegistry typeTransformerRegistry, - JsonLd jsonLd + JsonLd jsonLd, + PlaceholderEndpointService placeholderEndpointService ) { var edcPropertyUtils = new EdcPropertyUtils(); var assetJsonLdUtils = new AssetJsonLdUtils(); @@ -241,7 +244,7 @@ private static AssetMapper newAssetMapper( endpoint -> false ); var httpHeaderMapper = new HttpHeaderMapper(); - var httpDataSourceMapper = new HttpDataSourceMapper(httpHeaderMapper); + var httpDataSourceMapper = new HttpDataSourceMapper(httpHeaderMapper, placeholderEndpointService); var dataSourceMapper = new DataSourceMapper( edcPropertyUtils, httpDataSourceMapper From 73d00c04c1f697740a940f8c66846f2c290cc321 Mon Sep 17 00:00:00 2001 From: Kamil Czaja <46053356+kamilczaja@users.noreply.github.com> Date: Thu, 8 Aug 2024 13:04:30 +0200 Subject: [PATCH 14/14] chore: sync ap migrations (#1021) --- .../ext/catalog/crawler/dao/connectors/ConnectorQueries.java | 4 ++-- .../test/java/de/sovity/edc/ext/catalog/crawler/TestData.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorQueries.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorQueries.java index 4c3e3843b..b77a1b708 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorQueries.java +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorQueries.java @@ -60,11 +60,11 @@ private Set queryConnectorRefs( c.CONNECTOR_ID.as("connectorId"), c.ENVIRONMENT.as("environmentId"), o.NAME.as("organizationLegalName"), - o.MDS_ID.as("organizationId"), + o.ID.as("organizationId"), c.ENDPOINT_URL.as("endpoint") ) .from(c) - .leftJoin(o).on(c.MDS_ID.eq(o.MDS_ID)) + .leftJoin(o).on(c.ORGANIZATION_ID.eq(o.ID)) .where(condition.apply(c, o), c.ENVIRONMENT.eq(crawlerConfig.getEnvironmentId())) .fetchInto(ConnectorRef.class); diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/TestData.java b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/TestData.java index b522c8881..102beb54c 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/TestData.java +++ b/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/TestData.java @@ -44,13 +44,13 @@ public static void insertConnector( Consumer applier ) { var organization = dsl.newRecord(Tables.ORGANIZATION); - organization.setMdsId(connectorRef.getOrganizationId()); + organization.setId(connectorRef.getOrganizationId()); organization.setName(connectorRef.getOrganizationLegalName()); organization.insert(); var connector = dsl.newRecord(Tables.CONNECTOR); connector.setEnvironment(connectorRef.getEnvironmentId()); - connector.setMdsId(connectorRef.getOrganizationId()); + connector.setOrganizationId(connectorRef.getOrganizationId()); connector.setConnectorId(connectorRef.getConnectorId()); connector.setName(connectorRef.getConnectorId() + " Name"); connector.setEndpointUrl(connectorRef.getEndpoint());