From 9f45c26e84b7067e530ff5803d60a735899b78da Mon Sep 17 00:00:00 2001 From: Tomvbe <34196062+Tomvbe@users.noreply.github.com> Date: Thu, 30 Nov 2023 16:06:48 +0100 Subject: [PATCH] feat: Implement first version of the federated catalog connector (#2) * poc federated catalog * finalisation of poc. TODO: propper e2e text * First check * Add config description * Cleanup --------- Co-authored-by: Yalz --- README.MD | 16 ++- e2e-test/README.md | 131 ++++++++++++++++-- e2e-test/docker-compose.yml | 29 +++- .../catalog-configuration.properties | 6 + e2e-test/federated-catalog/nodes-dc.json | 9 ++ federated-catalog-connector/Dockerfile | 14 ++ federated-catalog-connector/README.MD | 9 ++ federated-catalog-connector/build.gradle.kts | 50 +++++++ .../filesystem/CatalogExtension.java | 48 +++++++ .../filesystem/FileBasedNodeDirectory.java | 65 +++++++++ ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + gradle/libs.versions.toml | 9 ++ Dockerfile => http-pull-connector/Dockerfile | 0 settings.gradle.kts | 1 + 14 files changed, 369 insertions(+), 19 deletions(-) create mode 100644 e2e-test/federated-catalog/catalog-configuration.properties create mode 100644 e2e-test/federated-catalog/nodes-dc.json create mode 100644 federated-catalog-connector/Dockerfile create mode 100644 federated-catalog-connector/README.MD create mode 100644 federated-catalog-connector/build.gradle.kts create mode 100644 federated-catalog-connector/src/main/java/org/eclipse/edc/catalog/node/directory/filesystem/CatalogExtension.java create mode 100644 federated-catalog-connector/src/main/java/org/eclipse/edc/catalog/node/directory/filesystem/FileBasedNodeDirectory.java create mode 100644 federated-catalog-connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension rename Dockerfile => http-pull-connector/Dockerfile (100%) diff --git a/README.MD b/README.MD index 5e6e2ab..9122379 100644 --- a/README.MD +++ b/README.MD @@ -1,12 +1,18 @@ # VSDS Dataspace connector -This repository provides a connector build that can be used to create a dataspace with an LDES Server. +This repository provides connector builds that can be used to create a dataspace with an LDES Server. -To create the image run the following docker command from the root directory: +To create the images run the following docker commands from the root directory: -```bash -docker build -t vsds-dataspace-connector . -``` +1. Image for consumer and provider connectors + ```bash + docker build -t vsds-dataspace-connector:local . -f ./http-pull-connector/Dockerfile + ``` + +2. Image for the federated catalog connector + ```bash + docker build -t vsds-federated-catalog-connector:local . -f ./federated-catalog-connector/Dockerfile + ``` > **Note**: These connectors are not production ready and should only be used for development purposes. diff --git a/e2e-test/README.md b/e2e-test/README.md index 0fca065..842f4f0 100644 --- a/e2e-test/README.md +++ b/e2e-test/README.md @@ -109,13 +109,7 @@ This property is used to define the endpoint exposed by the control plane to val ## Run the connectors -Before we can start the connectors, we are going to build the image: - -```bash -docker build -t vsds-dataspace-connector:local ../. -``` - -After building the image, we can start the connectors with the following command: +We can start the connectors with the following command: ```bash docker compose --profile connectors up -d @@ -134,7 +128,53 @@ order. > drop it from the command. it's just used to format the output, and the same advice should be > applied to all calls that use `jq`. -### 1. Register data plane instance for provider + +### 0. Federated catalog connector - State before datasets are provided by the provider connector + +When the federated catalog connector is started, it will crawl the connectors defined in [nodes-dc.json](federated-catalog/nodes-dc.json). +In our test, this is done for the first time, 5 seconds after startup as defined by "edc.catalog.cache.execution.delay.seconds" in the [config](federated-catalog/catalog-configuration.properties). + +We can request the Federated Catalog with the following request: +```bash +curl 'http://localhost:8181/api/federatedcatalog' \ + -H 'Content-Type: application/json' \ + -d '{"criteria":[]}' \ + -s | jq +``` + +If you do this before the provider connectors have been crawled, then you will get an empty response: +```json +[] +``` + +After the first crawl we get the following response, which contains the connector but no datasets yet: + +```json +[ + { + "@id": "b05dd09e-f1d4-4c6b-9174-3b532480eb7b", + "@type": "dcat:Catalog", + "dcat:dataset": [], + "dcat:service": { + "@id": "c3e3e29b-84c8-4322-af59-7a4c524e190e", + "@type": "dcat:DataService", + "dct:terms": "connector", + "dct:endpointUrl": "http://provider-connector:19194/protocol" + }, + "edc:originator": "http://provider-connector:19194/protocol", + "edc:participantId": "provider", + "@context": { + "dct": "https://purl.org/dc/terms/", + "edc": "https://w3id.org/edc/v0.0.1/ns/", + "dcat": "https://www.w3.org/ns/dcat/", + "odrl": "http://www.w3.org/ns/odrl/2/", + "dspace": "https://w3id.org/dspace/v0.8/" + } + } +] +``` + +### 1. Provider connector - Register data plane instance for provider Before a consumer can start talking to a provider, it is necessary to register the data plane instance of a connector. This is done by sending a POST request to the management API of the @@ -161,7 +201,7 @@ curl -H 'Content-Type: application/json' \ -X POST "http://localhost:19193/management/v2/dataplanes" | -s | jq ``` -### 2. Register data plane instance for consumer +### 2. Consumer connector - Register data plane instance for consumer The same thing that is done for the provider must be done for the consumer @@ -182,7 +222,7 @@ curl -H 'Content-Type: application/json' \ -X POST "http://localhost:29193/management/v2/dataplanes" ``` -### 3. Create an Asset on the provider side +### 3. Provider connector - Create an Asset on the provider side The provider connector needs to transfer a file to the location specified by the consumer connector when the data are requested. In order to offer any data, the provider must maintain an internal list @@ -226,7 +266,7 @@ Additional properties on `HttpData` can be used to allow consumers to enrich the - `proxyBody`: allows attaching a body. - `proxyMethod`: allows specifying the Http Method (default `GET`) -### 4. Create a Policy on the provider +### 4. Provider connector - Create a Policy on the provider In order to manage the accessibility rules of an asset, it is essential to create a policy. However, to keep things simple, we will choose a policy that gives direct access to all the assets that are @@ -250,7 +290,7 @@ curl -d '{ -s | jq ``` -### 5. Create a contract definition on Provider +### 5. Provider connector - Create a contract definition on Provider To ensure an exchange between providers and consumers, the supplier must create a contract offer for the good, on the basis of which a contract agreement can be negotiated. The contract definition @@ -354,6 +394,73 @@ Sample output: } ``` +Additionally, the Federated Catalog will now also include this entry. +This may take a couple of seconds as the federated catalog connector only polls the provider every 5 seconds as defined by "edc.catalog.cache.execution.period.seconds" in the [config](federated-catalog/catalog-configuration.properties). + +```bash +curl 'http://localhost:8181/api/federatedcatalog' \ + -H 'Content-Type: application/json' \ + -d '{"criteria":[]}' \ + -s | jq +``` + +should output something like this + +```json +[ + { + "@id": "960d2187-c845-4e1c-9f5e-8beebc83171f", + "@type": "dcat:Catalog", + "dcat:dataset": { + "@id": "devices", + "@type": "dcat:Dataset", + "odrl:hasPolicy": { + "@id": "MQ==:ZGV2aWNlcw==:MWFiNGIwMzYtM2Y1Ni00ZmIwLWJlNzMtYjg5YzM4MTNkMDYz", + "@type": "odrl:Set", + "odrl:permission": [], + "odrl:prohibition": [], + "odrl:obligation": [], + "odrl:target": "devices" + }, + "dcat:distribution": [ + { + "@type": "dcat:Distribution", + "dct:format": { + "@id": "HttpProxy" + }, + "dcat:accessService": "c3e3e29b-84c8-4322-af59-7a4c524e190e" + }, + { + "@type": "dcat:Distribution", + "dct:format": { + "@id": "HttpData" + }, + "dcat:accessService": "c3e3e29b-84c8-4322-af59-7a4c524e190e" + } + ], + "edc:name": "device models", + "edc:id": "devices", + "edc:contenttype": "application/n-quads" + }, + "dcat:service": { + "@id": "c3e3e29b-84c8-4322-af59-7a4c524e190e", + "@type": "dcat:DataService", + "dct:terms": "connector", + "dct:endpointUrl": "http://provider-connector:19194/protocol" + }, + "edc:originator": "http://provider-connector:19194/protocol", + "edc:participantId": "provider", + "@context": { + "dct": "https://purl.org/dc/terms/", + "edc": "https://w3id.org/edc/v0.0.1/ns/", + "dcat": "https://www.w3.org/ns/dcat/", + "odrl": "http://www.w3.org/ns/odrl/2/", + "dspace": "https://w3id.org/dspace/v0.8/" + } + } +] +``` + ### 8. Start the workbench with the LdesClient ```bash diff --git a/e2e-test/docker-compose.yml b/e2e-test/docker-compose.yml index a54e324..4c4f2d9 100644 --- a/e2e-test/docker-compose.yml +++ b/e2e-test/docker-compose.yml @@ -4,6 +4,9 @@ services: provider-connector: container_name: connector_provider-connector image: vsds-dataspace-connector:local + build: + context: .. + dockerfile: http-pull-connector/Dockerfile environment: - EDC_KEYSTORE=certs/cert.pfx - EDC_KEYSTORE_PASSWORD=123456 @@ -27,6 +30,9 @@ services: consumer-connector: container_name: connector_consumer-connector image: vsds-dataspace-connector:local + build: + context: .. + dockerfile: http-pull-connector/Dockerfile environment: - EDC_KEYSTORE=certs/cert.pfx - EDC_KEYSTORE_PASSWORD=123456 @@ -47,6 +53,26 @@ services: profiles: - connectors + federated-catalog-connector: + container_name: federated-catalog-connector + image: vsds-federated-catalog-connector:local + build: + context: .. + dockerfile: federated-catalog-connector/Dockerfile + environment: + - EDC_FS_CONFIG=config/catalog-configuration.properties + - FCC_DIRECTORY_FILE=config/nodes-dc.json + depends_on: [ provider-connector ] + volumes: + - ./federated-catalog/catalog-configuration.properties:/app/config/catalog-configuration.properties + - ./federated-catalog/nodes-dc.json:/app/config/nodes-dc.json + ports: + - "8181:8181" + networks: + - ldes + profiles: + - connectors + test-message-generator: container_name: connector_test-message-generator image: ghcr.io/informatievlaanderen/test-message-generator:latest @@ -112,8 +138,7 @@ services: - delay-started ldio-workbench: -# image: ghcr.io/informatievlaanderen/ldi-orchestrator:latest - image: conn-client:latest + image: ldes/ldi-orchestrator:1.10.0-SNAPSHOT container_name: connector_ldio-workbench environment: - SPRING_CONFIG_NAME=application diff --git a/e2e-test/federated-catalog/catalog-configuration.properties b/e2e-test/federated-catalog/catalog-configuration.properties new file mode 100644 index 0000000..8117c0b --- /dev/null +++ b/e2e-test/federated-catalog/catalog-configuration.properties @@ -0,0 +1,6 @@ +edc.catalog.cache.execution.delay.seconds=5 +edc.catalog.cache.execution.period.seconds=5 +edc.catalog.cache.partition.num.crawlers=1 +edc.ids.id=urn:connector:fcc +edc.web.rest.cors.enabled=true +edc.web.rest.cors.headers="origin,content-type,accept,authorization,x-api-key" \ No newline at end of file diff --git a/e2e-test/federated-catalog/nodes-dc.json b/e2e-test/federated-catalog/nodes-dc.json new file mode 100644 index 0000000..c05fec3 --- /dev/null +++ b/e2e-test/federated-catalog/nodes-dc.json @@ -0,0 +1,9 @@ +[ + { + "name": "provider-connector", + "url": "http://provider-connector:19194/protocol", + "supportedProtocols": [ + "dataspace-protocol-http" + ] + } +] diff --git a/federated-catalog-connector/Dockerfile b/federated-catalog-connector/Dockerfile new file mode 100644 index 0000000..a1b21fa --- /dev/null +++ b/federated-catalog-connector/Dockerfile @@ -0,0 +1,14 @@ +# Build the jar file +FROM openjdk:18-ea-bullseye AS build + +WORKDIR /app +COPY ./ . +RUN ./gradlew clean build + +# Run the jar file +FROM openjdk:18-ea-bullseye +WORKDIR /app +COPY --from=build /app/federated-catalog-connector/build/libs/federated-catalog-connector.jar . + +# Specify the command to run your application +CMD java -Dedc.fs.config=$EDC_FS_CONFIG -Dfcc.directory.file=$FCC_DIRECTORY_FILE -jar /app/federated-catalog-connector.jar diff --git a/federated-catalog-connector/README.MD b/federated-catalog-connector/README.MD new file mode 100644 index 0000000..786785c --- /dev/null +++ b/federated-catalog-connector/README.MD @@ -0,0 +1,9 @@ +# Federated catalog connector + +Supported configuration: + +| property | description | required | default | +|--------------------------------------------|------------------------------------------------------------------------------------------------------------------|----------|-----------------------------------------| +| edc.catalog.cache.execution.delay.seconds | The initial delay for the cache crawler engine. | no | N/A (no initial execution will be done) | +| edc.catalog.cache.execution.period.seconds | The time to elapse between two crawl runs. | no | 60 | +| edc.catalog.cache.partition.num.crawlers | The number of crawlers (execution threads) that should be used. The engine will re-use crawlers when necessary. | no | 2 | diff --git a/federated-catalog-connector/build.gradle.kts b/federated-catalog-connector/build.gradle.kts new file mode 100644 index 0000000..805c076 --- /dev/null +++ b/federated-catalog-connector/build.gradle.kts @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial API and implementation + * + */ + +plugins { + `java-library` + id("application") + alias(libs.plugins.shadow) +} + +dependencies { + runtimeOnly(libs.edc.catalog.core) + runtimeOnly(libs.edc.catalog.api) + implementation(libs.edc.catalog.spi) + implementation(libs.edc.configuration.filesystem) + + implementation(libs.edc.util) + runtimeOnly(libs.edc.spi.jsonld) + + runtimeOnly(libs.bundles.edc.connector) + runtimeOnly(libs.edc.control.plane.core) + runtimeOnly(libs.edc.data.plane.selector.core) + + // IDS stuff + runtimeOnly(libs.edc.dsp) + runtimeOnly(libs.edc.iam.mock) +} + +application { + mainClass.set("org.eclipse.edc.boot.system.runtime.BaseRuntime") +} + +var distTar = tasks.getByName("distTar") +var distZip = tasks.getByName("distZip") + +tasks.withType { + mergeServiceFiles() + archiveFileName.set("federated-catalog-connector.jar") + dependsOn(distTar, distZip) +} \ No newline at end of file diff --git a/federated-catalog-connector/src/main/java/org/eclipse/edc/catalog/node/directory/filesystem/CatalogExtension.java b/federated-catalog-connector/src/main/java/org/eclipse/edc/catalog/node/directory/filesystem/CatalogExtension.java new file mode 100644 index 0000000..76bdc3c --- /dev/null +++ b/federated-catalog-connector/src/main/java/org/eclipse/edc/catalog/node/directory/filesystem/CatalogExtension.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial API and implementation + * + */ + +package org.eclipse.edc.catalog.node.directory.filesystem; + +import org.eclipse.edc.catalog.spi.FederatedCacheNodeDirectory; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.runtime.metamodel.annotation.Provider; +import org.eclipse.edc.spi.EdcException; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; +import org.eclipse.edc.spi.types.TypeManager; +import org.eclipse.edc.util.concurrency.LockManager; + +import java.io.File; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import static java.lang.String.format; +import static java.util.Optional.ofNullable; + +public class CatalogExtension implements ServiceExtension { + + private static final String FILE_LOCATION_SETTING = "fcc.directory.file"; + + @Inject + private TypeManager typeManager; + + @Provider + public FederatedCacheNodeDirectory createFileSystemDirectory(ServiceExtensionContext context) { + var setting = ofNullable(context.getSetting(FILE_LOCATION_SETTING, null)) + .orElseThrow(() -> new EdcException(format("Config property [%s] not found, will ABORT!", FILE_LOCATION_SETTING))); + + File nodeFile = new File(setting); + return new FileBasedNodeDirectory(nodeFile, new LockManager(new ReentrantReadWriteLock()), typeManager.getMapper()); + } + +} diff --git a/federated-catalog-connector/src/main/java/org/eclipse/edc/catalog/node/directory/filesystem/FileBasedNodeDirectory.java b/federated-catalog-connector/src/main/java/org/eclipse/edc/catalog/node/directory/filesystem/FileBasedNodeDirectory.java new file mode 100644 index 0000000..2a37886 --- /dev/null +++ b/federated-catalog-connector/src/main/java/org/eclipse/edc/catalog/node/directory/filesystem/FileBasedNodeDirectory.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial API and implementation + * + */ + +package org.eclipse.edc.catalog.node.directory.filesystem; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.eclipse.edc.catalog.spi.FederatedCacheNode; +import org.eclipse.edc.catalog.spi.FederatedCacheNodeDirectory; +import org.eclipse.edc.spi.EdcException; +import org.eclipse.edc.util.concurrency.LockManager; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; + +/** + * File-based node directory, solely intended for use in testing, specifically with docker-compose + */ +public class FileBasedNodeDirectory implements FederatedCacheNodeDirectory { + + private static final TypeReference> NODE_LIST_TYPE = new TypeReference<>() { + }; + private final List nodes = new ArrayList<>(); + private final LockManager lockManager; + private final ObjectMapper objectMapper; + + public FileBasedNodeDirectory(File nodeFile, LockManager lockManager, ObjectMapper objectMapper) { + this.lockManager = lockManager; + this.objectMapper = objectMapper; + readAll(nodeFile); + } + + @Override + public List getAll() { + return lockManager.readLock(() -> nodes); + } + + @Override + public void insert(FederatedCacheNode node) { + lockManager.writeLock(() -> nodes.add(node)); + } + + private void readAll(File nodeFile) { + try { + var content = Files.readString(nodeFile.toPath()); + nodes.addAll(objectMapper.readValue(content, NODE_LIST_TYPE)); + } catch (IOException e) { + throw new EdcException(e); + } + } +} diff --git a/federated-catalog-connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/federated-catalog-connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 0000000..27e69f2 --- /dev/null +++ b/federated-catalog-connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +org.eclipse.edc.catalog.node.directory.filesystem.CatalogExtension \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 593f383..01ee83c 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -22,6 +22,9 @@ edc-api-observability = { module = "org.eclipse.edc:api-observability", version. edc-auth-tokenbased = { module = "org.eclipse.edc:auth-tokenbased", version.ref = "edc" } edc-boot = { module = "org.eclipse.edc:boot", version.ref = "edc" } edc-build-plugin = { module = "org.eclipse.edc.edc-build:org.eclipse.edc.edc-build.gradle.plugin", version.ref = "edc" } +edc-catalog-core = { module = "org.eclipse.edc:federated-catalog-core", version.ref = "edc" } +edc-catalog-api = { module = "org.eclipse.edc:federated-catalog-api", version.ref = "edc" } +edc-catalog-spi = { module = "org.eclipse.edc:federated-catalog-spi", version.ref = "edc" } edc-configuration-filesystem = { module = "org.eclipse.edc:configuration-filesystem", version.ref = "edc" } edc-connector-core = { module = "org.eclipse.edc:connector-core", version.ref = "edc" } edc-control-plane-core = { module = "org.eclipse.edc:control-plane-core", version.ref = "edc" } @@ -40,6 +43,7 @@ edc-data-plane-util = { module = "org.eclipse.edc:data-plane-util", version.ref edc-dsp = { module = "org.eclipse.edc:dsp", version.ref = "edc" } edc-http = { module = "org.eclipse.edc:http", version.ref = "edc" } edc-iam-mock = { module = "org.eclipse.edc:iam-mock", version.ref = "edc" } +edc-jersey-core = { module = "org.eclipse.edc:jersey-core", version.ref = "edc" } edc-jersey-micrometer = { module = "org.eclipse.edc:jersey-micrometer", version.ref = "edc" } edc-jetty-micrometer = { module = "org.eclipse.edc:jetty-micrometer", version.ref = "edc" } edc-json-ld = { module = "org.eclipse.edc:json-ld", version.ref = "edc" } @@ -49,6 +53,8 @@ edc-micrometer-core = { module = "org.eclipse.edc:micrometer-core", version.ref edc-monitor-jdk-logger = { module = "org.eclipse.edc:monitor-jdk-logger", version.ref = "edc" } edc-provision-aws-s3 = { module = "org.eclipse.edc:provision-aws-s3", version.ref = "edc" } edc-runtime-metamodel = { module = "org.eclipse.edc:runtime-metamodel", version.ref = "edc" } +edc-spi-catalog = { module = "org.eclipse.edc:catalog-spi", version.ref = "edc" } +edc-spi-jsonld = { module = "org.eclipse.edc:json-ld-spi", version.ref = "edc" } edc-transfer-data-plane = { module = "org.eclipse.edc:transfer-data-plane", version.ref = "edc" } edc-transfer-process-api = { module = "org.eclipse.edc:transfer-process-api", version.ref = "edc" } edc-transfer-pull-http-receiver = { module = "org.eclipse.edc:transfer-pull-http-receiver", version.ref = "edc" } @@ -67,5 +73,8 @@ okhttp-mockwebserver = { module = "com.squareup.okhttp3:mockwebserver", version. opentelemetry-annotations = { module = "io.opentelemetry:opentelemetry-extension-annotations", version.ref = "openTelemetry" } restAssured = { module = "io.rest-assured:rest-assured", version.ref = "restAssured" } +[bundles] +edc-connector = ["edc-boot", "edc-connector-core", "edc-jersey-core", "edc-api-observability"] + [plugins] shadow = { id = "com.github.johnrengelman.shadow", version = "8.1.1" } diff --git a/Dockerfile b/http-pull-connector/Dockerfile similarity index 100% rename from Dockerfile rename to http-pull-connector/Dockerfile diff --git a/settings.gradle.kts b/settings.gradle.kts index b14786f..d4c9207 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -15,3 +15,4 @@ dependencyResolutionManagement { } include("http-pull-connector") +include("federated-catalog-connector")