From ddbdf57b742c6efa0471305521135318165d452e Mon Sep 17 00:00:00 2001 From: Christophe Loiseau Date: Sun, 21 Jul 2024 18:46:16 +0200 Subject: [PATCH 01/24] Implement dummy endpoint --- docs/api/sovity-edc-api-wrapper.yaml | 4 +- .../ext/catalog/crawler/CrawlerE2eTest.java | 2 + .../ext/catalog/crawler/CrawlerExtension.java | 17 ++-- .../CrawlerExtensionContextBuilder.java | 21 +++-- extensions/placeholder-data-source/README.md | 73 +++++++++++++++ .../placeholder-data-source/build.gradle.kts | 30 ++++++ .../PlaceholderDataSourceExtension.java | 52 +++++++++++ .../PlaceholderEndpointController.java | 45 +++++++++ .../PlaceholderEndpointService.java | 36 ++++++++ ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + .../build.gradle.kts | 1 + .../wrapper-common-mappers/build.gradle.kts | 1 + .../api/common/mappers/AssetMapper.java | 2 + .../mappers/asset/AssetEditRequestMapper.java | 9 +- .../mappers/asset/AssetJsonLdBuilder.java | 29 +++++- .../mappers/dataaddress/DataSourceMapper.java | 2 + .../http/HttpDataSourceMapper.java | 10 +- .../wrapper/api/common/mappers/Factory.java | 3 +- .../mappers/asset/AssetJsonLdBuilderTest.java | 2 +- .../edc/ext/wrapper/WrapperExtension.java | 6 +- .../WrapperExtensionContextBuilder.java | 11 ++- settings.gradle.kts | 1 + tests/build.gradle.kts | 1 + .../edc/e2e/PlaceholderDataSourceTest.java | 92 +++++++++++++++++++ .../config/ConnectorConfigFactory.java | 5 + .../EdcRuntimeExtensionWithTestDatabase.java | 21 ++++- .../extension/e2e/extension/E2eScenario.java | 11 ++- 27 files changed, 453 insertions(+), 35 deletions(-) create mode 100644 extensions/placeholder-data-source/README.md create mode 100644 extensions/placeholder-data-source/build.gradle.kts create mode 100644 extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderDataSourceExtension.java create mode 100644 extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderEndpointController.java create mode 100644 extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderEndpointService.java create mode 100644 extensions/placeholder-data-source/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension create mode 100644 tests/src/test/java/de/sovity/edc/e2e/PlaceholderDataSourceTest.java diff --git a/docs/api/sovity-edc-api-wrapper.yaml b/docs/api/sovity-edc-api-wrapper.yaml index 3f9ca3a7f..4440e71de 100644 --- a/docs/api/sovity-edc-api-wrapper.yaml +++ b/docs/api/sovity-edc-api-wrapper.yaml @@ -584,11 +584,11 @@ components: DataSourceType: type: string description: Supported Data Source Types by UiDataSource - default: CUSTOM enum: - HTTP_DATA - ON_REQUEST - CUSTOM + default: CUSTOM SecretValue: type: object properties: @@ -796,7 +796,6 @@ components: UiDataSourceHttpDataMethod: type: string description: Supported HTTP Methods by UiDataSource - default: GET enum: - GET - POST @@ -804,6 +803,7 @@ components: - PATCH - DELETE - OPTIONS + default: GET UiDataSourceOnRequest: required: - contactEmail diff --git a/extensions/catalog-crawler/catalog-crawler-e2e-test/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerE2eTest.java b/extensions/catalog-crawler/catalog-crawler-e2e-test/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerE2eTest.java index 759f65cf4..bf2b0153d 100644 --- a/extensions/catalog-crawler/catalog-crawler-e2e-test/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerE2eTest.java +++ b/extensions/catalog-crawler/catalog-crawler-e2e-test/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerE2eTest.java @@ -36,6 +36,7 @@ import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorOnlineStatus; import de.sovity.edc.ext.catalog.crawler.utils.CrawlerDbAccess; import de.sovity.edc.ext.catalog.crawler.utils.TestData; +import de.sovity.edc.extension.placeholderdatasource.PlaceholderDataSourceExtension; import de.sovity.edc.extension.e2e.connector.config.ConnectorConfig; import de.sovity.edc.extension.e2e.db.EdcRuntimeExtensionWithTestDatabase; import de.sovity.edc.utils.jsonld.vocab.Prop; @@ -106,6 +107,7 @@ class CrawlerE2eTest { props.put(CrawlerExtension.CRON_DEAD_CONNECTOR_REFRESH, everySeconds); props.put(CrawlerExtension.SCHEDULED_KILL_OFFLINE_CONNECTORS, everySeconds); props.put(CrawlerExtension.KILL_OFFLINE_CONNECTORS_AFTER, "P1D"); + props.put(PlaceholderDataSourceExtension.MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL, "http://example.com/edc/backend"); return props; } diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java index 290c5c2c3..b53f47d7b 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java @@ -14,6 +14,7 @@ package de.sovity.edc.ext.catalog.crawler; +import de.sovity.edc.extension.placeholderdatasource.PlaceholderEndpointService; import org.eclipse.edc.connector.api.management.configuration.transform.ManagementApiTypeTransformerRegistry; import org.eclipse.edc.connector.spi.catalog.CatalogService; import org.eclipse.edc.jsonld.spi.JsonLd; @@ -90,6 +91,9 @@ public class CrawlerExtension implements ServiceExtension { @Inject private TypeManager typeManager; + @Inject + private PlaceholderEndpointService placeholderEndpointService; + @Inject private ManagementApiTypeTransformerRegistry typeTransformerRegistry; @@ -117,12 +121,13 @@ public void initialize(ServiceExtensionContext context) { } services = CrawlerExtensionContextBuilder.buildContext( - context.getConfig(), - context.getMonitor(), - typeManager, - typeTransformerRegistry, - jsonLd, - catalogService + context.getConfig(), + context.getMonitor(), + typeManager, + typeTransformerRegistry, + jsonLd, + catalogService, + placeholderEndpointService ); // Provide access for the tests diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java index f95463a9f..cbdc1252e 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java @@ -68,6 +68,7 @@ import de.sovity.edc.ext.wrapper.api.common.mappers.policy.LiteralMapper; import de.sovity.edc.ext.wrapper.api.common.mappers.policy.OperatorMapper; import de.sovity.edc.ext.wrapper.api.common.mappers.policy.PolicyValidator; +import de.sovity.edc.extension.placeholderdatasource.PlaceholderEndpointService; import de.sovity.edc.utils.catalog.DspCatalogService; import de.sovity.edc.utils.catalog.mapper.DspDataOfferBuilder; import lombok.NoArgsConstructor; @@ -96,12 +97,13 @@ public class CrawlerExtensionContextBuilder { public static CrawlerExtensionContext buildContext( - Config config, - Monitor monitor, - TypeManager typeManager, - TypeTransformerRegistry typeTransformerRegistry, - JsonLd jsonLd, - CatalogService catalogService + Config config, + Monitor monitor, + TypeManager typeManager, + TypeTransformerRegistry typeTransformerRegistry, + JsonLd jsonLd, + CatalogService catalogService, + PlaceholderEndpointService placeholderEndpointService ) { // Config var crawlerConfigFactory = new CrawlerConfigFactory(config); @@ -120,7 +122,7 @@ public static CrawlerExtensionContext buildContext( // Services var objectMapperJsonLd = getJsonLdObjectMapper(typeManager); - var assetMapper = newAssetMapper(typeTransformerRegistry, jsonLd); + var assetMapper = newAssetMapper(typeTransformerRegistry, jsonLd, placeholderEndpointService); var policyMapper = newPolicyMapper(typeTransformerRegistry, objectMapperJsonLd); var crawlerEventLogger = new CrawlerEventLogger(); var crawlerExecutionTimeLogger = new CrawlerExecutionTimeLogger(); @@ -229,7 +231,8 @@ private static PolicyMapper newPolicyMapper( @NotNull private static AssetMapper newAssetMapper( TypeTransformerRegistry typeTransformerRegistry, - JsonLd jsonLd + JsonLd jsonLd, + PlaceholderEndpointService placeholderEndpointService ) { var edcPropertyUtils = new EdcPropertyUtils(); var assetJsonLdUtils = new AssetJsonLdUtils(); @@ -241,7 +244,7 @@ private static AssetMapper newAssetMapper( endpoint -> false ); var httpHeaderMapper = new HttpHeaderMapper(); - var httpDataSourceMapper = new HttpDataSourceMapper(httpHeaderMapper); + var httpDataSourceMapper = new HttpDataSourceMapper(httpHeaderMapper, placeholderEndpointService); var dataSourceMapper = new DataSourceMapper( edcPropertyUtils, httpDataSourceMapper diff --git a/extensions/placeholder-data-source/README.md b/extensions/placeholder-data-source/README.md new file mode 100644 index 000000000..7dc593b48 --- /dev/null +++ b/extensions/placeholder-data-source/README.md @@ -0,0 +1,73 @@ + +
+
+ + Logo + + +

EDC-Connector Extension:
Sovity Messenger

+ +

+ Report Bug + · + Request Feature +

+
+ +## About this Extension + +Provides a placeholder endpoint for on-request offers. + +## Why does this extension exist? + +This extension exists to inform the asset's consumer upon data retrieval that they should contact the provider and take extra steps to access the data. + +## Configuration + +`MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL` / `my.edc.datasource.placeholder.baseurl` must be set to point to the placeholder endpoint's base URL. +`/data-source/placeholder/asset/` will be appended to this base URL to make it the placeholder data source endpoint. This will be the address as seen by the consumer. + +--- + +On a production system, the base URL it could be: + +`https://mycompany.com/path/to/backend` + +with a placeholder value: + +`https://mycompany.com/path/to/backend/data-source/placeholder/asset/` + +and a full path to the asset's data: + +`https://mycompany.com/path/to/backend/data-source/placeholder/asset?email=foo%40example.com&subject=Contact+us+now`. + +--- + +On a system started with docker-compose, it will be pointing to the DSP port on the provider's EDC + +`http://edc:11003/` + +`http://edc:11003/data-source/placeholder/asset/` + +`http://edc:11003/data-source/placeholder/asset?email=foo%40example.com&subject=Contact+us+now` + +--- + +During local/dev/unit test execution, it will be pointing to the DSP port on the provider's EDC + +`http://localhost:12345/` + +where `12345` would be chosen at random + +`http://localhost:12345/data-source/placeholder/asset/` + +`http://localhost:12345/data-source/placeholder/asset?email=foo%40example.com&subject=Contact+us+now` + +## License + +Apache License 2.0 - see [LICENSE](../../LICENSE) + +## Contact + +sovity GmbH - contact@sovity.de + diff --git a/extensions/placeholder-data-source/build.gradle.kts b/extensions/placeholder-data-source/build.gradle.kts new file mode 100644 index 000000000..eefea45bd --- /dev/null +++ b/extensions/placeholder-data-source/build.gradle.kts @@ -0,0 +1,30 @@ + +plugins { + `java-library` + `maven-publish` +} + +dependencies { + annotationProcessor(libs.lombok) + compileOnly(libs.lombok) + + api(libs.edc.coreSpi) + api(libs.edc.controlPlaneSpi) + implementation(libs.edc.apiCore) + implementation(libs.edc.dspApiConfiguration) + implementation(libs.okhttp.okhttp) + + testImplementation(libs.mockito.core) + testImplementation(libs.junit.api) + testRuntimeOnly(libs.junit.engine) +} + +group = libs.versions.sovityEdcExtensionGroup.get() + +publishing { + publications { + create(project.name) { + from(components["java"]) + } + } +} diff --git a/extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderDataSourceExtension.java b/extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderDataSourceExtension.java new file mode 100644 index 000000000..12db5c12a --- /dev/null +++ b/extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderDataSourceExtension.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2024 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.extension.placeholderdatasource; + +import lombok.val; +import org.eclipse.edc.protocol.dsp.api.configuration.DspApiConfiguration; +import org.eclipse.edc.runtime.metamodel.annotation.Extension; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.runtime.metamodel.annotation.Provides; +import org.eclipse.edc.runtime.metamodel.annotation.Setting; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; +import org.eclipse.edc.web.spi.WebService; + +import static de.sovity.edc.extension.placeholderdatasource.PlaceholderDataSourceExtension.NAME; + +@Extension(value = NAME) +@Provides(PlaceholderEndpointService.class) +public class PlaceholderDataSourceExtension implements ServiceExtension { + + public static final String NAME = "Placeholder Data Source"; + + @Setting(required = true) + public static final String MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL = "my.edc.datasource.placeholder.baseurl"; + + @Inject + private DspApiConfiguration dspApiConfiguration; + + @Inject + private WebService webService; + + @Override + public void initialize(ServiceExtensionContext context) { + val controller = new PlaceholderEndpointController(); + webService.registerResource(dspApiConfiguration.getContextAlias(), controller); + + val baseUrl = context.getConfig().getString(MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL); + context.registerService(PlaceholderEndpointService.class, new PlaceholderEndpointService(baseUrl)); + } +} diff --git a/extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderEndpointController.java b/extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderEndpointController.java new file mode 100644 index 000000000..d0dcf71ab --- /dev/null +++ b/extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderEndpointController.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2024 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.extension.placeholderdatasource; + +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.QueryParam; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +@Produces({MediaType.APPLICATION_JSON}) +@Path("/data-source/placeholder") +public class PlaceholderEndpointController { + + @GET + @Path("/{path:.*}") + @Produces(MediaType.TEXT_PLAIN) + @Consumes("*/*") + public Response get(@QueryParam("email") String email, @QueryParam("subject") String subject) { + return Response.ok(""" + This is not a real data offer. + + The offer you are trying to use only has this placeholder as a dummy endpoint and requires you top take extra actions to access it. + + Please contact the data provider for more information about how to access it. + """ + "\n\n" + "Email: " + email + "\n" + "Subject: " + subject + "\n" + ).build(); + } +} diff --git a/extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderEndpointService.java b/extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderEndpointService.java new file mode 100644 index 000000000..36b59f272 --- /dev/null +++ b/extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderEndpointService.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2024 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.extension.placeholderdatasource; + +import lombok.RequiredArgsConstructor; +import okhttp3.HttpUrl; + +@RequiredArgsConstructor +public class PlaceholderEndpointService { + + private final String baseUrl; + + public static final String DUMMY_ENDPOINT_URL = "/data-source/placeholder/asset"; + + // TODO: maybe add String contactEmail, String contactPreferredEmailSubject and show them in the placeholder + public String getPlaceholderEndpointForAsset(String email, String subject) { + return HttpUrl.parse(baseUrl + DUMMY_ENDPOINT_URL) + .newBuilder() + .addQueryParameter("email", email) + .addQueryParameter("subject", subject) + .build() + .toString(); + } +} diff --git a/extensions/placeholder-data-source/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/placeholder-data-source/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 000000000..1c4398d5d --- /dev/null +++ b/extensions/placeholder-data-source/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +de.sovity.edc.extension.placeholderdatasource.PlaceholderDataSourceExtension diff --git a/extensions/sovity-edc-extensions-package/build.gradle.kts b/extensions/sovity-edc-extensions-package/build.gradle.kts index dc4edc6d1..b9b74a6a5 100644 --- a/extensions/sovity-edc-extensions-package/build.gradle.kts +++ b/extensions/sovity-edc-extensions-package/build.gradle.kts @@ -6,6 +6,7 @@ plugins { dependencies { // Policies + api(project(":extensions:placeholder-data-source")) api(project(":extensions:policy-referring-connector")) api(project(":extensions:policy-time-interval")) api(project(":extensions:policy-always-true")) diff --git a/extensions/wrapper/wrapper-common-mappers/build.gradle.kts b/extensions/wrapper/wrapper-common-mappers/build.gradle.kts index fe5a1bf79..835e6475c 100644 --- a/extensions/wrapper/wrapper-common-mappers/build.gradle.kts +++ b/extensions/wrapper/wrapper-common-mappers/build.gradle.kts @@ -12,6 +12,7 @@ dependencies { api(libs.edc.coreSpi) api(libs.edc.transformCore) api(libs.edc.transformSpi) + api(project(":extensions:placeholder-data-source")) api(project(":extensions:wrapper:wrapper-common-api")) api(project(":utils:json-and-jsonld-utils")) implementation(libs.apache.commonsLang) diff --git a/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/AssetMapper.java b/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/AssetMapper.java index 4ac252075..52eb6cdac 100644 --- a/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/AssetMapper.java +++ b/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/AssetMapper.java @@ -53,6 +53,8 @@ public Asset editAsset( @NonNull UiAssetEditRequest editRequest ) { var assetJsonLd = buildAssetJsonLd(asset); + // edit request is all empty + // assetJsonLd contains the manually set baseUrl var editedJsonLd = assetJsonLdBuilder.editAssetJsonLd(assetJsonLd, editRequest); return buildAsset(editedJsonLd); } diff --git a/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/asset/AssetEditRequestMapper.java b/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/asset/AssetEditRequestMapper.java index c65fe5243..8a949cf7f 100644 --- a/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/asset/AssetEditRequestMapper.java +++ b/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/asset/AssetEditRequestMapper.java @@ -32,7 +32,10 @@ public class AssetEditRequestMapper { * @param editRequest {@link UiAssetEditRequest} * @return {@link UiAssetCreateRequest} */ - public UiAssetCreateRequest buildCreateRequest(@NonNull UiAssetEditRequest editRequest, @NonNull String assetId) { + public UiAssetCreateRequest buildCreateRequest( + @NonNull UiAssetEditRequest editRequest, + @NonNull String assetId + ) { var dataSource = editRequest.getDataSourceOverrideOrNull(); if (dataSource == null) { dataSource = dummyDataSource(); @@ -73,6 +76,8 @@ public UiAssetCreateRequest buildCreateRequest(@NonNull UiAssetEditRequest editR } private UiDataSource dummyDataSource() { - return UiDataSource.builder().type(DataSourceType.CUSTOM).build(); + return UiDataSource.builder() + .type(DataSourceType.CUSTOM) + .build(); } } diff --git a/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/asset/AssetJsonLdBuilder.java b/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/asset/AssetJsonLdBuilder.java index e8c7cd00b..ca5db7c59 100644 --- a/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/asset/AssetJsonLdBuilder.java +++ b/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/asset/AssetJsonLdBuilder.java @@ -63,20 +63,41 @@ public JsonObject createAssetJsonLd( ); } - @SneakyThrows @Nullable public JsonObject editAssetJsonLd( - JsonObject assetJsonLd, - UiAssetEditRequest editRequest + JsonObject assetJsonLd, // 3 + UiAssetEditRequest editRequest // 2 ) { + + /* + * Data source selection order: + * override? + * edit request + * jsonLd + * placeholder + */ + + // request base URL comes here var dataAddress = getDataAddressJsonLd(assetJsonLd); + // override is null if (editRequest.getDataSourceOverrideOrNull() != null) { + // this part of the code is not asserted + // what is this override used for? + // TODO should this part have higher or lower prio than the automatic assignment to the dummy data source? + // 1 + // there is another handling of this field in de.sovity.edc.ext.wrapper.api.common.mappers.asset.AssetEditRequestMapper.buildCreateRequest + // TODO: this is not a clear name on the API side: is `orNull` the Java null of the JSON null? dataAddress = dataSourceMapper.buildDataSourceJsonLd(editRequest.getDataSourceOverrideOrNull()); } + // the dataAddress that is used should be the one that's part of the createRequest, not a separate element + var assetId = Objects.requireNonNull(JsonLdUtils.string(assetJsonLd, Prop.ID), "Asset JSON-LD had no @id"); var organizationName = assetJsonLdParser.getCreatorOrganizationName(assetJsonLd); + // set dummy source to placeholder URL inside this createRequest + // this `createRequest` contains the correct dataSource + // 4: dummy var createRequest = assetEditRequestMapper.buildCreateRequest(editRequest, assetId); var properties = getAssetProperties(createRequest, dataAddress, organizationName); var privateProperties = getAssetPrivateProperties(createRequest); @@ -133,6 +154,8 @@ private JsonObject getAssetProperties( addMobilityTheme(properties, request); addCustomJsonLd(properties, request); +// request.getDataSource(). + // TODO: replace with request.getDataSource() addDataSourceHints(properties, dataAddressJsonLd); return properties.build(); } diff --git a/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/dataaddress/DataSourceMapper.java b/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/dataaddress/DataSourceMapper.java index b8ccae7ac..0dc5280bd 100644 --- a/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/dataaddress/DataSourceMapper.java +++ b/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/dataaddress/DataSourceMapper.java @@ -29,6 +29,7 @@ import lombok.RequiredArgsConstructor; import java.util.Map; +import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Supplier; @@ -41,6 +42,7 @@ public class DataSourceMapper { private final HttpDataSourceMapper httpDataSourceMapper; public JsonObject buildDataSourceJsonLd(@NonNull UiDataSource dataSource) { + // this datasource has the URL set in the creation request var props = this.matchDataSource( dataSource, httpDataSourceMapper::buildDataAddress, diff --git a/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/dataaddress/http/HttpDataSourceMapper.java b/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/dataaddress/http/HttpDataSourceMapper.java index 32812934d..5cb085ed7 100644 --- a/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/dataaddress/http/HttpDataSourceMapper.java +++ b/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/dataaddress/http/HttpDataSourceMapper.java @@ -16,6 +16,7 @@ import de.sovity.edc.ext.wrapper.api.common.model.UiDataSourceHttpData; import de.sovity.edc.ext.wrapper.api.common.model.UiDataSourceOnRequest; +import de.sovity.edc.extension.placeholderdatasource.PlaceholderEndpointService; import de.sovity.edc.utils.jsonld.vocab.Prop; import jakarta.json.Json; import jakarta.json.JsonObject; @@ -33,6 +34,7 @@ @RequiredArgsConstructor public class HttpDataSourceMapper { private final HttpHeaderMapper httpHeaderMapper; + private final PlaceholderEndpointService placeholderEndpointService; /** * Data Address for type HTTP_DATA @@ -92,8 +94,13 @@ public Map buildOnRequestDataAddress(@NonNull UiDataSourceOnRequ "Need contactPreferredEmailSubject" ); + // this seems to be in use + String placeholderEndpointForAsset = placeholderEndpointService.getPlaceholderEndpointForAsset( + onRequest.getContactEmail(), + onRequest.getContactPreferredEmailSubject()); + var actualDataSource = UiDataSourceHttpData.builder() - .baseUrl("http://0.0.0.0") + .baseUrl(placeholderEndpointForAsset) .build(); var props = buildDataAddress(actualDataSource); @@ -123,6 +130,7 @@ public JsonObject enhanceAssetWithDataSourceHints(Map dataAddres Prop.Edc.PROXY_BODY, Prop.SovityDcatExt.HttpDatasourceHints.BODY ).forEach((prop, hint) -> // Will add hints as "true" or "false" + // TODO: what are those hints for? External info about the content of those fields?? json.add(hint, String.valueOf("true".equals(dataAddress.get(prop)))) ); } diff --git a/extensions/wrapper/wrapper-common-mappers/src/test/java/de/sovity/edc/ext/wrapper/api/common/mappers/Factory.java b/extensions/wrapper/wrapper-common-mappers/src/test/java/de/sovity/edc/ext/wrapper/api/common/mappers/Factory.java index 4c9d7093b..99505398e 100644 --- a/extensions/wrapper/wrapper-common-mappers/src/test/java/de/sovity/edc/ext/wrapper/api/common/mappers/Factory.java +++ b/extensions/wrapper/wrapper-common-mappers/src/test/java/de/sovity/edc/ext/wrapper/api/common/mappers/Factory.java @@ -24,6 +24,7 @@ import de.sovity.edc.ext.wrapper.api.common.mappers.dataaddress.DataSourceMapper; import de.sovity.edc.ext.wrapper.api.common.mappers.dataaddress.http.HttpDataSourceMapper; import de.sovity.edc.ext.wrapper.api.common.mappers.dataaddress.http.HttpHeaderMapper; +import de.sovity.edc.extension.placeholderdatasource.PlaceholderEndpointService; import org.eclipse.edc.jsonld.TitaniumJsonLd; import org.eclipse.edc.spi.monitor.Monitor; import org.eclipse.edc.transform.spi.TypeTransformerRegistry; @@ -50,7 +51,7 @@ public static AssetJsonLdBuilder newAssetJsonLdBuilder(OwnConnectorEndpointServi return new AssetJsonLdBuilder( new DataSourceMapper( new EdcPropertyUtils(), - new HttpDataSourceMapper(new HttpHeaderMapper()) + new HttpDataSourceMapper(new HttpHeaderMapper(), new PlaceholderEndpointService("http://example.com/dummy/baseUrl")) ), newAssetJsonLdParser(ownConnectorEndpointService), new AssetEditRequestMapper() diff --git a/extensions/wrapper/wrapper-common-mappers/src/test/java/de/sovity/edc/ext/wrapper/api/common/mappers/asset/AssetJsonLdBuilderTest.java b/extensions/wrapper/wrapper-common-mappers/src/test/java/de/sovity/edc/ext/wrapper/api/common/mappers/asset/AssetJsonLdBuilderTest.java index ea0b8658e..060c29d03 100644 --- a/extensions/wrapper/wrapper-common-mappers/src/test/java/de/sovity/edc/ext/wrapper/api/common/mappers/asset/AssetJsonLdBuilderTest.java +++ b/extensions/wrapper/wrapper-common-mappers/src/test/java/de/sovity/edc/ext/wrapper/api/common/mappers/asset/AssetJsonLdBuilderTest.java @@ -534,7 +534,7 @@ void test_create_onRequest() { var dataAddress = Json.createObjectBuilder() .add(Prop.TYPE, Prop.Edc.TYPE_DATA_ADDRESS) .add(Prop.Edc.TYPE, Prop.Edc.DATA_ADDRESS_TYPE_HTTP_DATA) - .add(Prop.Edc.BASE_URL, "http://0.0.0.0") + .add(Prop.Edc.BASE_URL, "http://example.com/dummy/baseUrl/data-source/placeholder/asset?email=contact%40example.com&subject=Test") .add(Prop.SovityDcatExt.DATA_SOURCE_AVAILABILITY, Prop.SovityDcatExt.DATA_SOURCE_AVAILABILITY_ON_REQUEST) .add(Prop.SovityDcatExt.CONTACT_EMAIL, "contact@example.com") .add(Prop.SovityDcatExt.CONTACT_PREFERRED_EMAIL_SUBJECT, "Test"); diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtension.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtension.java index 8f2b29210..df6cd42bb 100644 --- a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtension.java +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtension.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import de.sovity.edc.extension.contacttermination.ContractAgreementTerminationService; import de.sovity.edc.extension.db.directaccess.DslContextFactory; +import de.sovity.edc.extension.placeholderdatasource.PlaceholderEndpointService; import de.sovity.edc.extension.messenger.SovityMessenger; import org.eclipse.edc.connector.api.management.configuration.ManagementApiConfiguration; import org.eclipse.edc.connector.api.management.configuration.transform.ManagementApiTypeTransformerRegistry; @@ -69,6 +70,8 @@ public class WrapperExtension implements ServiceExtension { @Inject private ManagementApiConfiguration dataManagementApiConfiguration; @Inject + private PlaceholderEndpointService placeholderEndpointService; + @Inject private PolicyDefinitionStore policyDefinitionStore; @Inject private PolicyEngine policyEngine; @@ -122,7 +125,8 @@ public void initialize(ServiceExtensionContext context) { sovityMessenger, transferProcessService, transferProcessStore, - typeTransformerRegistry + typeTransformerRegistry, + placeholderEndpointService ); wrapperExtensionContext.selfDescriptionService().validateSelfDescriptionConfig(); diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java index d554dd74f..334fc8b0d 100644 --- a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java @@ -80,6 +80,7 @@ import de.sovity.edc.extension.contacttermination.query.TerminateContractQuery; import de.sovity.edc.extension.db.directaccess.DslContextFactory; import de.sovity.edc.extension.messenger.SovityMessenger; +import de.sovity.edc.extension.placeholderdatasource.PlaceholderEndpointService; import de.sovity.edc.utils.catalog.DspCatalogService; import de.sovity.edc.utils.catalog.mapper.DspDataOfferBuilder; import lombok.NoArgsConstructor; @@ -138,7 +139,8 @@ public static WrapperExtensionContext buildContext( SovityMessenger sovityMessenger, TransferProcessService transferProcessService, TransferProcessStore transferProcessStore, - TypeTransformerRegistry typeTransformerRegistry + TypeTransformerRegistry typeTransformerRegistry, + PlaceholderEndpointService placeholderEndpointService ) { // UI API var operatorMapper = new OperatorMapper(); @@ -148,7 +150,7 @@ public static WrapperExtensionContext buildContext( var edcPropertyUtils = new EdcPropertyUtils(); var selfDescriptionService = new SelfDescriptionService(config, monitor); var ownConnectorEndpointService = new OwnConnectorEndpointServiceImpl(selfDescriptionService); - var assetMapper = newAssetMapper(typeTransformerRegistry, jsonLd, ownConnectorEndpointService); + var assetMapper = newAssetMapper(typeTransformerRegistry, jsonLd, ownConnectorEndpointService, placeholderEndpointService); var policyMapper = newPolicyMapper(objectMapper, typeTransformerRegistry, operatorMapper); var transferProcessStateService = new TransferProcessStateService(); var contractNegotiationUtils = new ContractNegotiationUtils( @@ -306,7 +308,8 @@ public static WrapperExtensionContext buildContext( private static AssetMapper newAssetMapper( TypeTransformerRegistry typeTransformerRegistry, JsonLd jsonLd, - OwnConnectorEndpointService ownConnectorEndpointService + OwnConnectorEndpointService ownConnectorEndpointService, + PlaceholderEndpointService placeholderEndpointService ) { var edcPropertyUtils = new EdcPropertyUtils(); var assetJsonLdUtils = new AssetJsonLdUtils(); @@ -318,7 +321,7 @@ private static AssetMapper newAssetMapper( ownConnectorEndpointService ); var httpHeaderMapper = new HttpHeaderMapper(); - var httpDataSourceMapper = new HttpDataSourceMapper(httpHeaderMapper); + var httpDataSourceMapper = new HttpDataSourceMapper(httpHeaderMapper, placeholderEndpointService); var dataSourceMapper = new DataSourceMapper( edcPropertyUtils, httpDataSourceMapper diff --git a/settings.gradle.kts b/settings.gradle.kts index 78a0903b2..6e5ae061e 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -6,6 +6,7 @@ include(":extensions:catalog-crawler:catalog-crawler-launcher-base") include(":extensions:catalog-crawler:catalog-crawler-e2e-test") include(":extensions:contract-termination") include(":extensions:database-direct-access") +include(":extensions:placeholder-data-source") include(":extensions:edc-ui-config") include(":extensions:last-commit-info") include(":extensions:policy-always-true") diff --git a/tests/build.gradle.kts b/tests/build.gradle.kts index 8b232a5f1..e5e0d221b 100644 --- a/tests/build.gradle.kts +++ b/tests/build.gradle.kts @@ -19,6 +19,7 @@ dependencies { testImplementation(libs.junit.api) testImplementation(libs.junit.params) testImplementation(libs.mockserver.netty) + testImplementation(libs.okhttp.okhttp) testImplementation(libs.restAssured.restAssured) testRuntimeOnly(libs.junit.engine) } diff --git a/tests/src/test/java/de/sovity/edc/e2e/PlaceholderDataSourceTest.java b/tests/src/test/java/de/sovity/edc/e2e/PlaceholderDataSourceTest.java new file mode 100644 index 000000000..51b8059d7 --- /dev/null +++ b/tests/src/test/java/de/sovity/edc/e2e/PlaceholderDataSourceTest.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2024 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.e2e; + +import de.sovity.edc.client.EdcClient; +import de.sovity.edc.client.gen.model.DataSourceType; +import de.sovity.edc.client.gen.model.UiAssetCreateRequest; +import de.sovity.edc.client.gen.model.UiDataSource; +import de.sovity.edc.client.gen.model.UiDataSourceOnRequest; +import de.sovity.edc.extension.placeholderdatasource.PlaceholderDataSourceExtension; +import de.sovity.edc.extension.placeholderdatasource.PlaceholderEndpointService; +import de.sovity.edc.extension.e2e.connector.config.ConnectorConfig; +import de.sovity.edc.extension.e2e.extension.E2eTestExtension; +import de.sovity.edc.extension.e2e.extension.Provider; +import de.sovity.edc.utils.JsonUtils; +import de.sovity.edc.utils.jsonld.JsonLdUtils; +import de.sovity.edc.utils.jsonld.vocab.Prop; +import lombok.SneakyThrows; +import lombok.val; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import org.eclipse.edc.junit.extensions.EdcExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import static org.assertj.core.api.Assertions.assertThat; + +@ExtendWith(E2eTestExtension.class) +public class PlaceholderDataSourceTest { + + @SneakyThrows + @Test + void shouldAccessDummyEndpoint( + @Provider EdcExtension providerExtension, + @Provider ConnectorConfig providerConfig, + @Provider EdcClient providerClient + ) { + // arrange + + val assetId = "asset-1"; + val email = "contact@example.com"; + val subject = "Data request"; + providerClient.uiApi().createAsset( + UiAssetCreateRequest.builder() + .dataSource(UiDataSource.builder() + .type(DataSourceType.ON_REQUEST) + .onRequest(UiDataSourceOnRequest.builder() + .contactEmail(email) + .contactPreferredEmailSubject(subject) + .build()) + .build()) + .id(assetId) + .build() + ); + + val a = providerClient.uiApi().getAssetPage().getAssets().get(0).getAssetJsonLd(); + val compacted = JsonLdUtils.tryCompact(JsonUtils.parseJsonObj(a)); + val dataAddress = compacted.getJsonObject(Prop.Edc.DATA_ADDRESS); + val baseUrl = dataAddress.getString(Prop.Edc.BASE_URL); + + val service = providerExtension.getContext().getService(PlaceholderEndpointService.class); + val expected = service.getPlaceholderEndpointForAsset(email, subject); + + // assert + assertThat(baseUrl) + .startsWith(providerConfig.getProperties().get(PlaceholderDataSourceExtension.MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL)) + .isEqualTo(expected); + + val request = new Request.Builder() + .url(baseUrl) + .build(); + + val client = new OkHttpClient(); + val content = client.newCall(request).execute().body().string(); + + assertThat(content).contains("This is not a real data offer."); + assertThat(content).contains(email); + assertThat(content).contains(subject); + } +} diff --git a/utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/connector/config/ConnectorConfigFactory.java b/utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/connector/config/ConnectorConfigFactory.java index 4608b9396..22393d49b 100644 --- a/utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/connector/config/ConnectorConfigFactory.java +++ b/utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/connector/config/ConnectorConfigFactory.java @@ -85,6 +85,7 @@ public static ConnectorConfig basicEdcConfig(String participantId, int firstPort properties.put("edc.last.commit.info", "test env commit message"); properties.put("edc.build.date", "2023-05-08T15:30:00Z"); + // TODO: rm properties.put("edc.server.db.connection.timeout.in.ms", "5000"); properties.put("my.edc.participant.id", participantId); @@ -95,8 +96,12 @@ public static ConnectorConfig basicEdcConfig(String participantId, int firstPort properties.put("my.edc.maintainer.url", "http://maintainer.%s".formatted(participantId)); properties.put("my.edc.maintainer.name", "Maintainer Name %s".formatted(participantId)); + properties.put("my.edc.datasource.placeholder.baseurl", apiConfig.getProtocolApiGroup().getUri().toString()); + + // TODO: rm properties.put("edc.server.db.connection.pool.size", "3"); + // TODO: rm? properties.put("web.http.port", String.valueOf(apiConfig.getDefaultApiGroup().port())); properties.put("web.http.path", String.valueOf(apiConfig.getDefaultApiGroup().path())); properties.put("web.http.protocol.port", String.valueOf(apiConfig.getProtocolApiGroup().port())); diff --git a/utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/db/EdcRuntimeExtensionWithTestDatabase.java b/utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/db/EdcRuntimeExtensionWithTestDatabase.java index 868c3d89d..eab4e8eea 100644 --- a/utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/db/EdcRuntimeExtensionWithTestDatabase.java +++ b/utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/db/EdcRuntimeExtensionWithTestDatabase.java @@ -4,6 +4,7 @@ import lombok.RequiredArgsConstructor; import lombok.experimental.Delegate; import lombok.val; +import org.eclipse.edc.junit.extensions.EdcExtension; import org.jooq.DSLContext; import org.jooq.impl.DSL; import org.junit.jupiter.api.extension.AfterAllCallback; @@ -47,16 +48,28 @@ public void beforeAll(ExtensionContext extensionContext) throws Exception { @Override public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { - boolean isJooqDsl = parameterContext.getParameter().getType().equals(DSLContext.class); - return isJooqDsl || edcRuntimeExtension.supportsParameter(parameterContext, extensionContext); + + val type = parameterContext.getParameter().getType(); + + if (DSLContext.class.equals(type)) { + return true; + } else if (EdcExtension.class.equals(type)) { + return true; + } + + return edcRuntimeExtension.supportsParameter(parameterContext, extensionContext); } @Override public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { - boolean isJooqDsl = parameterContext.getParameter().getType().equals(DSLContext.class); - if (isJooqDsl) { + + val type = parameterContext.getParameter().getType(); + + if (DSLContext.class.equals(type)) { return getDslContext().dsl(); + } else if (EdcExtension.class.equals(type)) { + return edcRuntimeExtension; } else { return edcRuntimeExtension.resolveParameter(parameterContext, extensionContext); } diff --git a/utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/extension/E2eScenario.java b/utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/extension/E2eScenario.java index 346e256e6..ce2a4fe54 100644 --- a/utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/extension/E2eScenario.java +++ b/utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/extension/E2eScenario.java @@ -43,6 +43,7 @@ import lombok.val; import org.awaitility.Awaitility; import org.eclipse.edc.connector.contract.spi.types.negotiation.ContractNegotiation; +import org.jetbrains.annotations.NotNull; import org.mockserver.integration.ClientAndServer; import org.mockserver.model.HttpRequest; import org.mockserver.model.HttpResponse; @@ -96,7 +97,11 @@ public String createAsset() { .build()) .build(); - return internalCreateAsset("asset-" + assetCounter.getAndIncrement(), dummyDataSource).getId(); + return internalCreateAsset(nextAssetId(), dummyDataSource).getId(); + } + + private @NotNull String nextAssetId() { + return "asset-" + assetCounter.getAndIncrement(); } public String createAsset(String id, UiDataSourceHttpData uiDataSourceHttpData) { @@ -108,6 +113,10 @@ public String createAsset(String id, UiDataSourceHttpData uiDataSourceHttpData) return internalCreateAsset(id, uiDataSource).getId(); } + public String createAsset(String id, UiDataSource uiDataSource) { + return internalCreateAsset(id, uiDataSource).getId(); + } + public MockedAsset createAssetWithMockResource(String id) { val path = "/assets/" + id; From a9c9539b7cdffb6f2aab92d6606fa0a29dbe1df3 Mon Sep 17 00:00:00 2001 From: Christophe Loiseau Date: Mon, 22 Jul 2024 13:43:55 +0200 Subject: [PATCH 02/24] Self review --- CHANGELOG.md | 3 ++ .../catalog-crawler/build.gradle.kts | 1 + extensions/placeholder-data-source/README.md | 10 +++---- .../placeholder-data-source/build.gradle.kts | 10 +++++-- .../PlaceholderEndpointController.java | 6 ++-- .../PlaceholderEndpointService.java | 1 - .../PlaceholderDataSourceExtensionTest.java | 8 ++--- .../ui/model/ContractAgreementPageQuery.java | 1 - .../wrapper/api/usecase/UseCaseResource.java | 1 - .../wrapper-common-mappers/build.gradle.kts | 2 +- .../api/common/mappers/AssetMapper.java | 2 -- .../mappers/asset/AssetEditRequestMapper.java | 9 ++---- .../mappers/asset/AssetJsonLdBuilder.java | 29 ++----------------- .../mappers/dataaddress/DataSourceMapper.java | 2 -- .../http/HttpDataSourceMapper.java | 2 -- extensions/wrapper/wrapper/build.gradle.kts | 1 + .../edc/ext/wrapper/WrapperExtension.java | 2 +- .../services/ContractAgreementData.java | 1 - .../ContractAgreementPageCardBuilder.java | 2 -- utils/test-utils/build.gradle.kts | 1 + .../config/ConnectorConfigFactory.java | 7 ----- 21 files changed, 32 insertions(+), 69 deletions(-) rename tests/src/test/java/de/sovity/edc/e2e/PlaceholderDataSourceTest.java => extensions/placeholder-data-source/src/test/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderDataSourceExtensionTest.java (91%) diff --git a/CHANGELOG.md b/CHANGELOG.md index de3b23884..8717d21ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,9 @@ New configuration to access the database: - `EDC_SERVER_DB_CONNECTION_POOL_SIZE` - The property controls the maximum size that the pool is allowed to reach, including both idle and in-use connections. Basically this value will determine the maximum number of actual connections to the database backend. - Defaults to `3` +- `MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL` + - The property on which URL an asset is seen from an external source (internet, VPN, ...) + - It is documented in the asset data placeholder [README.md](extensions/placeholder-data-source/README.md) #### Compatible Versions diff --git a/extensions/catalog-crawler/catalog-crawler/build.gradle.kts b/extensions/catalog-crawler/catalog-crawler/build.gradle.kts index 7c2b46f7e..9f4f98b95 100644 --- a/extensions/catalog-crawler/catalog-crawler/build.gradle.kts +++ b/extensions/catalog-crawler/catalog-crawler/build.gradle.kts @@ -17,6 +17,7 @@ dependencies { api(project(":utils:json-and-jsonld-utils")) api(project(":extensions:wrapper:wrapper-common-mappers")) api(project(":extensions:catalog-crawler:catalog-crawler-db")) + api(project(":extensions:placeholder-data-source")) api(project(":extensions:postgres-flyway-core")) testAnnotationProcessor(libs.lombok) diff --git a/extensions/placeholder-data-source/README.md b/extensions/placeholder-data-source/README.md index 7dc593b48..d398989f0 100644 --- a/extensions/placeholder-data-source/README.md +++ b/extensions/placeholder-data-source/README.md @@ -10,7 +10,7 @@

Report Bug · - Request Feature + Asset Data Source Placeholder

@@ -35,7 +35,7 @@ On a production system, the base URL it could be: with a placeholder value: -`https://mycompany.com/path/to/backend/data-source/placeholder/asset/` +`https://mycompany.com/path/to/backend/data-source/placeholder/asset` and a full path to the asset's data: @@ -47,7 +47,7 @@ On a system started with docker-compose, it will be pointing to the DSP port on `http://edc:11003/` -`http://edc:11003/data-source/placeholder/asset/` +`http://edc:11003/data-source/placeholder/asset` `http://edc:11003/data-source/placeholder/asset?email=foo%40example.com&subject=Contact+us+now` @@ -57,9 +57,9 @@ During local/dev/unit test execution, it will be pointing to the DSP port on the `http://localhost:12345/` -where `12345` would be chosen at random +where the port that is here `12345` would actually be chosen at random -`http://localhost:12345/data-source/placeholder/asset/` +`http://localhost:12345/data-source/placeholder/asset` `http://localhost:12345/data-source/placeholder/asset?email=foo%40example.com&subject=Contact+us+now` diff --git a/extensions/placeholder-data-source/build.gradle.kts b/extensions/placeholder-data-source/build.gradle.kts index eefea45bd..584971a28 100644 --- a/extensions/placeholder-data-source/build.gradle.kts +++ b/extensions/placeholder-data-source/build.gradle.kts @@ -8,13 +8,17 @@ dependencies { annotationProcessor(libs.lombok) compileOnly(libs.lombok) - api(libs.edc.coreSpi) - api(libs.edc.controlPlaneSpi) + implementation(libs.edc.coreSpi) implementation(libs.edc.apiCore) implementation(libs.edc.dspApiConfiguration) implementation(libs.okhttp.okhttp) - testImplementation(libs.mockito.core) + + testAnnotationProcessor(libs.lombok) + testCompileOnly(libs.lombok) + + testImplementation(project(":utils:test-utils")) + testImplementation(libs.assertj.core) testImplementation(libs.junit.api) testRuntimeOnly(libs.junit.engine) } diff --git a/extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderEndpointController.java b/extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderEndpointController.java index d0dcf71ab..41f4b44a3 100644 --- a/extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderEndpointController.java +++ b/extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderEndpointController.java @@ -34,9 +34,11 @@ public class PlaceholderEndpointController { @Consumes("*/*") public Response get(@QueryParam("email") String email, @QueryParam("subject") String subject) { return Response.ok(""" - This is not a real data offer. + This is not real data. - The offer you are trying to use only has this placeholder as a dummy endpoint and requires you top take extra actions to access it. + This asset is accessible on request. + + The offer you are trying to use only has this placeholder as a dummy endpoint and requires you to take extra steps to access it. Please contact the data provider for more information about how to access it. """ + "\n\n" + "Email: " + email + "\n" + "Subject: " + subject + "\n" diff --git a/extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderEndpointService.java b/extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderEndpointService.java index 36b59f272..515f2cf4d 100644 --- a/extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderEndpointService.java +++ b/extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderEndpointService.java @@ -24,7 +24,6 @@ public class PlaceholderEndpointService { public static final String DUMMY_ENDPOINT_URL = "/data-source/placeholder/asset"; - // TODO: maybe add String contactEmail, String contactPreferredEmailSubject and show them in the placeholder public String getPlaceholderEndpointForAsset(String email, String subject) { return HttpUrl.parse(baseUrl + DUMMY_ENDPOINT_URL) .newBuilder() diff --git a/tests/src/test/java/de/sovity/edc/e2e/PlaceholderDataSourceTest.java b/extensions/placeholder-data-source/src/test/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderDataSourceExtensionTest.java similarity index 91% rename from tests/src/test/java/de/sovity/edc/e2e/PlaceholderDataSourceTest.java rename to extensions/placeholder-data-source/src/test/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderDataSourceExtensionTest.java index 51b8059d7..5d739edf1 100644 --- a/tests/src/test/java/de/sovity/edc/e2e/PlaceholderDataSourceTest.java +++ b/extensions/placeholder-data-source/src/test/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderDataSourceExtensionTest.java @@ -12,15 +12,13 @@ * */ -package de.sovity.edc.e2e; +package de.sovity.edc.extension.placeholderdatasource; import de.sovity.edc.client.EdcClient; import de.sovity.edc.client.gen.model.DataSourceType; import de.sovity.edc.client.gen.model.UiAssetCreateRequest; import de.sovity.edc.client.gen.model.UiDataSource; import de.sovity.edc.client.gen.model.UiDataSourceOnRequest; -import de.sovity.edc.extension.placeholderdatasource.PlaceholderDataSourceExtension; -import de.sovity.edc.extension.placeholderdatasource.PlaceholderEndpointService; import de.sovity.edc.extension.e2e.connector.config.ConnectorConfig; import de.sovity.edc.extension.e2e.extension.E2eTestExtension; import de.sovity.edc.extension.e2e.extension.Provider; @@ -38,7 +36,7 @@ import static org.assertj.core.api.Assertions.assertThat; @ExtendWith(E2eTestExtension.class) -public class PlaceholderDataSourceTest { +class PlaceholderDataSourceExtensionTest { @SneakyThrows @Test @@ -85,7 +83,7 @@ void shouldAccessDummyEndpoint( val client = new OkHttpClient(); val content = client.newCall(request).execute().body().string(); - assertThat(content).contains("This is not a real data offer."); + assertThat(content).contains("This is not real data."); assertThat(content).contains(email); assertThat(content).contains(subject); } diff --git a/extensions/wrapper/wrapper-api/src/main/java/de/sovity/edc/ext/wrapper/api/ui/model/ContractAgreementPageQuery.java b/extensions/wrapper/wrapper-api/src/main/java/de/sovity/edc/ext/wrapper/api/ui/model/ContractAgreementPageQuery.java index f836e5399..d9ef4a47e 100644 --- a/extensions/wrapper/wrapper-api/src/main/java/de/sovity/edc/ext/wrapper/api/ui/model/ContractAgreementPageQuery.java +++ b/extensions/wrapper/wrapper-api/src/main/java/de/sovity/edc/ext/wrapper/api/ui/model/ContractAgreementPageQuery.java @@ -14,7 +14,6 @@ package de.sovity.edc.ext.wrapper.api.ui.model; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import io.swagger.v3.oas.annotations.media.Schema; import lombok.AllArgsConstructor; diff --git a/extensions/wrapper/wrapper-api/src/main/java/de/sovity/edc/ext/wrapper/api/usecase/UseCaseResource.java b/extensions/wrapper/wrapper-api/src/main/java/de/sovity/edc/ext/wrapper/api/usecase/UseCaseResource.java index b959ac0ec..e02e50e9c 100644 --- a/extensions/wrapper/wrapper-api/src/main/java/de/sovity/edc/ext/wrapper/api/usecase/UseCaseResource.java +++ b/extensions/wrapper/wrapper-api/src/main/java/de/sovity/edc/ext/wrapper/api/usecase/UseCaseResource.java @@ -19,7 +19,6 @@ import de.sovity.edc.ext.wrapper.api.usecase.model.KpiResult; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; -import jakarta.validation.Valid; import jakarta.validation.constraints.NotNull; import jakarta.ws.rs.Consumes; import jakarta.ws.rs.GET; diff --git a/extensions/wrapper/wrapper-common-mappers/build.gradle.kts b/extensions/wrapper/wrapper-common-mappers/build.gradle.kts index 835e6475c..94eafc23b 100644 --- a/extensions/wrapper/wrapper-common-mappers/build.gradle.kts +++ b/extensions/wrapper/wrapper-common-mappers/build.gradle.kts @@ -12,7 +12,7 @@ dependencies { api(libs.edc.coreSpi) api(libs.edc.transformCore) api(libs.edc.transformSpi) - api(project(":extensions:placeholder-data-source")) + implementation(project(":extensions:placeholder-data-source")) api(project(":extensions:wrapper:wrapper-common-api")) api(project(":utils:json-and-jsonld-utils")) implementation(libs.apache.commonsLang) diff --git a/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/AssetMapper.java b/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/AssetMapper.java index 52eb6cdac..4ac252075 100644 --- a/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/AssetMapper.java +++ b/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/AssetMapper.java @@ -53,8 +53,6 @@ public Asset editAsset( @NonNull UiAssetEditRequest editRequest ) { var assetJsonLd = buildAssetJsonLd(asset); - // edit request is all empty - // assetJsonLd contains the manually set baseUrl var editedJsonLd = assetJsonLdBuilder.editAssetJsonLd(assetJsonLd, editRequest); return buildAsset(editedJsonLd); } diff --git a/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/asset/AssetEditRequestMapper.java b/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/asset/AssetEditRequestMapper.java index 8a949cf7f..c65fe5243 100644 --- a/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/asset/AssetEditRequestMapper.java +++ b/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/asset/AssetEditRequestMapper.java @@ -32,10 +32,7 @@ public class AssetEditRequestMapper { * @param editRequest {@link UiAssetEditRequest} * @return {@link UiAssetCreateRequest} */ - public UiAssetCreateRequest buildCreateRequest( - @NonNull UiAssetEditRequest editRequest, - @NonNull String assetId - ) { + public UiAssetCreateRequest buildCreateRequest(@NonNull UiAssetEditRequest editRequest, @NonNull String assetId) { var dataSource = editRequest.getDataSourceOverrideOrNull(); if (dataSource == null) { dataSource = dummyDataSource(); @@ -76,8 +73,6 @@ public UiAssetCreateRequest buildCreateRequest( } private UiDataSource dummyDataSource() { - return UiDataSource.builder() - .type(DataSourceType.CUSTOM) - .build(); + return UiDataSource.builder().type(DataSourceType.CUSTOM).build(); } } diff --git a/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/asset/AssetJsonLdBuilder.java b/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/asset/AssetJsonLdBuilder.java index ca5db7c59..e8c7cd00b 100644 --- a/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/asset/AssetJsonLdBuilder.java +++ b/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/asset/AssetJsonLdBuilder.java @@ -63,41 +63,20 @@ public JsonObject createAssetJsonLd( ); } + @SneakyThrows @Nullable public JsonObject editAssetJsonLd( - JsonObject assetJsonLd, // 3 - UiAssetEditRequest editRequest // 2 + JsonObject assetJsonLd, + UiAssetEditRequest editRequest ) { - - /* - * Data source selection order: - * override? - * edit request - * jsonLd - * placeholder - */ - - // request base URL comes here var dataAddress = getDataAddressJsonLd(assetJsonLd); - // override is null if (editRequest.getDataSourceOverrideOrNull() != null) { - // this part of the code is not asserted - // what is this override used for? - // TODO should this part have higher or lower prio than the automatic assignment to the dummy data source? - // 1 - // there is another handling of this field in de.sovity.edc.ext.wrapper.api.common.mappers.asset.AssetEditRequestMapper.buildCreateRequest - // TODO: this is not a clear name on the API side: is `orNull` the Java null of the JSON null? dataAddress = dataSourceMapper.buildDataSourceJsonLd(editRequest.getDataSourceOverrideOrNull()); } - // the dataAddress that is used should be the one that's part of the createRequest, not a separate element - var assetId = Objects.requireNonNull(JsonLdUtils.string(assetJsonLd, Prop.ID), "Asset JSON-LD had no @id"); var organizationName = assetJsonLdParser.getCreatorOrganizationName(assetJsonLd); - // set dummy source to placeholder URL inside this createRequest - // this `createRequest` contains the correct dataSource - // 4: dummy var createRequest = assetEditRequestMapper.buildCreateRequest(editRequest, assetId); var properties = getAssetProperties(createRequest, dataAddress, organizationName); var privateProperties = getAssetPrivateProperties(createRequest); @@ -154,8 +133,6 @@ private JsonObject getAssetProperties( addMobilityTheme(properties, request); addCustomJsonLd(properties, request); -// request.getDataSource(). - // TODO: replace with request.getDataSource() addDataSourceHints(properties, dataAddressJsonLd); return properties.build(); } diff --git a/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/dataaddress/DataSourceMapper.java b/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/dataaddress/DataSourceMapper.java index 0dc5280bd..b8ccae7ac 100644 --- a/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/dataaddress/DataSourceMapper.java +++ b/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/dataaddress/DataSourceMapper.java @@ -29,7 +29,6 @@ import lombok.RequiredArgsConstructor; import java.util.Map; -import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Supplier; @@ -42,7 +41,6 @@ public class DataSourceMapper { private final HttpDataSourceMapper httpDataSourceMapper; public JsonObject buildDataSourceJsonLd(@NonNull UiDataSource dataSource) { - // this datasource has the URL set in the creation request var props = this.matchDataSource( dataSource, httpDataSourceMapper::buildDataAddress, diff --git a/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/dataaddress/http/HttpDataSourceMapper.java b/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/dataaddress/http/HttpDataSourceMapper.java index 5cb085ed7..8d06de0cb 100644 --- a/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/dataaddress/http/HttpDataSourceMapper.java +++ b/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/dataaddress/http/HttpDataSourceMapper.java @@ -94,7 +94,6 @@ public Map buildOnRequestDataAddress(@NonNull UiDataSourceOnRequ "Need contactPreferredEmailSubject" ); - // this seems to be in use String placeholderEndpointForAsset = placeholderEndpointService.getPlaceholderEndpointForAsset( onRequest.getContactEmail(), onRequest.getContactPreferredEmailSubject()); @@ -130,7 +129,6 @@ public JsonObject enhanceAssetWithDataSourceHints(Map dataAddres Prop.Edc.PROXY_BODY, Prop.SovityDcatExt.HttpDatasourceHints.BODY ).forEach((prop, hint) -> // Will add hints as "true" or "false" - // TODO: what are those hints for? External info about the content of those fields?? json.add(hint, String.valueOf("true".equals(dataAddress.get(prop)))) ); } diff --git a/extensions/wrapper/wrapper/build.gradle.kts b/extensions/wrapper/wrapper/build.gradle.kts index 2a080c73b..1fe22d807 100644 --- a/extensions/wrapper/wrapper/build.gradle.kts +++ b/extensions/wrapper/wrapper/build.gradle.kts @@ -21,6 +21,7 @@ dependencies { implementation(project(":extensions:contract-termination")) implementation(project(":extensions:database-direct-access")) + implementation(project(":extensions:placeholder-data-source")) implementation(project(":extensions:sovity-messenger")) implementation(project(":utils:jooq-database-access")) diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtension.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtension.java index df6cd42bb..5ea98e08a 100644 --- a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtension.java +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtension.java @@ -19,8 +19,8 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import de.sovity.edc.extension.contacttermination.ContractAgreementTerminationService; import de.sovity.edc.extension.db.directaccess.DslContextFactory; -import de.sovity.edc.extension.placeholderdatasource.PlaceholderEndpointService; import de.sovity.edc.extension.messenger.SovityMessenger; +import de.sovity.edc.extension.placeholderdatasource.PlaceholderEndpointService; import org.eclipse.edc.connector.api.management.configuration.ManagementApiConfiguration; import org.eclipse.edc.connector.api.management.configuration.transform.ManagementApiTypeTransformerRegistry; import org.eclipse.edc.connector.contract.spi.negotiation.store.ContractNegotiationStore; diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/contract_agreements/services/ContractAgreementData.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/contract_agreements/services/ContractAgreementData.java index 6e3411762..47801afaf 100644 --- a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/contract_agreements/services/ContractAgreementData.java +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/contract_agreements/services/ContractAgreementData.java @@ -21,7 +21,6 @@ import org.eclipse.edc.spi.types.domain.asset.Asset; import java.util.List; -import java.util.Map; /** diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/contract_agreements/services/ContractAgreementPageCardBuilder.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/contract_agreements/services/ContractAgreementPageCardBuilder.java index 7e35f1a3d..87ba240d3 100644 --- a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/contract_agreements/services/ContractAgreementPageCardBuilder.java +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/contract_agreements/services/ContractAgreementPageCardBuilder.java @@ -23,7 +23,6 @@ import de.sovity.edc.ext.wrapper.api.ui.model.ContractAgreementTransferProcess; import de.sovity.edc.ext.wrapper.api.ui.model.ContractTerminatedBy; import de.sovity.edc.ext.wrapper.api.ui.pages.transferhistory.TransferProcessStateService; -import jakarta.validation.constraints.Null; import lombok.NonNull; import lombok.RequiredArgsConstructor; import org.eclipse.edc.connector.contract.spi.types.agreement.ContractAgreement; @@ -34,7 +33,6 @@ import java.util.Comparator; import java.util.List; -import java.util.Map; import static de.sovity.edc.ext.wrapper.api.ui.model.ContractTerminationStatus.ONGOING; import static de.sovity.edc.ext.wrapper.api.ui.model.ContractTerminationStatus.TERMINATED; diff --git a/utils/test-utils/build.gradle.kts b/utils/test-utils/build.gradle.kts index e2d6e9038..b6b1eab88 100644 --- a/utils/test-utils/build.gradle.kts +++ b/utils/test-utils/build.gradle.kts @@ -12,6 +12,7 @@ dependencies { api(libs.edc.junit) api(libs.awaitility.java) + api(libs.postgres) api(project(":extensions:wrapper:clients:java-client")) api(project(":utils:json-and-jsonld-utils")) diff --git a/utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/connector/config/ConnectorConfigFactory.java b/utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/connector/config/ConnectorConfigFactory.java index 22393d49b..b2fab5ea9 100644 --- a/utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/connector/config/ConnectorConfigFactory.java +++ b/utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/connector/config/ConnectorConfigFactory.java @@ -85,9 +85,6 @@ public static ConnectorConfig basicEdcConfig(String participantId, int firstPort properties.put("edc.last.commit.info", "test env commit message"); properties.put("edc.build.date", "2023-05-08T15:30:00Z"); - // TODO: rm - properties.put("edc.server.db.connection.timeout.in.ms", "5000"); - properties.put("my.edc.participant.id", participantId); properties.put("my.edc.title", "Connector Title %s".formatted(participantId)); properties.put("my.edc.description", "Connector Description %s".formatted(participantId)); @@ -98,10 +95,6 @@ public static ConnectorConfig basicEdcConfig(String participantId, int firstPort properties.put("my.edc.datasource.placeholder.baseurl", apiConfig.getProtocolApiGroup().getUri().toString()); - // TODO: rm - properties.put("edc.server.db.connection.pool.size", "3"); - - // TODO: rm? properties.put("web.http.port", String.valueOf(apiConfig.getDefaultApiGroup().port())); properties.put("web.http.path", String.valueOf(apiConfig.getDefaultApiGroup().path())); properties.put("web.http.protocol.port", String.valueOf(apiConfig.getProtocolApiGroup().port())); From a7c9d3ddd82a365b682e2ffcc718de127272db7a Mon Sep 17 00:00:00 2001 From: Christophe Loiseau Date: Mon, 22 Jul 2024 15:18:00 +0200 Subject: [PATCH 03/24] Merge main cleanup --- CHANGELOG.md | 2 +- .../crawler/CrawlerExtensionContextBuilder.java | 16 ++++++++-------- extensions/placeholder-data-source/README.md | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8717d21ae..43c9cd382 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ New configuration to access the database: - `EDC_SERVER_DB_CONNECTION_POOL_SIZE` - The property controls the maximum size that the pool is allowed to reach, including both idle and in-use connections. Basically this value will determine the maximum number of actual connections to the database backend. - Defaults to `3` -- `MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL` +- `MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL` - The property on which URL an asset is seen from an external source (internet, VPN, ...) - It is documented in the asset data placeholder [README.md](extensions/placeholder-data-source/README.md) diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java index cbdc1252e..cec917df7 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java @@ -97,13 +97,13 @@ public class CrawlerExtensionContextBuilder { public static CrawlerExtensionContext buildContext( - Config config, - Monitor monitor, - TypeManager typeManager, - TypeTransformerRegistry typeTransformerRegistry, - JsonLd jsonLd, - CatalogService catalogService, - PlaceholderEndpointService placeholderEndpointService + Config config, + Monitor monitor, + TypeManager typeManager, + TypeTransformerRegistry typeTransformerRegistry, + JsonLd jsonLd, + CatalogService catalogService, + PlaceholderEndpointService placeholderEndpointService ) { // Config var crawlerConfigFactory = new CrawlerConfigFactory(config); @@ -232,7 +232,7 @@ private static PolicyMapper newPolicyMapper( private static AssetMapper newAssetMapper( TypeTransformerRegistry typeTransformerRegistry, JsonLd jsonLd, - PlaceholderEndpointService placeholderEndpointService + PlaceholderEndpointService placeholderEndpointService ) { var edcPropertyUtils = new EdcPropertyUtils(); var assetJsonLdUtils = new AssetJsonLdUtils(); diff --git a/extensions/placeholder-data-source/README.md b/extensions/placeholder-data-source/README.md index d398989f0..d0e161701 100644 --- a/extensions/placeholder-data-source/README.md +++ b/extensions/placeholder-data-source/README.md @@ -29,13 +29,13 @@ This extension exists to inform the asset's consumer upon data retrieval that th --- -On a production system, the base URL it could be: +On a production system, the base URL it could be: -`https://mycompany.com/path/to/backend` +`https://mycompany.com/path/to/backend` with a placeholder value: -`https://mycompany.com/path/to/backend/data-source/placeholder/asset` +`https://mycompany.com/path/to/backend/data-source/placeholder/asset` and a full path to the asset's data: From 765db0b5b606a17654ccc218b9805b2a67663cbd Mon Sep 17 00:00:00 2001 From: Christophe Loiseau Date: Mon, 22 Jul 2024 15:22:25 +0200 Subject: [PATCH 04/24] reword --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43c9cd382..c32841efc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,7 +30,7 @@ New configuration to access the database: - The property controls the maximum size that the pool is allowed to reach, including both idle and in-use connections. Basically this value will determine the maximum number of actual connections to the database backend. - Defaults to `3` - `MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL` - - The property on which URL an asset is seen from an external source (internet, VPN, ...) + - Base URL of an asset as seen from an external source, from the asset's consumer (typically from the internet, ...) - It is documented in the asset data placeholder [README.md](extensions/placeholder-data-source/README.md) #### Compatible Versions From ba2fae5618d2640cf1a8a44a99aab1c916150fad Mon Sep 17 00:00:00 2001 From: Christophe Loiseau Date: Mon, 22 Jul 2024 16:01:45 +0200 Subject: [PATCH 05/24] Disable test for GH --- .../src/test/java/de/sovity/edc/e2e/ContractTerminationTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/src/test/java/de/sovity/edc/e2e/ContractTerminationTest.java b/tests/src/test/java/de/sovity/edc/e2e/ContractTerminationTest.java index a20f44b0c..010297aaa 100644 --- a/tests/src/test/java/de/sovity/edc/e2e/ContractTerminationTest.java +++ b/tests/src/test/java/de/sovity/edc/e2e/ContractTerminationTest.java @@ -137,6 +137,7 @@ void canGetAgreementPageForTerminatedContract( assertTermination(agreementsAfterTermination, details, reason, SELF); } + @DisabledOnGithub @Test @SneakyThrows void canTerminateFromConsumer( From 5319c9e196aede7d344f72ab3701e520c5058e67 Mon Sep 17 00:00:00 2001 From: Christophe Loiseau Date: Mon, 22 Jul 2024 16:20:07 +0200 Subject: [PATCH 06/24] Cleanup and disabling more tests on github --- .../query/TerminateContractQueryTest.java | 2 ++ .../edc/e2e/ContractTerminationTest.java | 1 + .../extension/e2e/extension/E2eScenario.java | 26 +------------------ 3 files changed, 4 insertions(+), 25 deletions(-) diff --git a/extensions/contract-termination/src/test/java/de/sovity/edc/extension/contacttermination/query/TerminateContractQueryTest.java b/extensions/contract-termination/src/test/java/de/sovity/edc/extension/contacttermination/query/TerminateContractQueryTest.java index c4e5dc825..1308eaf1c 100644 --- a/extensions/contract-termination/src/test/java/de/sovity/edc/extension/contacttermination/query/TerminateContractQueryTest.java +++ b/extensions/contract-termination/src/test/java/de/sovity/edc/extension/contacttermination/query/TerminateContractQueryTest.java @@ -21,6 +21,7 @@ import de.sovity.edc.extension.e2e.extension.E2eScenario; import de.sovity.edc.extension.e2e.extension.E2eTestExtension; import de.sovity.edc.extension.e2e.extension.Provider; +import de.sovity.edc.extension.utils.junit.DisabledOnGithub; import lombok.val; import org.eclipse.edc.connector.contract.spi.types.negotiation.ContractNegotiation; import org.junit.jupiter.api.Test; @@ -34,6 +35,7 @@ @ExtendWith(E2eTestExtension.class) class TerminateContractQueryTest { + @DisabledOnGithub @Test void terminateConsumerAgreementOrThrow_shouldInsertRowInTerminationTable( E2eScenario scenario, diff --git a/tests/src/test/java/de/sovity/edc/e2e/ContractTerminationTest.java b/tests/src/test/java/de/sovity/edc/e2e/ContractTerminationTest.java index 010297aaa..4aaa76c0c 100644 --- a/tests/src/test/java/de/sovity/edc/e2e/ContractTerminationTest.java +++ b/tests/src/test/java/de/sovity/edc/e2e/ContractTerminationTest.java @@ -292,6 +292,7 @@ List theDetailsAreMandatory( })).toList(); } + @DisabledOnGithub @Test @SneakyThrows void canTerminateFromProvider( diff --git a/utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/extension/E2eScenario.java b/utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/extension/E2eScenario.java index ce2a4fe54..a8560544e 100644 --- a/utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/extension/E2eScenario.java +++ b/utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/extension/E2eScenario.java @@ -50,7 +50,6 @@ import java.time.Duration; import java.time.OffsetDateTime; -import java.util.Arrays; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; @@ -150,31 +149,8 @@ private IdResponseDto internalCreateAsset(String assetId, UiDataSource dataSourc .build()); } - public String createPolicyDefinition(String policyId, UiPolicyConstraint... constraints) { - return createPolicyDefinition(policyId, Arrays.stream(constraints).toList()).getId(); - } - - private IdResponseDto createPolicyDefinition(String policyId, List constraints) { - var expression = UiPolicyExpression.builder() - .type(UiPolicyExpressionType.AND) - .expressions(constraints.stream() - .map(it -> UiPolicyExpression.builder() - .type(UiPolicyExpressionType.CONSTRAINT) - .constraint(it) - .build()) - .toList()) - .build(); - - var policyDefinition = PolicyDefinitionCreateDto.builder() - .policyDefinitionId(policyId) - .expression(expression) - .build(); - - return providerClient.uiApi().createPolicyDefinitionV2(policyDefinition); - } - public String createContractDefinition(String assetId) { - return createContractDefinition(POLICY_DEFINITION_ID, assetId).getId(); + return createContractDefinition(alwaysTruePolicyId, assetId).getId(); } public IdResponseDto createContractDefinition(String policyId, String assetId) { From 6fa77026f1ebd5071b0b639cc9ec5f72c97a41cf Mon Sep 17 00:00:00 2001 From: Christophe Loiseau Date: Mon, 22 Jul 2024 16:32:15 +0200 Subject: [PATCH 07/24] More tests disabling --- .../src/test/java/de/sovity/edc/e2e/ContractTerminationTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/src/test/java/de/sovity/edc/e2e/ContractTerminationTest.java b/tests/src/test/java/de/sovity/edc/e2e/ContractTerminationTest.java index 4aaa76c0c..7b39169a7 100644 --- a/tests/src/test/java/de/sovity/edc/e2e/ContractTerminationTest.java +++ b/tests/src/test/java/de/sovity/edc/e2e/ContractTerminationTest.java @@ -254,6 +254,7 @@ void limitTheDetailSizeAt1000Chars( // termination completed == success } + @DisabledOnGithub @TestFactory List theDetailsAreMandatory( E2eScenario scenario, From b4ff5792e3d6701a9d9586b8273811eec6f71aff Mon Sep 17 00:00:00 2001 From: Christophe Loiseau Date: Tue, 23 Jul 2024 08:54:28 +0200 Subject: [PATCH 08/24] checkstyle --- .../java/de/sovity/edc/ext/catalog/crawler/CrawlerE2eTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/catalog-crawler/catalog-crawler-e2e-test/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerE2eTest.java b/extensions/catalog-crawler/catalog-crawler-e2e-test/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerE2eTest.java index bf2b0153d..ad3fa91ec 100644 --- a/extensions/catalog-crawler/catalog-crawler-e2e-test/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerE2eTest.java +++ b/extensions/catalog-crawler/catalog-crawler-e2e-test/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerE2eTest.java @@ -36,9 +36,9 @@ import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorOnlineStatus; import de.sovity.edc.ext.catalog.crawler.utils.CrawlerDbAccess; import de.sovity.edc.ext.catalog.crawler.utils.TestData; -import de.sovity.edc.extension.placeholderdatasource.PlaceholderDataSourceExtension; import de.sovity.edc.extension.e2e.connector.config.ConnectorConfig; import de.sovity.edc.extension.e2e.db.EdcRuntimeExtensionWithTestDatabase; +import de.sovity.edc.extension.placeholderdatasource.PlaceholderDataSourceExtension; import de.sovity.edc.utils.jsonld.vocab.Prop; import org.awaitility.Awaitility; import org.awaitility.core.ThrowingRunnable; From 04deb457eb5f210dfe971eec1cf801a1b0957f63 Mon Sep 17 00:00:00 2001 From: Christophe Loiseau Date: Tue, 23 Jul 2024 09:41:03 +0200 Subject: [PATCH 09/24] Force non snapshot metamodel runtime --- extensions/edc-ui-config/build.gradle.kts | 1 + extensions/last-commit-info/build.gradle.kts | 1 + extensions/policy-always-true/build.gradle.kts | 1 + extensions/test-backend-controller/build.gradle.kts | 1 + extensions/wrapper/wrapper-common-mappers/build.gradle.kts | 6 +++++- extensions/wrapper/wrapper/build.gradle.kts | 1 + gradle/libs.versions.toml | 1 + utils/catalog-parser/build.gradle.kts | 2 ++ 8 files changed, 13 insertions(+), 1 deletion(-) diff --git a/extensions/edc-ui-config/build.gradle.kts b/extensions/edc-ui-config/build.gradle.kts index 1f193451f..a6a3cf2ed 100644 --- a/extensions/edc-ui-config/build.gradle.kts +++ b/extensions/edc-ui-config/build.gradle.kts @@ -9,6 +9,7 @@ dependencies { api(libs.edc.controlPlaneSpi) implementation(libs.edc.apiCore) implementation(libs.edc.managementApiConfiguration) + implementation(libs.edc.runtimeMetamodel) implementation(libs.jakarta.rsApi) implementation(libs.jakarta.validationApi) diff --git a/extensions/last-commit-info/build.gradle.kts b/extensions/last-commit-info/build.gradle.kts index eac954d07..f282295fb 100644 --- a/extensions/last-commit-info/build.gradle.kts +++ b/extensions/last-commit-info/build.gradle.kts @@ -12,6 +12,7 @@ dependencies { api(libs.edc.controlPlaneSpi) implementation(libs.edc.apiCore) implementation(libs.edc.managementApiConfiguration) + implementation(libs.edc.runtimeMetamodel) implementation(libs.jakarta.rsApi) implementation(libs.jakarta.validationApi) diff --git a/extensions/policy-always-true/build.gradle.kts b/extensions/policy-always-true/build.gradle.kts index dd96f4f1a..44267ab67 100644 --- a/extensions/policy-always-true/build.gradle.kts +++ b/extensions/policy-always-true/build.gradle.kts @@ -9,6 +9,7 @@ dependencies { api(libs.edc.policyEngineSpi) api(libs.edc.controlPlaneSpi) implementation(libs.edc.apiCore) + implementation(libs.edc.runtimeMetamodel) testImplementation(libs.edc.controlPlaneCore) testImplementation(libs.edc.junit) diff --git a/extensions/test-backend-controller/build.gradle.kts b/extensions/test-backend-controller/build.gradle.kts index 9e5364de4..7dc3ae9ff 100644 --- a/extensions/test-backend-controller/build.gradle.kts +++ b/extensions/test-backend-controller/build.gradle.kts @@ -6,6 +6,7 @@ dependencies { api(libs.edc.apiCore) api(libs.edc.coreSpi) api(libs.edc.http) + implementation(libs.edc.runtimeMetamodel) } group = libs.versions.sovityEdcExtensionGroup.get() diff --git a/extensions/wrapper/wrapper-common-mappers/build.gradle.kts b/extensions/wrapper/wrapper-common-mappers/build.gradle.kts index 94eafc23b..def04723e 100644 --- a/extensions/wrapper/wrapper-common-mappers/build.gradle.kts +++ b/extensions/wrapper/wrapper-common-mappers/build.gradle.kts @@ -12,13 +12,17 @@ dependencies { api(libs.edc.coreSpi) api(libs.edc.transformCore) api(libs.edc.transformSpi) - implementation(project(":extensions:placeholder-data-source")) + api(project(":extensions:wrapper:wrapper-common-api")) api(project(":utils:json-and-jsonld-utils")) + + implementation(project(":extensions:placeholder-data-source")) implementation(libs.apache.commonsLang) implementation(libs.apache.commonsCollections) + implementation(libs.edc.runtimeMetamodel) implementation(libs.flexmark.all) + testAnnotationProcessor(libs.lombok) testCompileOnly(libs.lombok) testImplementation(project(":utils:test-utils")) diff --git a/extensions/wrapper/wrapper/build.gradle.kts b/extensions/wrapper/wrapper/build.gradle.kts index 1fe22d807..31dac2f1a 100644 --- a/extensions/wrapper/wrapper/build.gradle.kts +++ b/extensions/wrapper/wrapper/build.gradle.kts @@ -29,6 +29,7 @@ dependencies { implementation(libs.edc.apiCore) implementation(libs.edc.managementApiConfiguration) implementation(libs.edc.dspHttpSpi) + implementation(libs.edc.runtimeMetamodel) implementation(libs.jooq.jooq) implementation(libs.hibernate.validation) implementation(libs.hikari) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index f32a5d574..748f8b6ec 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -117,6 +117,7 @@ edc-policyDefinitionApi = { module = "org.eclipse.edc:policy-definition-api", ve edc-policyEngineSpi = { module = "org.eclipse.edc:policy-engine-spi", version.ref = "edc" } edc-policyModel = { module = "org.eclipse.edc:policy-model", version.ref = "edc" } edc-policySpi = { module = "org.eclipse.edc:policy-spi", version.ref = "edc" } +edc-runtimeMetamodel = { module = "org.eclipse.edc:runtime-metamodel", version = "0.2.1" } edc-sqlCore = { module = "org.eclipse.edc:sql-core", version.ref = "edc" } edc-transactionLocal = { module = "org.eclipse.edc:transaction-local", version.ref = "edc" } edc-transferDataPlane = { module = "org.eclipse.edc:transfer-data-plane", version.ref = "edc" } diff --git a/utils/catalog-parser/build.gradle.kts b/utils/catalog-parser/build.gradle.kts index dee5ebaf0..cd0b320d5 100644 --- a/utils/catalog-parser/build.gradle.kts +++ b/utils/catalog-parser/build.gradle.kts @@ -18,7 +18,9 @@ dependencies { implementation(libs.apache.commonsLang) implementation(libs.apache.commonsCollections) implementation(libs.apache.commonsIo) + implementation(libs.edc.runtimeMetamodel) + testAnnotationProcessor(libs.lombok) testCompileOnly(libs.lombok) testImplementation(project(":utils:test-utils")) From acfd9a6e9cbb07b8b6b956e6986e78e1fb38d9cb Mon Sep 17 00:00:00 2001 From: Christophe Loiseau Date: Tue, 23 Jul 2024 09:48:10 +0200 Subject: [PATCH 10/24] Force non snapshot metamodel runtime --- .../catalog-crawler/catalog-crawler-e2e-test/build.gradle.kts | 1 + .../catalog-crawler-launcher-base/build.gradle.kts | 1 + extensions/catalog-crawler/catalog-crawler/build.gradle.kts | 1 + extensions/contract-termination/build.gradle.kts | 3 ++- extensions/database-direct-access/build.gradle.kts | 1 + extensions/placeholder-data-source/build.gradle.kts | 3 ++- extensions/policy-referring-connector/build.gradle.kts | 3 +++ extensions/policy-time-interval/build.gradle.kts | 4 ++++ extensions/postgres-flyway-core/build.gradle.kts | 2 ++ extensions/postgres-flyway/build.gradle.kts | 2 ++ extensions/sovity-messenger/build.gradle.kts | 2 +- extensions/transfer-process-status-checker/build.gradle.kts | 1 + utils/catalog-parser/build.gradle.kts | 2 +- utils/versions/build.gradle.kts | 4 +--- 14 files changed, 23 insertions(+), 7 deletions(-) diff --git a/extensions/catalog-crawler/catalog-crawler-e2e-test/build.gradle.kts b/extensions/catalog-crawler/catalog-crawler-e2e-test/build.gradle.kts index 9f3492139..1b4f3c7a4 100644 --- a/extensions/catalog-crawler/catalog-crawler-e2e-test/build.gradle.kts +++ b/extensions/catalog-crawler/catalog-crawler-e2e-test/build.gradle.kts @@ -19,6 +19,7 @@ dependencies { testImplementation(libs.assertj.core) testImplementation(libs.mockito.core) testImplementation(libs.edc.junit) + testImplementation(libs.edc.runtimeMetamodel) testImplementation(libs.restAssured.restAssured) testImplementation(libs.testcontainers.testcontainers) testImplementation(libs.flyway.core) diff --git a/extensions/catalog-crawler/catalog-crawler-launcher-base/build.gradle.kts b/extensions/catalog-crawler/catalog-crawler-launcher-base/build.gradle.kts index 2c376a085..3cf6464c7 100644 --- a/extensions/catalog-crawler/catalog-crawler-launcher-base/build.gradle.kts +++ b/extensions/catalog-crawler/catalog-crawler-launcher-base/build.gradle.kts @@ -11,6 +11,7 @@ dependencies { api(libs.edc.http) api(libs.edc.dsp) api(libs.edc.jsonLd) + implementation(libs.edc.runtimeMetamodel) // Data Catalog Crawler api(project(":extensions:catalog-crawler:catalog-crawler")) diff --git a/extensions/catalog-crawler/catalog-crawler/build.gradle.kts b/extensions/catalog-crawler/catalog-crawler/build.gradle.kts index 9f4f98b95..ccf617371 100644 --- a/extensions/catalog-crawler/catalog-crawler/build.gradle.kts +++ b/extensions/catalog-crawler/catalog-crawler/build.gradle.kts @@ -8,6 +8,7 @@ dependencies { implementation(libs.edc.controlPlaneSpi) implementation(libs.edc.managementApiConfiguration) + implementation(libs.edc.runtimeMetamodel) implementation(libs.quartz.quartz) implementation(libs.apache.commonsLang) diff --git a/extensions/contract-termination/build.gradle.kts b/extensions/contract-termination/build.gradle.kts index 61c1ad4ec..44eb0cdb8 100644 --- a/extensions/contract-termination/build.gradle.kts +++ b/extensions/contract-termination/build.gradle.kts @@ -13,8 +13,9 @@ dependencies { implementation(project(":extensions:sovity-messenger")) implementation(libs.edc.coreSpi) - implementation(libs.edc.transferSpi) implementation(libs.edc.dspNegotiationTransform) + implementation(libs.edc.runtimeMetamodel) + implementation(libs.edc.transferSpi) implementation(libs.jakarta.rsApi) diff --git a/extensions/database-direct-access/build.gradle.kts b/extensions/database-direct-access/build.gradle.kts index 3aecfb4d1..61f5235b9 100644 --- a/extensions/database-direct-access/build.gradle.kts +++ b/extensions/database-direct-access/build.gradle.kts @@ -9,6 +9,7 @@ dependencies { compileOnly(libs.lombok) implementation(libs.edc.coreSpi) + implementation(libs.edc.runtimeMetamodel) implementation(libs.jooq.jooq) implementation(libs.hikari) diff --git a/extensions/placeholder-data-source/build.gradle.kts b/extensions/placeholder-data-source/build.gradle.kts index 584971a28..e7e33ef1c 100644 --- a/extensions/placeholder-data-source/build.gradle.kts +++ b/extensions/placeholder-data-source/build.gradle.kts @@ -8,9 +8,10 @@ dependencies { annotationProcessor(libs.lombok) compileOnly(libs.lombok) - implementation(libs.edc.coreSpi) implementation(libs.edc.apiCore) + implementation(libs.edc.coreSpi) implementation(libs.edc.dspApiConfiguration) + implementation(libs.edc.runtimeMetamodel) implementation(libs.okhttp.okhttp) diff --git a/extensions/policy-referring-connector/build.gradle.kts b/extensions/policy-referring-connector/build.gradle.kts index c7fa700f5..e4fe94431 100644 --- a/extensions/policy-referring-connector/build.gradle.kts +++ b/extensions/policy-referring-connector/build.gradle.kts @@ -8,6 +8,9 @@ dependencies { api(libs.edc.authSpi) api(libs.edc.policyEngineSpi) api(libs.edc.contractSpi) + implementation(libs.edc.runtimeMetamodel) + + testImplementation(libs.edc.junit) testImplementation(libs.mockito.core) diff --git a/extensions/policy-time-interval/build.gradle.kts b/extensions/policy-time-interval/build.gradle.kts index 66b2183c3..7725750a7 100644 --- a/extensions/policy-time-interval/build.gradle.kts +++ b/extensions/policy-time-interval/build.gradle.kts @@ -7,6 +7,10 @@ plugins { dependencies { api(libs.edc.authSpi) api(libs.edc.policyEngineSpi) + + implementation(libs.edc.runtimeMetamodel) + + testImplementation(libs.edc.junit) } diff --git a/extensions/postgres-flyway-core/build.gradle.kts b/extensions/postgres-flyway-core/build.gradle.kts index 5e8b2ef2f..c7b87ddd4 100644 --- a/extensions/postgres-flyway-core/build.gradle.kts +++ b/extensions/postgres-flyway-core/build.gradle.kts @@ -13,6 +13,8 @@ dependencies { api(libs.hikari) implementation(libs.apache.commonsLang) + + implementation(libs.edc.runtimeMetamodel) } group = libs.versions.sovityEdcExtensionGroup.get() diff --git a/extensions/postgres-flyway/build.gradle.kts b/extensions/postgres-flyway/build.gradle.kts index 8b0b67dca..6efbf0fe6 100644 --- a/extensions/postgres-flyway/build.gradle.kts +++ b/extensions/postgres-flyway/build.gradle.kts @@ -13,7 +13,9 @@ dependencies { // Adds Database-Related EDC-Extensions (EDC-SQL-Stores, JDBC-Driver, Pool and Transactions) implementation(libs.edc.controlPlaneSql) + implementation(libs.edc.runtimeMetamodel) implementation(libs.edc.transactionLocal) + implementation(libs.tractus.sqlPool) implementation(libs.apache.commonsLang) diff --git a/extensions/sovity-messenger/build.gradle.kts b/extensions/sovity-messenger/build.gradle.kts index cd9c7655d..63701b8b3 100644 --- a/extensions/sovity-messenger/build.gradle.kts +++ b/extensions/sovity-messenger/build.gradle.kts @@ -15,9 +15,9 @@ dependencies { implementation(libs.edc.dspHttpSpi) implementation(libs.edc.httpSpi) implementation(libs.edc.managementApiConfiguration) + implementation(libs.edc.runtimeMetamodel) implementation(libs.edc.transformCore) - testAnnotationProcessor(libs.lombok) testCompileOnly(libs.lombok) diff --git a/extensions/transfer-process-status-checker/build.gradle.kts b/extensions/transfer-process-status-checker/build.gradle.kts index e420cbdaf..bba0e69bb 100644 --- a/extensions/transfer-process-status-checker/build.gradle.kts +++ b/extensions/transfer-process-status-checker/build.gradle.kts @@ -5,6 +5,7 @@ plugins { dependencies { api(libs.edc.transferSpi) + implementation(libs.edc.runtimeMetamodel) testImplementation(libs.edc.junit) } diff --git a/utils/catalog-parser/build.gradle.kts b/utils/catalog-parser/build.gradle.kts index cd0b320d5..3527b383e 100644 --- a/utils/catalog-parser/build.gradle.kts +++ b/utils/catalog-parser/build.gradle.kts @@ -20,7 +20,7 @@ dependencies { implementation(libs.apache.commonsIo) implementation(libs.edc.runtimeMetamodel) - + testAnnotationProcessor(libs.lombok) testCompileOnly(libs.lombok) testImplementation(project(":utils:test-utils")) diff --git a/utils/versions/build.gradle.kts b/utils/versions/build.gradle.kts index 2d27f8408..6d9aca986 100644 --- a/utils/versions/build.gradle.kts +++ b/utils/versions/build.gradle.kts @@ -5,8 +5,6 @@ import com.squareup.javapoet.TypeSpec import javax.lang.model.element.Modifier.FINAL import javax.lang.model.element.Modifier.PUBLIC import javax.lang.model.element.Modifier.STATIC -import java.lang.String as JavaString - plugins { `java-library` @@ -26,7 +24,7 @@ val generateGradleVersions by tasks.creating { val versionsClass = TypeSpec.classBuilder("GradleVersions") .addModifiers(PUBLIC, FINAL) .addField( - FieldSpec.builder(TypeName.get(JavaString::class.java), "POSTGRES_IMAGE_TAG") + FieldSpec.builder(TypeName.get(String::class.java), "POSTGRES_IMAGE_TAG") .initializer("\$S", libs.versions.postgresDbImage.get()) .addModifiers(PUBLIC, STATIC, FINAL) .build() From dd91be1339f786f535fc189e95c34468971d618a Mon Sep 17 00:00:00 2001 From: Christophe Loiseau Date: Tue, 23 Jul 2024 09:52:14 +0200 Subject: [PATCH 11/24] Force non snapshot metamodel runtime --- extensions/wrapper/wrapper-api/build.gradle.kts | 1 + 1 file changed, 1 insertion(+) diff --git a/extensions/wrapper/wrapper-api/build.gradle.kts b/extensions/wrapper/wrapper-api/build.gradle.kts index 48644fa8b..dd0c50142 100644 --- a/extensions/wrapper/wrapper-api/build.gradle.kts +++ b/extensions/wrapper/wrapper-api/build.gradle.kts @@ -16,6 +16,7 @@ dependencies { api(project(":extensions:wrapper:wrapper-ee-api")) implementation(libs.apache.commonsLang) + implementation(libs.edc.runtimeMetamodel) implementation(libs.hibernate.validation) implementation(libs.jakarta.el) implementation(libs.jakarta.rsApi) From 83f92fdbe66e48b88432e52910207518e95f0963 Mon Sep 17 00:00:00 2001 From: Christophe Loiseau Date: Tue, 23 Jul 2024 09:57:04 +0200 Subject: [PATCH 12/24] Force non snapshot metamodel runtime --- utils/test-utils/build.gradle.kts | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/test-utils/build.gradle.kts b/utils/test-utils/build.gradle.kts index b6b1eab88..cf0d63eed 100644 --- a/utils/test-utils/build.gradle.kts +++ b/utils/test-utils/build.gradle.kts @@ -20,6 +20,7 @@ dependencies { implementation(project(":utils:versions")) implementation(libs.edc.jsonLdSpi) implementation(libs.edc.jsonLd) + implementation(libs.edc.runtimeMetamodel) implementation(libs.edc.sqlCore) implementation(libs.assertj.core) implementation(libs.jooq.jooq) From 28becc0a7bdce4695803dc233ec46868344a8b4c Mon Sep 17 00:00:00 2001 From: Christophe Loiseau Date: Tue, 23 Jul 2024 17:59:28 +0200 Subject: [PATCH 13/24] Move into broker --- .../ext/catalog/crawler/CrawlerE2eTest.java | 3 +- .../catalog-crawler/build.gradle.kts | 1 - .../ext/catalog/crawler/CrawlerExtension.java | 7 +- .../CrawlerExtensionContextBuilder.java | 2 +- extensions/placeholder-data-source/README.md | 73 ------------------- .../placeholder-data-source/build.gradle.kts | 35 --------- .../PlaceholderDataSourceExtension.java | 52 ------------- ...rg.eclipse.edc.spi.system.ServiceExtension | 1 - .../build.gradle.kts | 1 - .../wrapper-common-mappers/build.gradle.kts | 2 +- .../mappers}/PlaceholderEndpointService.java | 2 +- .../http/HttpDataSourceMapper.java | 2 +- .../wrapper/api/common/mappers/Factory.java | 1 - extensions/wrapper/wrapper/build.gradle.kts | 2 +- .../edc/ext/wrapper/WrapperExtension.java | 22 +++++- .../WrapperExtensionContextBuilder.java | 2 +- .../PlaceholderEndpointController.java | 2 +- settings.gradle.kts | 1 - .../PlaceholderDataSourceExtensionTest.java | 6 +- 19 files changed, 33 insertions(+), 184 deletions(-) delete mode 100644 extensions/placeholder-data-source/README.md delete mode 100644 extensions/placeholder-data-source/build.gradle.kts delete mode 100644 extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderDataSourceExtension.java delete mode 100644 extensions/placeholder-data-source/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension rename extensions/{placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource => wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers}/PlaceholderEndpointService.java (94%) rename extensions/{placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource => wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/controller}/PlaceholderEndpointController.java (96%) rename {extensions/placeholder-data-source/src/test/java/de/sovity/edc/extension/placeholderdatasource => tests/src/test/java/de/sovity/edc/e2e}/PlaceholderDataSourceExtensionTest.java (91%) diff --git a/extensions/catalog-crawler/catalog-crawler-e2e-test/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerE2eTest.java b/extensions/catalog-crawler/catalog-crawler-e2e-test/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerE2eTest.java index ad3fa91ec..1882f283a 100644 --- a/extensions/catalog-crawler/catalog-crawler-e2e-test/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerE2eTest.java +++ b/extensions/catalog-crawler/catalog-crawler-e2e-test/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerE2eTest.java @@ -38,7 +38,6 @@ import de.sovity.edc.ext.catalog.crawler.utils.TestData; import de.sovity.edc.extension.e2e.connector.config.ConnectorConfig; import de.sovity.edc.extension.e2e.db.EdcRuntimeExtensionWithTestDatabase; -import de.sovity.edc.extension.placeholderdatasource.PlaceholderDataSourceExtension; import de.sovity.edc.utils.jsonld.vocab.Prop; import org.awaitility.Awaitility; import org.awaitility.core.ThrowingRunnable; @@ -107,7 +106,7 @@ class CrawlerE2eTest { props.put(CrawlerExtension.CRON_DEAD_CONNECTOR_REFRESH, everySeconds); props.put(CrawlerExtension.SCHEDULED_KILL_OFFLINE_CONNECTORS, everySeconds); props.put(CrawlerExtension.KILL_OFFLINE_CONNECTORS_AFTER, "P1D"); - props.put(PlaceholderDataSourceExtension.MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL, "http://example.com/edc/backend"); + props.put("my.edc.datasource.placeholder.baseurl", "http://example.com/edc/backend"); return props; } diff --git a/extensions/catalog-crawler/catalog-crawler/build.gradle.kts b/extensions/catalog-crawler/catalog-crawler/build.gradle.kts index ccf617371..829822d92 100644 --- a/extensions/catalog-crawler/catalog-crawler/build.gradle.kts +++ b/extensions/catalog-crawler/catalog-crawler/build.gradle.kts @@ -18,7 +18,6 @@ dependencies { api(project(":utils:json-and-jsonld-utils")) api(project(":extensions:wrapper:wrapper-common-mappers")) api(project(":extensions:catalog-crawler:catalog-crawler-db")) - api(project(":extensions:placeholder-data-source")) api(project(":extensions:postgres-flyway-core")) testAnnotationProcessor(libs.lombok) diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java index b53f47d7b..ae1f0eca4 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java @@ -14,7 +14,7 @@ package de.sovity.edc.ext.catalog.crawler; -import de.sovity.edc.extension.placeholderdatasource.PlaceholderEndpointService; +import de.sovity.edc.ext.wrapper.api.common.mappers.PlaceholderEndpointService; import org.eclipse.edc.connector.api.management.configuration.transform.ManagementApiTypeTransformerRegistry; import org.eclipse.edc.connector.spi.catalog.CatalogService; import org.eclipse.edc.jsonld.spi.JsonLd; @@ -91,9 +91,6 @@ public class CrawlerExtension implements ServiceExtension { @Inject private TypeManager typeManager; - @Inject - private PlaceholderEndpointService placeholderEndpointService; - @Inject private ManagementApiTypeTransformerRegistry typeTransformerRegistry; @@ -127,7 +124,7 @@ public void initialize(ServiceExtensionContext context) { typeTransformerRegistry, jsonLd, catalogService, - placeholderEndpointService + new PlaceholderEndpointService("http://0.0.0.0/") ); // Provide access for the tests diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java index cec917df7..89aa9e491 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java +++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java @@ -52,6 +52,7 @@ import de.sovity.edc.ext.catalog.crawler.orchestration.schedules.QuartzScheduleInitializer; import de.sovity.edc.ext.catalog.crawler.orchestration.schedules.utils.CronJobRef; import de.sovity.edc.ext.wrapper.api.common.mappers.AssetMapper; +import de.sovity.edc.ext.wrapper.api.common.mappers.PlaceholderEndpointService; import de.sovity.edc.ext.wrapper.api.common.mappers.PolicyMapper; import de.sovity.edc.ext.wrapper.api.common.mappers.asset.AssetEditRequestMapper; import de.sovity.edc.ext.wrapper.api.common.mappers.asset.AssetJsonLdBuilder; @@ -68,7 +69,6 @@ import de.sovity.edc.ext.wrapper.api.common.mappers.policy.LiteralMapper; import de.sovity.edc.ext.wrapper.api.common.mappers.policy.OperatorMapper; import de.sovity.edc.ext.wrapper.api.common.mappers.policy.PolicyValidator; -import de.sovity.edc.extension.placeholderdatasource.PlaceholderEndpointService; import de.sovity.edc.utils.catalog.DspCatalogService; import de.sovity.edc.utils.catalog.mapper.DspDataOfferBuilder; import lombok.NoArgsConstructor; diff --git a/extensions/placeholder-data-source/README.md b/extensions/placeholder-data-source/README.md deleted file mode 100644 index d0e161701..000000000 --- a/extensions/placeholder-data-source/README.md +++ /dev/null @@ -1,73 +0,0 @@ - -
-
- - Logo - - -

EDC-Connector Extension:
Sovity Messenger

- -

- Report Bug - · - Asset Data Source Placeholder -

-
- -## About this Extension - -Provides a placeholder endpoint for on-request offers. - -## Why does this extension exist? - -This extension exists to inform the asset's consumer upon data retrieval that they should contact the provider and take extra steps to access the data. - -## Configuration - -`MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL` / `my.edc.datasource.placeholder.baseurl` must be set to point to the placeholder endpoint's base URL. -`/data-source/placeholder/asset/` will be appended to this base URL to make it the placeholder data source endpoint. This will be the address as seen by the consumer. - ---- - -On a production system, the base URL it could be: - -`https://mycompany.com/path/to/backend` - -with a placeholder value: - -`https://mycompany.com/path/to/backend/data-source/placeholder/asset` - -and a full path to the asset's data: - -`https://mycompany.com/path/to/backend/data-source/placeholder/asset?email=foo%40example.com&subject=Contact+us+now`. - ---- - -On a system started with docker-compose, it will be pointing to the DSP port on the provider's EDC - -`http://edc:11003/` - -`http://edc:11003/data-source/placeholder/asset` - -`http://edc:11003/data-source/placeholder/asset?email=foo%40example.com&subject=Contact+us+now` - ---- - -During local/dev/unit test execution, it will be pointing to the DSP port on the provider's EDC - -`http://localhost:12345/` - -where the port that is here `12345` would actually be chosen at random - -`http://localhost:12345/data-source/placeholder/asset` - -`http://localhost:12345/data-source/placeholder/asset?email=foo%40example.com&subject=Contact+us+now` - -## License - -Apache License 2.0 - see [LICENSE](../../LICENSE) - -## Contact - -sovity GmbH - contact@sovity.de - diff --git a/extensions/placeholder-data-source/build.gradle.kts b/extensions/placeholder-data-source/build.gradle.kts deleted file mode 100644 index e7e33ef1c..000000000 --- a/extensions/placeholder-data-source/build.gradle.kts +++ /dev/null @@ -1,35 +0,0 @@ - -plugins { - `java-library` - `maven-publish` -} - -dependencies { - annotationProcessor(libs.lombok) - compileOnly(libs.lombok) - - implementation(libs.edc.apiCore) - implementation(libs.edc.coreSpi) - implementation(libs.edc.dspApiConfiguration) - implementation(libs.edc.runtimeMetamodel) - implementation(libs.okhttp.okhttp) - - - testAnnotationProcessor(libs.lombok) - testCompileOnly(libs.lombok) - - testImplementation(project(":utils:test-utils")) - testImplementation(libs.assertj.core) - testImplementation(libs.junit.api) - testRuntimeOnly(libs.junit.engine) -} - -group = libs.versions.sovityEdcExtensionGroup.get() - -publishing { - publications { - create(project.name) { - from(components["java"]) - } - } -} diff --git a/extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderDataSourceExtension.java b/extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderDataSourceExtension.java deleted file mode 100644 index 12db5c12a..000000000 --- a/extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderDataSourceExtension.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2024 sovity GmbH - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * sovity GmbH - initial API and implementation - * - */ - -package de.sovity.edc.extension.placeholderdatasource; - -import lombok.val; -import org.eclipse.edc.protocol.dsp.api.configuration.DspApiConfiguration; -import org.eclipse.edc.runtime.metamodel.annotation.Extension; -import org.eclipse.edc.runtime.metamodel.annotation.Inject; -import org.eclipse.edc.runtime.metamodel.annotation.Provides; -import org.eclipse.edc.runtime.metamodel.annotation.Setting; -import org.eclipse.edc.spi.system.ServiceExtension; -import org.eclipse.edc.spi.system.ServiceExtensionContext; -import org.eclipse.edc.web.spi.WebService; - -import static de.sovity.edc.extension.placeholderdatasource.PlaceholderDataSourceExtension.NAME; - -@Extension(value = NAME) -@Provides(PlaceholderEndpointService.class) -public class PlaceholderDataSourceExtension implements ServiceExtension { - - public static final String NAME = "Placeholder Data Source"; - - @Setting(required = true) - public static final String MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL = "my.edc.datasource.placeholder.baseurl"; - - @Inject - private DspApiConfiguration dspApiConfiguration; - - @Inject - private WebService webService; - - @Override - public void initialize(ServiceExtensionContext context) { - val controller = new PlaceholderEndpointController(); - webService.registerResource(dspApiConfiguration.getContextAlias(), controller); - - val baseUrl = context.getConfig().getString(MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL); - context.registerService(PlaceholderEndpointService.class, new PlaceholderEndpointService(baseUrl)); - } -} diff --git a/extensions/placeholder-data-source/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/placeholder-data-source/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension deleted file mode 100644 index 1c4398d5d..000000000 --- a/extensions/placeholder-data-source/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension +++ /dev/null @@ -1 +0,0 @@ -de.sovity.edc.extension.placeholderdatasource.PlaceholderDataSourceExtension diff --git a/extensions/sovity-edc-extensions-package/build.gradle.kts b/extensions/sovity-edc-extensions-package/build.gradle.kts index b9b74a6a5..dc4edc6d1 100644 --- a/extensions/sovity-edc-extensions-package/build.gradle.kts +++ b/extensions/sovity-edc-extensions-package/build.gradle.kts @@ -6,7 +6,6 @@ plugins { dependencies { // Policies - api(project(":extensions:placeholder-data-source")) api(project(":extensions:policy-referring-connector")) api(project(":extensions:policy-time-interval")) api(project(":extensions:policy-always-true")) diff --git a/extensions/wrapper/wrapper-common-mappers/build.gradle.kts b/extensions/wrapper/wrapper-common-mappers/build.gradle.kts index def04723e..5fbc898e8 100644 --- a/extensions/wrapper/wrapper-common-mappers/build.gradle.kts +++ b/extensions/wrapper/wrapper-common-mappers/build.gradle.kts @@ -16,11 +16,11 @@ dependencies { api(project(":extensions:wrapper:wrapper-common-api")) api(project(":utils:json-and-jsonld-utils")) - implementation(project(":extensions:placeholder-data-source")) implementation(libs.apache.commonsLang) implementation(libs.apache.commonsCollections) implementation(libs.edc.runtimeMetamodel) implementation(libs.flexmark.all) + implementation(libs.okhttp.okhttp) testAnnotationProcessor(libs.lombok) diff --git a/extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderEndpointService.java b/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/PlaceholderEndpointService.java similarity index 94% rename from extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderEndpointService.java rename to extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/PlaceholderEndpointService.java index 515f2cf4d..d3c1c955a 100644 --- a/extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderEndpointService.java +++ b/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/PlaceholderEndpointService.java @@ -12,7 +12,7 @@ * */ -package de.sovity.edc.extension.placeholderdatasource; +package de.sovity.edc.ext.wrapper.api.common.mappers; import lombok.RequiredArgsConstructor; import okhttp3.HttpUrl; diff --git a/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/dataaddress/http/HttpDataSourceMapper.java b/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/dataaddress/http/HttpDataSourceMapper.java index 8d06de0cb..ac134fde6 100644 --- a/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/dataaddress/http/HttpDataSourceMapper.java +++ b/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/dataaddress/http/HttpDataSourceMapper.java @@ -14,9 +14,9 @@ package de.sovity.edc.ext.wrapper.api.common.mappers.dataaddress.http; +import de.sovity.edc.ext.wrapper.api.common.mappers.PlaceholderEndpointService; import de.sovity.edc.ext.wrapper.api.common.model.UiDataSourceHttpData; import de.sovity.edc.ext.wrapper.api.common.model.UiDataSourceOnRequest; -import de.sovity.edc.extension.placeholderdatasource.PlaceholderEndpointService; import de.sovity.edc.utils.jsonld.vocab.Prop; import jakarta.json.Json; import jakarta.json.JsonObject; diff --git a/extensions/wrapper/wrapper-common-mappers/src/test/java/de/sovity/edc/ext/wrapper/api/common/mappers/Factory.java b/extensions/wrapper/wrapper-common-mappers/src/test/java/de/sovity/edc/ext/wrapper/api/common/mappers/Factory.java index 99505398e..5d03c1a8e 100644 --- a/extensions/wrapper/wrapper-common-mappers/src/test/java/de/sovity/edc/ext/wrapper/api/common/mappers/Factory.java +++ b/extensions/wrapper/wrapper-common-mappers/src/test/java/de/sovity/edc/ext/wrapper/api/common/mappers/Factory.java @@ -24,7 +24,6 @@ import de.sovity.edc.ext.wrapper.api.common.mappers.dataaddress.DataSourceMapper; import de.sovity.edc.ext.wrapper.api.common.mappers.dataaddress.http.HttpDataSourceMapper; import de.sovity.edc.ext.wrapper.api.common.mappers.dataaddress.http.HttpHeaderMapper; -import de.sovity.edc.extension.placeholderdatasource.PlaceholderEndpointService; import org.eclipse.edc.jsonld.TitaniumJsonLd; import org.eclipse.edc.spi.monitor.Monitor; import org.eclipse.edc.transform.spi.TypeTransformerRegistry; diff --git a/extensions/wrapper/wrapper/build.gradle.kts b/extensions/wrapper/wrapper/build.gradle.kts index 31dac2f1a..90de0d990 100644 --- a/extensions/wrapper/wrapper/build.gradle.kts +++ b/extensions/wrapper/wrapper/build.gradle.kts @@ -21,13 +21,13 @@ dependencies { implementation(project(":extensions:contract-termination")) implementation(project(":extensions:database-direct-access")) - implementation(project(":extensions:placeholder-data-source")) implementation(project(":extensions:sovity-messenger")) implementation(project(":utils:jooq-database-access")) implementation(libs.apache.commonsLang) implementation(libs.edc.apiCore) implementation(libs.edc.managementApiConfiguration) + implementation(libs.edc.dspApiConfiguration) implementation(libs.edc.dspHttpSpi) implementation(libs.edc.runtimeMetamodel) implementation(libs.jooq.jooq) diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtension.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtension.java index 5ea98e08a..c2369aa10 100644 --- a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtension.java +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtension.java @@ -17,10 +17,12 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import de.sovity.edc.ext.wrapper.api.common.mappers.PlaceholderEndpointService; +import de.sovity.edc.ext.wrapper.controller.PlaceholderEndpointController; import de.sovity.edc.extension.contacttermination.ContractAgreementTerminationService; import de.sovity.edc.extension.db.directaccess.DslContextFactory; import de.sovity.edc.extension.messenger.SovityMessenger; -import de.sovity.edc.extension.placeholderdatasource.PlaceholderEndpointService; +import lombok.val; import org.eclipse.edc.connector.api.management.configuration.ManagementApiConfiguration; import org.eclipse.edc.connector.api.management.configuration.transform.ManagementApiTypeTransformerRegistry; import org.eclipse.edc.connector.contract.spi.negotiation.store.ContractNegotiationStore; @@ -36,7 +38,9 @@ import org.eclipse.edc.connector.transfer.spi.store.TransferProcessStore; import org.eclipse.edc.jsonld.spi.JsonLd; import org.eclipse.edc.policy.engine.spi.PolicyEngine; +import org.eclipse.edc.protocol.dsp.api.configuration.DspApiConfiguration; import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.runtime.metamodel.annotation.Setting; import org.eclipse.edc.spi.CoreConstants; import org.eclipse.edc.spi.asset.AssetIndex; import org.eclipse.edc.spi.system.ServiceExtension; @@ -46,6 +50,8 @@ public class WrapperExtension implements ServiceExtension { + @Setting(value = "Defaults to tne env var EDC_DSP_CALLBACK_ADDRESS") + public static final String MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL = "my.edc.datasource.placeholder.baseurl"; public static final String EXTENSION_NAME = "WrapperExtension"; @@ -68,9 +74,9 @@ public class WrapperExtension implements ServiceExtension { @Inject private DslContextFactory dslContextFactory; @Inject - private ManagementApiConfiguration dataManagementApiConfiguration; + private DspApiConfiguration dspApiConfiguration; @Inject - private PlaceholderEndpointService placeholderEndpointService; + private ManagementApiConfiguration dataManagementApiConfiguration; @Inject private PolicyDefinitionStore policyDefinitionStore; @Inject @@ -104,6 +110,11 @@ public void initialize(ServiceExtensionContext context) { var objectMapper = typeManager.getMapper(CoreConstants.JSON_LD); fixObjectMapperDateSerialization(objectMapper); + val baseUrl = context.getConfig().getString(MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL); + val placeholderEndpointService = new PlaceholderEndpointService(baseUrl); + setupPlaceholderEndpoint(); + context.registerService(PlaceholderEndpointService.class, placeholderEndpointService); + var wrapperExtensionContext = WrapperExtensionContextBuilder.buildContext( assetIndex, assetService, @@ -135,6 +146,11 @@ public void initialize(ServiceExtensionContext context) { webService.registerResource(dataManagementApiConfiguration.getContextAlias(), resource)); } + private void setupPlaceholderEndpoint() { + val controller = new PlaceholderEndpointController(); + webService.registerResource(dspApiConfiguration.getContextAlias(), controller); + } + private void fixObjectMapperDateSerialization(ObjectMapper objectMapper) { // Fixes Dates in JSON-LD Object Mapper // The Core EDC uses longs over OffsetDateTime, so they never fixed the date format diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java index 334fc8b0d..55e02349b 100644 --- a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java @@ -80,7 +80,7 @@ import de.sovity.edc.extension.contacttermination.query.TerminateContractQuery; import de.sovity.edc.extension.db.directaccess.DslContextFactory; import de.sovity.edc.extension.messenger.SovityMessenger; -import de.sovity.edc.extension.placeholderdatasource.PlaceholderEndpointService; +import de.sovity.edc.ext.wrapper.api.common.mappers.PlaceholderEndpointService; import de.sovity.edc.utils.catalog.DspCatalogService; import de.sovity.edc.utils.catalog.mapper.DspDataOfferBuilder; import lombok.NoArgsConstructor; diff --git a/extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderEndpointController.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/controller/PlaceholderEndpointController.java similarity index 96% rename from extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderEndpointController.java rename to extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/controller/PlaceholderEndpointController.java index 41f4b44a3..a399afa39 100644 --- a/extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderEndpointController.java +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/controller/PlaceholderEndpointController.java @@ -12,7 +12,7 @@ * */ -package de.sovity.edc.extension.placeholderdatasource; +package de.sovity.edc.ext.wrapper.controller; import jakarta.ws.rs.Consumes; import jakarta.ws.rs.GET; diff --git a/settings.gradle.kts b/settings.gradle.kts index 6e5ae061e..78a0903b2 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -6,7 +6,6 @@ include(":extensions:catalog-crawler:catalog-crawler-launcher-base") include(":extensions:catalog-crawler:catalog-crawler-e2e-test") include(":extensions:contract-termination") include(":extensions:database-direct-access") -include(":extensions:placeholder-data-source") include(":extensions:edc-ui-config") include(":extensions:last-commit-info") include(":extensions:policy-always-true") diff --git a/extensions/placeholder-data-source/src/test/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderDataSourceExtensionTest.java b/tests/src/test/java/de/sovity/edc/e2e/PlaceholderDataSourceExtensionTest.java similarity index 91% rename from extensions/placeholder-data-source/src/test/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderDataSourceExtensionTest.java rename to tests/src/test/java/de/sovity/edc/e2e/PlaceholderDataSourceExtensionTest.java index 5d739edf1..dd9a913f4 100644 --- a/extensions/placeholder-data-source/src/test/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderDataSourceExtensionTest.java +++ b/tests/src/test/java/de/sovity/edc/e2e/PlaceholderDataSourceExtensionTest.java @@ -12,13 +12,15 @@ * */ -package de.sovity.edc.extension.placeholderdatasource; +package de.sovity.edc.e2e; import de.sovity.edc.client.EdcClient; import de.sovity.edc.client.gen.model.DataSourceType; import de.sovity.edc.client.gen.model.UiAssetCreateRequest; import de.sovity.edc.client.gen.model.UiDataSource; import de.sovity.edc.client.gen.model.UiDataSourceOnRequest; +import de.sovity.edc.ext.wrapper.WrapperExtension; +import de.sovity.edc.ext.wrapper.api.common.mappers.PlaceholderEndpointService; import de.sovity.edc.extension.e2e.connector.config.ConnectorConfig; import de.sovity.edc.extension.e2e.extension.E2eTestExtension; import de.sovity.edc.extension.e2e.extension.Provider; @@ -73,7 +75,7 @@ void shouldAccessDummyEndpoint( // assert assertThat(baseUrl) - .startsWith(providerConfig.getProperties().get(PlaceholderDataSourceExtension.MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL)) + .startsWith(providerConfig.getProperties().get(WrapperExtension.MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL)) .isEqualTo(expected); val request = new Request.Builder() From 6afe6ea3bc864d754f54e9400f334620a20d7197 Mon Sep 17 00:00:00 2001 From: Christophe Loiseau Date: Wed, 24 Jul 2024 08:37:00 +0200 Subject: [PATCH 14/24] Code review --- launchers/.env.connector | 2 ++ 1 file changed, 2 insertions(+) diff --git a/launchers/.env.connector b/launchers/.env.connector index d58c1267a..4b6f7b27c 100644 --- a/launchers/.env.connector +++ b/launchers/.env.connector @@ -95,3 +95,5 @@ EDC_AGENT_IDENTITY_KEY=referringConnector # but for some reason it is required, and EDC won't start up if it isn't configured # it is created in the Dockerfile EDC_VAULT=/app/empty-properties-file.properties + +MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL=${EDC_DSP_CALLBACK_ADDRESS} From 7db46b62d415563d4151269f695b25e4091172b0 Mon Sep 17 00:00:00 2001 From: Christophe Loiseau Date: Wed, 24 Jul 2024 08:52:18 +0200 Subject: [PATCH 15/24] Code review --- .../common/mappers/dataaddress/http/HttpDataSourceMapper.java | 2 +- .../sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/dataaddress/http/HttpDataSourceMapper.java b/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/dataaddress/http/HttpDataSourceMapper.java index ac134fde6..1c8b3a4bf 100644 --- a/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/dataaddress/http/HttpDataSourceMapper.java +++ b/extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/dataaddress/http/HttpDataSourceMapper.java @@ -94,7 +94,7 @@ public Map buildOnRequestDataAddress(@NonNull UiDataSourceOnRequ "Need contactPreferredEmailSubject" ); - String placeholderEndpointForAsset = placeholderEndpointService.getPlaceholderEndpointForAsset( + var placeholderEndpointForAsset = placeholderEndpointService.getPlaceholderEndpointForAsset( onRequest.getContactEmail(), onRequest.getContactPreferredEmailSubject()); diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java index 55e02349b..1c8db11ad 100644 --- a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java @@ -17,6 +17,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import de.sovity.edc.ext.wrapper.api.common.mappers.AssetMapper; import de.sovity.edc.ext.wrapper.api.common.mappers.LegacyPolicyMapper; +import de.sovity.edc.ext.wrapper.api.common.mappers.PlaceholderEndpointService; import de.sovity.edc.ext.wrapper.api.common.mappers.PolicyMapper; import de.sovity.edc.ext.wrapper.api.common.mappers.asset.AssetEditRequestMapper; import de.sovity.edc.ext.wrapper.api.common.mappers.asset.AssetJsonLdBuilder; @@ -80,7 +81,6 @@ import de.sovity.edc.extension.contacttermination.query.TerminateContractQuery; import de.sovity.edc.extension.db.directaccess.DslContextFactory; import de.sovity.edc.extension.messenger.SovityMessenger; -import de.sovity.edc.ext.wrapper.api.common.mappers.PlaceholderEndpointService; import de.sovity.edc.utils.catalog.DspCatalogService; import de.sovity.edc.utils.catalog.mapper.DspDataOfferBuilder; import lombok.NoArgsConstructor; From d710eced066626780e64a2ad7762d3d41d1a36a7 Mon Sep 17 00:00:00 2001 From: Christophe Loiseau Date: Wed, 24 Jul 2024 08:55:26 +0200 Subject: [PATCH 16/24] Links check --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c32841efc..a8233fed8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,6 @@ New configuration to access the database: - Defaults to `3` - `MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL` - Base URL of an asset as seen from an external source, from the asset's consumer (typically from the internet, ...) - - It is documented in the asset data placeholder [README.md](extensions/placeholder-data-source/README.md) #### Compatible Versions From 11bbca592032edba8f3ffc446a1d1a56c9a33085 Mon Sep 17 00:00:00 2001 From: Christophe Loiseau Date: Wed, 24 Jul 2024 10:37:22 +0200 Subject: [PATCH 17/24] Test negotiation --- .../sovity/edc/e2e/PlaceholderDataSourceExtensionTest.java | 5 +++++ .../sovity/edc/extension/e2e/extension/E2eTestExtension.java | 1 + 2 files changed, 6 insertions(+) diff --git a/tests/src/test/java/de/sovity/edc/e2e/PlaceholderDataSourceExtensionTest.java b/tests/src/test/java/de/sovity/edc/e2e/PlaceholderDataSourceExtensionTest.java index dd9a913f4..23aaafd48 100644 --- a/tests/src/test/java/de/sovity/edc/e2e/PlaceholderDataSourceExtensionTest.java +++ b/tests/src/test/java/de/sovity/edc/e2e/PlaceholderDataSourceExtensionTest.java @@ -22,6 +22,7 @@ import de.sovity.edc.ext.wrapper.WrapperExtension; import de.sovity.edc.ext.wrapper.api.common.mappers.PlaceholderEndpointService; import de.sovity.edc.extension.e2e.connector.config.ConnectorConfig; +import de.sovity.edc.extension.e2e.extension.E2eScenario; import de.sovity.edc.extension.e2e.extension.E2eTestExtension; import de.sovity.edc.extension.e2e.extension.Provider; import de.sovity.edc.utils.JsonUtils; @@ -43,6 +44,7 @@ class PlaceholderDataSourceExtensionTest { @SneakyThrows @Test void shouldAccessDummyEndpoint( + E2eScenario scenario, @Provider EdcExtension providerExtension, @Provider ConnectorConfig providerConfig, @Provider EdcClient providerClient @@ -73,6 +75,9 @@ void shouldAccessDummyEndpoint( val service = providerExtension.getContext().getService(PlaceholderEndpointService.class); val expected = service.getPlaceholderEndpointForAsset(email, subject); + scenario.createContractDefinition(assetId); + scenario.negotiateAssetAndAwait(assetId); + // assert assertThat(baseUrl) .startsWith(providerConfig.getProperties().get(WrapperExtension.MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL)) diff --git a/utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/extension/E2eTestExtension.java b/utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/extension/E2eTestExtension.java index ddbd592ae..3c774c9da 100644 --- a/utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/extension/E2eTestExtension.java +++ b/utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/extension/E2eTestExtension.java @@ -20,6 +20,7 @@ import de.sovity.edc.extension.e2e.connector.config.ConnectorRemoteConfig; import de.sovity.edc.extension.e2e.db.EdcRuntimeExtensionWithTestDatabase; import de.sovity.edc.extension.utils.Lazy; +import lombok.Builder; import lombok.val; import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.extension.AfterAllCallback; From 82a43b4eb37292628b621354d598da4a89586ec9 Mon Sep 17 00:00:00 2001 From: Christophe Loiseau Date: Wed, 24 Jul 2024 10:45:27 +0200 Subject: [PATCH 18/24] Checkstyle --- .../sovity/edc/extension/e2e/extension/E2eTestExtension.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/extension/E2eTestExtension.java b/utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/extension/E2eTestExtension.java index 3c774c9da..95f45ec4b 100644 --- a/utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/extension/E2eTestExtension.java +++ b/utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/extension/E2eTestExtension.java @@ -20,7 +20,6 @@ import de.sovity.edc.extension.e2e.connector.config.ConnectorRemoteConfig; import de.sovity.edc.extension.e2e.db.EdcRuntimeExtensionWithTestDatabase; import de.sovity.edc.extension.utils.Lazy; -import lombok.Builder; import lombok.val; import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.extension.AfterAllCallback; @@ -51,7 +50,8 @@ public class E2eTestExtension private ConnectorConfig providerConfig; private final EdcRuntimeExtensionWithTestDatabase providerExtension; - private final List> partySupportedTypes = List.of(ConnectorConfig.class, EdcClient.class, ConnectorRemote.class, ClientAndServer.class); + private final List> partySupportedTypes = + List.of(ConnectorConfig.class, EdcClient.class, ConnectorRemote.class, ClientAndServer.class); private final List> supportedTypes = Stream.concat(partySupportedTypes.stream(), Stream.of(E2eScenario.class)).toList(); private Lazy clientAndServer; From f1b9d7778bc6db3215fdb8633cb404f12200c1b6 Mon Sep 17 00:00:00 2001 From: Christophe Loiseau Date: Wed, 24 Jul 2024 12:25:28 +0200 Subject: [PATCH 19/24] Add test for the data placeholder --- .../PlaceholderDataSourceExtensionTest.java | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/tests/src/test/java/de/sovity/edc/e2e/PlaceholderDataSourceExtensionTest.java b/tests/src/test/java/de/sovity/edc/e2e/PlaceholderDataSourceExtensionTest.java index 23aaafd48..c1e264cd3 100644 --- a/tests/src/test/java/de/sovity/edc/e2e/PlaceholderDataSourceExtensionTest.java +++ b/tests/src/test/java/de/sovity/edc/e2e/PlaceholderDataSourceExtensionTest.java @@ -16,6 +16,7 @@ import de.sovity.edc.client.EdcClient; import de.sovity.edc.client.gen.model.DataSourceType; +import de.sovity.edc.client.gen.model.InitiateTransferRequest; import de.sovity.edc.client.gen.model.UiAssetCreateRequest; import de.sovity.edc.client.gen.model.UiDataSource; import de.sovity.edc.client.gen.model.UiDataSourceOnRequest; @@ -28,6 +29,7 @@ import de.sovity.edc.utils.JsonUtils; import de.sovity.edc.utils.jsonld.JsonLdUtils; import de.sovity.edc.utils.jsonld.vocab.Prop; +import jakarta.ws.rs.HttpMethod; import lombok.SneakyThrows; import lombok.val; import okhttp3.OkHttpClient; @@ -35,8 +37,18 @@ import org.eclipse.edc.junit.extensions.EdcExtension; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockserver.integration.ClientAndServer; +import org.mockserver.model.HttpRequest; +import org.mockserver.model.HttpResponse; +import org.mockserver.model.HttpStatusCode; + +import java.util.Base64; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import static org.assertj.core.api.Assertions.assertThat; +import static org.eclipse.edc.spi.CoreConstants.EDC_NAMESPACE; @ExtendWith(E2eTestExtension.class) class PlaceholderDataSourceExtensionTest { @@ -45,6 +57,7 @@ class PlaceholderDataSourceExtensionTest { @Test void shouldAccessDummyEndpoint( E2eScenario scenario, + ClientAndServer clientAndServer, @Provider EdcExtension providerExtension, @Provider ConnectorConfig providerConfig, @Provider EdcClient providerClient @@ -76,7 +89,26 @@ void shouldAccessDummyEndpoint( val expected = service.getPlaceholderEndpointForAsset(email, subject); scenario.createContractDefinition(assetId); - scenario.negotiateAssetAndAwait(assetId); + val negotiation = scenario.negotiateAssetAndAwait(assetId); + + val accessed = new AtomicReference<>("Not accessed."); + val destinationPath = "/foo/bar"; + val destinationUrl = "http://localhost:" + clientAndServer.getPort() + destinationPath; + + clientAndServer.when(HttpRequest.request().withMethod(HttpMethod.POST)) + .respond((it) -> { + accessed.set(it.getBodyAsString()); + return HttpResponse.response().withStatusCode(HttpStatusCode.OK_200.code()); + }); + + scenario.transferAndAwait(InitiateTransferRequest.builder() + .contractAgreementId(negotiation.getContractAgreementId()) + .dataSinkProperties(Map.of( + EDC_NAMESPACE + "baseUrl", destinationUrl, + EDC_NAMESPACE + "method", HttpMethod.POST, + EDC_NAMESPACE + "type", "HttpData" + )) + .build()); // assert assertThat(baseUrl) @@ -93,5 +125,7 @@ void shouldAccessDummyEndpoint( assertThat(content).contains("This is not real data."); assertThat(content).contains(email); assertThat(content).contains(subject); + + assertThat(new String(Base64.getDecoder().decode(accessed.get()))).contains("This is not real data."); } } From fb0f26e1aab856e58a1011bdc7baff9d2c20c615 Mon Sep 17 00:00:00 2001 From: Richard Treier Date: Wed, 24 Jul 2024 13:57:34 +0300 Subject: [PATCH 20/24] chore: small refactoring --- CHANGELOG.md | 8 ------- .../edc/ext/wrapper/WrapperExtension.java | 23 ++++--------------- .../ext/wrapper/WrapperExtensionContext.java | 6 +++-- .../WrapperExtensionContextBuilder.java | 20 +++++++++------- launchers/.env.connector | 1 + 5 files changed, 21 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8233fed8..b04aaaea8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,14 +24,6 @@ please see [changelog_updates.md](docs/dev/changelog_updates.md). ### Deployment Migration Notes -New configuration to access the database: - -- `EDC_SERVER_DB_CONNECTION_POOL_SIZE` - - The property controls the maximum size that the pool is allowed to reach, including both idle and in-use connections. Basically this value will determine the maximum number of actual connections to the database backend. - - Defaults to `3` -- `MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL` - - Base URL of an asset as seen from an external source, from the asset's consumer (typically from the internet, ...) - #### Compatible Versions - Connector Backend Docker Images: diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtension.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtension.java index c2369aa10..ba56efc1f 100644 --- a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtension.java +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtension.java @@ -17,12 +17,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import de.sovity.edc.ext.wrapper.api.common.mappers.PlaceholderEndpointService; -import de.sovity.edc.ext.wrapper.controller.PlaceholderEndpointController; import de.sovity.edc.extension.contacttermination.ContractAgreementTerminationService; import de.sovity.edc.extension.db.directaccess.DslContextFactory; -import de.sovity.edc.extension.messenger.SovityMessenger; -import lombok.val; import org.eclipse.edc.connector.api.management.configuration.ManagementApiConfiguration; import org.eclipse.edc.connector.api.management.configuration.transform.ManagementApiTypeTransformerRegistry; import org.eclipse.edc.connector.contract.spi.negotiation.store.ContractNegotiationStore; @@ -82,8 +78,6 @@ public class WrapperExtension implements ServiceExtension { @Inject private PolicyEngine policyEngine; @Inject - private SovityMessenger sovityMessenger; - @Inject private TransferProcessService transferProcessService; @Inject private TransferProcessStore transferProcessStore; @@ -110,11 +104,6 @@ public void initialize(ServiceExtensionContext context) { var objectMapper = typeManager.getMapper(CoreConstants.JSON_LD); fixObjectMapperDateSerialization(objectMapper); - val baseUrl = context.getConfig().getString(MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL); - val placeholderEndpointService = new PlaceholderEndpointService(baseUrl); - setupPlaceholderEndpoint(); - context.registerService(PlaceholderEndpointService.class, placeholderEndpointService); - var wrapperExtensionContext = WrapperExtensionContextBuilder.buildContext( assetIndex, assetService, @@ -133,22 +122,18 @@ public void initialize(ServiceExtensionContext context) { policyDefinitionService, policyDefinitionStore, policyEngine, - sovityMessenger, transferProcessService, transferProcessStore, - typeTransformerRegistry, - placeholderEndpointService + typeTransformerRegistry ); wrapperExtensionContext.selfDescriptionService().validateSelfDescriptionConfig(); - wrapperExtensionContext.jaxRsResources().forEach(resource -> + wrapperExtensionContext.managementApiResources().forEach(resource -> webService.registerResource(dataManagementApiConfiguration.getContextAlias(), resource)); - } - private void setupPlaceholderEndpoint() { - val controller = new PlaceholderEndpointController(); - webService.registerResource(dspApiConfiguration.getContextAlias(), controller); + wrapperExtensionContext.dspApiResources().forEach(resource -> + webService.registerResource(dspApiConfiguration.getContextAlias(), resource)); } private void fixObjectMapperDateSerialization(ObjectMapper objectMapper) { diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContext.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContext.java index 7280df3b0..cc135b3bf 100644 --- a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContext.java +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContext.java @@ -22,13 +22,15 @@ /** * Manual Dependency Injection result * - * @param jaxRsResources Jax RS Resource implementations to register. Implementations of + * @param managementApiResources Jax RS Resource implementations to register. Implementations of * APIs supported by our EDC API Client that don't have their own * extension should land here. + * @param dspApiResources Jax RS Resource implementations to register. Rare DSP API * @param selfDescriptionService Required here for validation on start-up */ public record WrapperExtensionContext( - List jaxRsResources, + List managementApiResources, + List dspApiResources, SelfDescriptionService selfDescriptionService ) { } diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java index 1c8db11ad..3f64c46b0 100644 --- a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java @@ -76,14 +76,15 @@ import de.sovity.edc.ext.wrapper.api.usecase.pages.catalog.UseCaseCatalogApiService; import de.sovity.edc.ext.wrapper.api.usecase.services.KpiApiService; import de.sovity.edc.ext.wrapper.api.usecase.services.SupportedPolicyApiService; +import de.sovity.edc.ext.wrapper.controller.PlaceholderEndpointController; import de.sovity.edc.extension.contacttermination.ContractAgreementTerminationService; import de.sovity.edc.extension.contacttermination.query.ContractAgreementTerminationDetailsQuery; import de.sovity.edc.extension.contacttermination.query.TerminateContractQuery; import de.sovity.edc.extension.db.directaccess.DslContextFactory; -import de.sovity.edc.extension.messenger.SovityMessenger; import de.sovity.edc.utils.catalog.DspCatalogService; import de.sovity.edc.utils.catalog.mapper.DspDataOfferBuilder; import lombok.NoArgsConstructor; +import lombok.val; import org.eclipse.edc.connector.contract.spi.negotiation.store.ContractNegotiationStore; import org.eclipse.edc.connector.contract.spi.offer.store.ContractDefinitionStore; import org.eclipse.edc.connector.policy.spi.store.PolicyDefinitionStore; @@ -136,11 +137,9 @@ public static WrapperExtensionContext buildContext( PolicyDefinitionService policyDefinitionService, PolicyDefinitionStore policyDefinitionStore, PolicyEngine policyEngine, - SovityMessenger sovityMessenger, TransferProcessService transferProcessService, TransferProcessStore transferProcessStore, - TypeTransformerRegistry typeTransformerRegistry, - PlaceholderEndpointService placeholderEndpointService + TypeTransformerRegistry typeTransformerRegistry ) { // UI API var operatorMapper = new OperatorMapper(); @@ -150,6 +149,9 @@ public static WrapperExtensionContext buildContext( var edcPropertyUtils = new EdcPropertyUtils(); var selfDescriptionService = new SelfDescriptionService(config, monitor); var ownConnectorEndpointService = new OwnConnectorEndpointServiceImpl(selfDescriptionService); + var placeholderEndpointService = new PlaceholderEndpointService( + config.getString(WrapperExtension.MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL) + ); var assetMapper = newAssetMapper(typeTransformerRegistry, jsonLd, ownConnectorEndpointService, placeholderEndpointService); var policyMapper = newPolicyMapper(objectMapper, typeTransformerRegistry, operatorMapper); var transferProcessStateService = new TransferProcessStateService(); @@ -296,12 +298,14 @@ public static WrapperExtensionContext buildContext( supportedPolicyApiService, useCaseCatalogApiService ); + val placeholderEndpointController = new PlaceholderEndpointController(); // Collect all JAX-RS resources - return new WrapperExtensionContext(List.of( - uiResource, - useCaseResource - ), selfDescriptionService); + return new WrapperExtensionContext( + List.of(uiResource, useCaseResource), + List.of(placeholderEndpointController), + selfDescriptionService + ); } @NotNull diff --git a/launchers/.env.connector b/launchers/.env.connector index 4b6f7b27c..5a723560a 100644 --- a/launchers/.env.connector +++ b/launchers/.env.connector @@ -51,6 +51,7 @@ EDC_DSP_CALLBACK_ADDRESS=${MY_EDC_PROTOCOL}${MY_EDC_FQDN}${WEB_HTTP_PROTOCOL_PAT EDC_UI_MANAGEMENT_API_URL_SHOWN_IN_DASHBOARD=${MY_EDC_PROTOCOL}${MY_EDC_FQDN}${WEB_HTTP_MANAGEMENT_PATH} # Flyway Extension: Defaults +EDC_SERVER_DB_CONNECTION_POOL_SIZE=10 EDC_DATASOURCE_DEFAULT_NAME=default EDC_DATASOURCE_DEFAULT_URL=$MY_EDC_JDBC_URL EDC_DATASOURCE_DEFAULT_USER=$MY_EDC_JDBC_USER From cec91ffe7b09637fd56c596e1fede8382bb8d263 Mon Sep 17 00:00:00 2001 From: Richard Treier Date: Wed, 24 Jul 2024 14:00:13 +0300 Subject: [PATCH 21/24] chore: remove unsused code --- .../edc/ext/wrapper/WrapperExtensionContextBuilder.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java index 3f64c46b0..4091df13d 100644 --- a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java @@ -78,8 +78,6 @@ import de.sovity.edc.ext.wrapper.api.usecase.services.SupportedPolicyApiService; import de.sovity.edc.ext.wrapper.controller.PlaceholderEndpointController; import de.sovity.edc.extension.contacttermination.ContractAgreementTerminationService; -import de.sovity.edc.extension.contacttermination.query.ContractAgreementTerminationDetailsQuery; -import de.sovity.edc.extension.contacttermination.query.TerminateContractQuery; import de.sovity.edc.extension.db.directaccess.DslContextFactory; import de.sovity.edc.utils.catalog.DspCatalogService; import de.sovity.edc.utils.catalog.mapper.DspDataOfferBuilder; @@ -214,8 +212,6 @@ public static WrapperExtensionContext buildContext( transferRequestBuilder, transferProcessService ); - var agreementDetailsQuery = new ContractAgreementTerminationDetailsQuery(); - var terminateContractQuery = new TerminateContractQuery(); var contractAgreementTerminationApiService = new ContractAgreementTerminationApiService(contractAgreementTerminationService); var legacyPolicyMapper = new LegacyPolicyMapper(); var policyDefinitionApiService = new PolicyDefinitionApiService( From 685d8a4c1cec9f77450021869d0d2bc2e14b5acb Mon Sep 17 00:00:00 2001 From: Richard Treier Date: Wed, 24 Jul 2024 14:12:08 +0300 Subject: [PATCH 22/24] chore: simplify test --- build.gradle.kts | 4 +++ docs/api/sovity-edc-api-wrapper.yaml | 4 +-- .../catalog-crawler-e2e-test/build.gradle.kts | 1 - .../build.gradle.kts | 1 - .../catalog-crawler/build.gradle.kts | 1 - .../contract-termination/build.gradle.kts | 1 - .../database-direct-access/build.gradle.kts | 1 - extensions/edc-ui-config/build.gradle.kts | 1 - extensions/last-commit-info/build.gradle.kts | 1 - .../policy-always-true/build.gradle.kts | 1 - .../build.gradle.kts | 1 - .../policy-time-interval/build.gradle.kts | 2 -- .../postgres-flyway-core/build.gradle.kts | 2 -- extensions/postgres-flyway/build.gradle.kts | 1 - extensions/sovity-messenger/build.gradle.kts | 1 - .../test-backend-controller/build.gradle.kts | 1 - .../build.gradle.kts | 1 - .../wrapper/wrapper-api/build.gradle.kts | 1 - .../wrapper-common-mappers/build.gradle.kts | 1 - extensions/wrapper/wrapper/build.gradle.kts | 1 - .../PlaceholderDataSourceExtensionTest.java | 30 +++---------------- utils/catalog-parser/build.gradle.kts | 1 - utils/test-utils/build.gradle.kts | 1 - 23 files changed, 10 insertions(+), 50 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index f9ff62348..9ee6fa69e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -45,6 +45,10 @@ allprojects { apply(plugin = "java") apply(plugin = "checkstyle") + configurations.all { + resolutionStrategy.force("org.eclipse.edc:runtime-metamodel:0.2.1") + } + tasks.withType { options.encoding = "UTF-8" sourceCompatibility = JavaVersion.VERSION_17.toString() diff --git a/docs/api/sovity-edc-api-wrapper.yaml b/docs/api/sovity-edc-api-wrapper.yaml index 4440e71de..3f9ca3a7f 100644 --- a/docs/api/sovity-edc-api-wrapper.yaml +++ b/docs/api/sovity-edc-api-wrapper.yaml @@ -584,11 +584,11 @@ components: DataSourceType: type: string description: Supported Data Source Types by UiDataSource + default: CUSTOM enum: - HTTP_DATA - ON_REQUEST - CUSTOM - default: CUSTOM SecretValue: type: object properties: @@ -796,6 +796,7 @@ components: UiDataSourceHttpDataMethod: type: string description: Supported HTTP Methods by UiDataSource + default: GET enum: - GET - POST @@ -803,7 +804,6 @@ components: - PATCH - DELETE - OPTIONS - default: GET UiDataSourceOnRequest: required: - contactEmail diff --git a/extensions/catalog-crawler/catalog-crawler-e2e-test/build.gradle.kts b/extensions/catalog-crawler/catalog-crawler-e2e-test/build.gradle.kts index 1b4f3c7a4..9f3492139 100644 --- a/extensions/catalog-crawler/catalog-crawler-e2e-test/build.gradle.kts +++ b/extensions/catalog-crawler/catalog-crawler-e2e-test/build.gradle.kts @@ -19,7 +19,6 @@ dependencies { testImplementation(libs.assertj.core) testImplementation(libs.mockito.core) testImplementation(libs.edc.junit) - testImplementation(libs.edc.runtimeMetamodel) testImplementation(libs.restAssured.restAssured) testImplementation(libs.testcontainers.testcontainers) testImplementation(libs.flyway.core) diff --git a/extensions/catalog-crawler/catalog-crawler-launcher-base/build.gradle.kts b/extensions/catalog-crawler/catalog-crawler-launcher-base/build.gradle.kts index 3cf6464c7..2c376a085 100644 --- a/extensions/catalog-crawler/catalog-crawler-launcher-base/build.gradle.kts +++ b/extensions/catalog-crawler/catalog-crawler-launcher-base/build.gradle.kts @@ -11,7 +11,6 @@ dependencies { api(libs.edc.http) api(libs.edc.dsp) api(libs.edc.jsonLd) - implementation(libs.edc.runtimeMetamodel) // Data Catalog Crawler api(project(":extensions:catalog-crawler:catalog-crawler")) diff --git a/extensions/catalog-crawler/catalog-crawler/build.gradle.kts b/extensions/catalog-crawler/catalog-crawler/build.gradle.kts index 829822d92..7c2b46f7e 100644 --- a/extensions/catalog-crawler/catalog-crawler/build.gradle.kts +++ b/extensions/catalog-crawler/catalog-crawler/build.gradle.kts @@ -8,7 +8,6 @@ dependencies { implementation(libs.edc.controlPlaneSpi) implementation(libs.edc.managementApiConfiguration) - implementation(libs.edc.runtimeMetamodel) implementation(libs.quartz.quartz) implementation(libs.apache.commonsLang) diff --git a/extensions/contract-termination/build.gradle.kts b/extensions/contract-termination/build.gradle.kts index 44eb0cdb8..a220fc241 100644 --- a/extensions/contract-termination/build.gradle.kts +++ b/extensions/contract-termination/build.gradle.kts @@ -14,7 +14,6 @@ dependencies { implementation(libs.edc.coreSpi) implementation(libs.edc.dspNegotiationTransform) - implementation(libs.edc.runtimeMetamodel) implementation(libs.edc.transferSpi) implementation(libs.jakarta.rsApi) diff --git a/extensions/database-direct-access/build.gradle.kts b/extensions/database-direct-access/build.gradle.kts index 61f5235b9..3aecfb4d1 100644 --- a/extensions/database-direct-access/build.gradle.kts +++ b/extensions/database-direct-access/build.gradle.kts @@ -9,7 +9,6 @@ dependencies { compileOnly(libs.lombok) implementation(libs.edc.coreSpi) - implementation(libs.edc.runtimeMetamodel) implementation(libs.jooq.jooq) implementation(libs.hikari) diff --git a/extensions/edc-ui-config/build.gradle.kts b/extensions/edc-ui-config/build.gradle.kts index a6a3cf2ed..1f193451f 100644 --- a/extensions/edc-ui-config/build.gradle.kts +++ b/extensions/edc-ui-config/build.gradle.kts @@ -9,7 +9,6 @@ dependencies { api(libs.edc.controlPlaneSpi) implementation(libs.edc.apiCore) implementation(libs.edc.managementApiConfiguration) - implementation(libs.edc.runtimeMetamodel) implementation(libs.jakarta.rsApi) implementation(libs.jakarta.validationApi) diff --git a/extensions/last-commit-info/build.gradle.kts b/extensions/last-commit-info/build.gradle.kts index f282295fb..eac954d07 100644 --- a/extensions/last-commit-info/build.gradle.kts +++ b/extensions/last-commit-info/build.gradle.kts @@ -12,7 +12,6 @@ dependencies { api(libs.edc.controlPlaneSpi) implementation(libs.edc.apiCore) implementation(libs.edc.managementApiConfiguration) - implementation(libs.edc.runtimeMetamodel) implementation(libs.jakarta.rsApi) implementation(libs.jakarta.validationApi) diff --git a/extensions/policy-always-true/build.gradle.kts b/extensions/policy-always-true/build.gradle.kts index 44267ab67..dd96f4f1a 100644 --- a/extensions/policy-always-true/build.gradle.kts +++ b/extensions/policy-always-true/build.gradle.kts @@ -9,7 +9,6 @@ dependencies { api(libs.edc.policyEngineSpi) api(libs.edc.controlPlaneSpi) implementation(libs.edc.apiCore) - implementation(libs.edc.runtimeMetamodel) testImplementation(libs.edc.controlPlaneCore) testImplementation(libs.edc.junit) diff --git a/extensions/policy-referring-connector/build.gradle.kts b/extensions/policy-referring-connector/build.gradle.kts index e4fe94431..1df2e5ac9 100644 --- a/extensions/policy-referring-connector/build.gradle.kts +++ b/extensions/policy-referring-connector/build.gradle.kts @@ -8,7 +8,6 @@ dependencies { api(libs.edc.authSpi) api(libs.edc.policyEngineSpi) api(libs.edc.contractSpi) - implementation(libs.edc.runtimeMetamodel) testImplementation(libs.edc.junit) diff --git a/extensions/policy-time-interval/build.gradle.kts b/extensions/policy-time-interval/build.gradle.kts index 7725750a7..a469643c7 100644 --- a/extensions/policy-time-interval/build.gradle.kts +++ b/extensions/policy-time-interval/build.gradle.kts @@ -8,8 +8,6 @@ dependencies { api(libs.edc.authSpi) api(libs.edc.policyEngineSpi) - implementation(libs.edc.runtimeMetamodel) - testImplementation(libs.edc.junit) } diff --git a/extensions/postgres-flyway-core/build.gradle.kts b/extensions/postgres-flyway-core/build.gradle.kts index c7b87ddd4..5e8b2ef2f 100644 --- a/extensions/postgres-flyway-core/build.gradle.kts +++ b/extensions/postgres-flyway-core/build.gradle.kts @@ -13,8 +13,6 @@ dependencies { api(libs.hikari) implementation(libs.apache.commonsLang) - - implementation(libs.edc.runtimeMetamodel) } group = libs.versions.sovityEdcExtensionGroup.get() diff --git a/extensions/postgres-flyway/build.gradle.kts b/extensions/postgres-flyway/build.gradle.kts index 6efbf0fe6..96b90d1b5 100644 --- a/extensions/postgres-flyway/build.gradle.kts +++ b/extensions/postgres-flyway/build.gradle.kts @@ -13,7 +13,6 @@ dependencies { // Adds Database-Related EDC-Extensions (EDC-SQL-Stores, JDBC-Driver, Pool and Transactions) implementation(libs.edc.controlPlaneSql) - implementation(libs.edc.runtimeMetamodel) implementation(libs.edc.transactionLocal) implementation(libs.tractus.sqlPool) diff --git a/extensions/sovity-messenger/build.gradle.kts b/extensions/sovity-messenger/build.gradle.kts index 63701b8b3..b47765a69 100644 --- a/extensions/sovity-messenger/build.gradle.kts +++ b/extensions/sovity-messenger/build.gradle.kts @@ -15,7 +15,6 @@ dependencies { implementation(libs.edc.dspHttpSpi) implementation(libs.edc.httpSpi) implementation(libs.edc.managementApiConfiguration) - implementation(libs.edc.runtimeMetamodel) implementation(libs.edc.transformCore) testAnnotationProcessor(libs.lombok) diff --git a/extensions/test-backend-controller/build.gradle.kts b/extensions/test-backend-controller/build.gradle.kts index 7dc3ae9ff..9e5364de4 100644 --- a/extensions/test-backend-controller/build.gradle.kts +++ b/extensions/test-backend-controller/build.gradle.kts @@ -6,7 +6,6 @@ dependencies { api(libs.edc.apiCore) api(libs.edc.coreSpi) api(libs.edc.http) - implementation(libs.edc.runtimeMetamodel) } group = libs.versions.sovityEdcExtensionGroup.get() diff --git a/extensions/transfer-process-status-checker/build.gradle.kts b/extensions/transfer-process-status-checker/build.gradle.kts index bba0e69bb..e420cbdaf 100644 --- a/extensions/transfer-process-status-checker/build.gradle.kts +++ b/extensions/transfer-process-status-checker/build.gradle.kts @@ -5,7 +5,6 @@ plugins { dependencies { api(libs.edc.transferSpi) - implementation(libs.edc.runtimeMetamodel) testImplementation(libs.edc.junit) } diff --git a/extensions/wrapper/wrapper-api/build.gradle.kts b/extensions/wrapper/wrapper-api/build.gradle.kts index dd0c50142..48644fa8b 100644 --- a/extensions/wrapper/wrapper-api/build.gradle.kts +++ b/extensions/wrapper/wrapper-api/build.gradle.kts @@ -16,7 +16,6 @@ dependencies { api(project(":extensions:wrapper:wrapper-ee-api")) implementation(libs.apache.commonsLang) - implementation(libs.edc.runtimeMetamodel) implementation(libs.hibernate.validation) implementation(libs.jakarta.el) implementation(libs.jakarta.rsApi) diff --git a/extensions/wrapper/wrapper-common-mappers/build.gradle.kts b/extensions/wrapper/wrapper-common-mappers/build.gradle.kts index 5fbc898e8..909ba8dbe 100644 --- a/extensions/wrapper/wrapper-common-mappers/build.gradle.kts +++ b/extensions/wrapper/wrapper-common-mappers/build.gradle.kts @@ -18,7 +18,6 @@ dependencies { implementation(libs.apache.commonsLang) implementation(libs.apache.commonsCollections) - implementation(libs.edc.runtimeMetamodel) implementation(libs.flexmark.all) implementation(libs.okhttp.okhttp) diff --git a/extensions/wrapper/wrapper/build.gradle.kts b/extensions/wrapper/wrapper/build.gradle.kts index 90de0d990..027f3c567 100644 --- a/extensions/wrapper/wrapper/build.gradle.kts +++ b/extensions/wrapper/wrapper/build.gradle.kts @@ -29,7 +29,6 @@ dependencies { implementation(libs.edc.managementApiConfiguration) implementation(libs.edc.dspApiConfiguration) implementation(libs.edc.dspHttpSpi) - implementation(libs.edc.runtimeMetamodel) implementation(libs.jooq.jooq) implementation(libs.hibernate.validation) implementation(libs.hikari) diff --git a/tests/src/test/java/de/sovity/edc/e2e/PlaceholderDataSourceExtensionTest.java b/tests/src/test/java/de/sovity/edc/e2e/PlaceholderDataSourceExtensionTest.java index c1e264cd3..42a5c740c 100644 --- a/tests/src/test/java/de/sovity/edc/e2e/PlaceholderDataSourceExtensionTest.java +++ b/tests/src/test/java/de/sovity/edc/e2e/PlaceholderDataSourceExtensionTest.java @@ -58,8 +58,6 @@ class PlaceholderDataSourceExtensionTest { void shouldAccessDummyEndpoint( E2eScenario scenario, ClientAndServer clientAndServer, - @Provider EdcExtension providerExtension, - @Provider ConnectorConfig providerConfig, @Provider EdcClient providerClient ) { // arrange @@ -80,14 +78,6 @@ void shouldAccessDummyEndpoint( .build() ); - val a = providerClient.uiApi().getAssetPage().getAssets().get(0).getAssetJsonLd(); - val compacted = JsonLdUtils.tryCompact(JsonUtils.parseJsonObj(a)); - val dataAddress = compacted.getJsonObject(Prop.Edc.DATA_ADDRESS); - val baseUrl = dataAddress.getString(Prop.Edc.BASE_URL); - - val service = providerExtension.getContext().getService(PlaceholderEndpointService.class); - val expected = service.getPlaceholderEndpointForAsset(email, subject); - scenario.createContractDefinition(assetId); val negotiation = scenario.negotiateAssetAndAwait(assetId); @@ -111,21 +101,9 @@ void shouldAccessDummyEndpoint( .build()); // assert - assertThat(baseUrl) - .startsWith(providerConfig.getProperties().get(WrapperExtension.MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL)) - .isEqualTo(expected); - - val request = new Request.Builder() - .url(baseUrl) - .build(); - - val client = new OkHttpClient(); - val content = client.newCall(request).execute().body().string(); - - assertThat(content).contains("This is not real data."); - assertThat(content).contains(email); - assertThat(content).contains(subject); - - assertThat(new String(Base64.getDecoder().decode(accessed.get()))).contains("This is not real data."); + assertThat(new String(Base64.getDecoder().decode(accessed.get()))) + .contains("This is not real data.") + .contains(email) + .contains(subject); } } diff --git a/utils/catalog-parser/build.gradle.kts b/utils/catalog-parser/build.gradle.kts index 3527b383e..f5d8fc433 100644 --- a/utils/catalog-parser/build.gradle.kts +++ b/utils/catalog-parser/build.gradle.kts @@ -18,7 +18,6 @@ dependencies { implementation(libs.apache.commonsLang) implementation(libs.apache.commonsCollections) implementation(libs.apache.commonsIo) - implementation(libs.edc.runtimeMetamodel) testAnnotationProcessor(libs.lombok) diff --git a/utils/test-utils/build.gradle.kts b/utils/test-utils/build.gradle.kts index cf0d63eed..b6b1eab88 100644 --- a/utils/test-utils/build.gradle.kts +++ b/utils/test-utils/build.gradle.kts @@ -20,7 +20,6 @@ dependencies { implementation(project(":utils:versions")) implementation(libs.edc.jsonLdSpi) implementation(libs.edc.jsonLd) - implementation(libs.edc.runtimeMetamodel) implementation(libs.edc.sqlCore) implementation(libs.assertj.core) implementation(libs.jooq.jooq) From a771bc31fc17bfeb6c16f4306d70c77d217b11bc Mon Sep 17 00:00:00 2001 From: Richard Treier Date: Wed, 24 Jul 2024 14:16:55 +0300 Subject: [PATCH 23/24] chore: self-review --- .../main/java/de/sovity/edc/ext/wrapper/WrapperExtension.java | 2 +- .../sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java | 2 +- launchers/.env.connector | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtension.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtension.java index ba56efc1f..3c0f81079 100644 --- a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtension.java +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtension.java @@ -46,7 +46,7 @@ public class WrapperExtension implements ServiceExtension { - @Setting(value = "Defaults to tne env var EDC_DSP_CALLBACK_ADDRESS") + @Setting(value = "Base URL for the On Request asset datasource, as reachable by the data plane") public static final String MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL = "my.edc.datasource.placeholder.baseurl"; public static final String EXTENSION_NAME = "WrapperExtension"; diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java index 4091df13d..48574717f 100644 --- a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java @@ -148,7 +148,7 @@ public static WrapperExtensionContext buildContext( var selfDescriptionService = new SelfDescriptionService(config, monitor); var ownConnectorEndpointService = new OwnConnectorEndpointServiceImpl(selfDescriptionService); var placeholderEndpointService = new PlaceholderEndpointService( - config.getString(WrapperExtension.MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL) + config.getString(WrapperExtension.MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL, "http://0.0.0.0") ); var assetMapper = newAssetMapper(typeTransformerRegistry, jsonLd, ownConnectorEndpointService, placeholderEndpointService); var policyMapper = newPolicyMapper(objectMapper, typeTransformerRegistry, operatorMapper); diff --git a/launchers/.env.connector b/launchers/.env.connector index 5a723560a..5b2eb4284 100644 --- a/launchers/.env.connector +++ b/launchers/.env.connector @@ -97,4 +97,5 @@ EDC_AGENT_IDENTITY_KEY=referringConnector # it is created in the Dockerfile EDC_VAULT=/app/empty-properties-file.properties +# Base URL for the On Request asset datasource, as reachable by the data plane MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL=${EDC_DSP_CALLBACK_ADDRESS} From 82bbd5381c3c013b19ecc3699ea919776600dd34 Mon Sep 17 00:00:00 2001 From: Richard Treier Date: Wed, 24 Jul 2024 14:20:23 +0300 Subject: [PATCH 24/24] chore: fix ci --- .../edc/e2e/PlaceholderDataSourceExtensionTest.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/src/test/java/de/sovity/edc/e2e/PlaceholderDataSourceExtensionTest.java b/tests/src/test/java/de/sovity/edc/e2e/PlaceholderDataSourceExtensionTest.java index 42a5c740c..5e6fb8b7f 100644 --- a/tests/src/test/java/de/sovity/edc/e2e/PlaceholderDataSourceExtensionTest.java +++ b/tests/src/test/java/de/sovity/edc/e2e/PlaceholderDataSourceExtensionTest.java @@ -20,21 +20,12 @@ import de.sovity.edc.client.gen.model.UiAssetCreateRequest; import de.sovity.edc.client.gen.model.UiDataSource; import de.sovity.edc.client.gen.model.UiDataSourceOnRequest; -import de.sovity.edc.ext.wrapper.WrapperExtension; -import de.sovity.edc.ext.wrapper.api.common.mappers.PlaceholderEndpointService; -import de.sovity.edc.extension.e2e.connector.config.ConnectorConfig; import de.sovity.edc.extension.e2e.extension.E2eScenario; import de.sovity.edc.extension.e2e.extension.E2eTestExtension; import de.sovity.edc.extension.e2e.extension.Provider; -import de.sovity.edc.utils.JsonUtils; -import de.sovity.edc.utils.jsonld.JsonLdUtils; -import de.sovity.edc.utils.jsonld.vocab.Prop; import jakarta.ws.rs.HttpMethod; import lombok.SneakyThrows; import lombok.val; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import org.eclipse.edc.junit.extensions.EdcExtension; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockserver.integration.ClientAndServer; @@ -44,7 +35,6 @@ import java.util.Base64; import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import static org.assertj.core.api.Assertions.assertThat;