feat: on request assets now return placeholder data (sovity#1004)

Co-authored-by: Richard Treier <[email protected]>
truzzt · Jul 30, 2024 · 11cc466 · 11cc466
1 parent 8df1ecd
commit 11cc466
Show file tree

Hide file tree

Showing 37 changed files with 296 additions and 83 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -24,12 +24,6 @@ please see [changelog_updates.md](docs/dev/changelog_updates.md).
 
 ### Deployment Migration Notes
 
-New configuration to access the database:
-
-- `EDC_SERVER_DB_CONNECTION_POOL_SIZE`
-  - The property controls the maximum size that the pool is allowed to reach, including both idle and in-use connections. Basically this value will determine the maximum number of actual connections to the database backend.
-  - Defaults to `3`
-
 #### Compatible Versions
 
 - Connector Backend Docker Images:

diff --git a/build.gradle.kts b/build.gradle.kts
@@ -45,6 +45,10 @@ allprojects {
     apply(plugin = "java")
     apply(plugin = "checkstyle")
 
+    configurations.all {
+        resolutionStrategy.force("org.eclipse.edc:runtime-metamodel:0.2.1")
+    }
+
     tasks.withType<JavaCompile> {
         options.encoding = "UTF-8"
         sourceCompatibility = JavaVersion.VERSION_17.toString()

diff --git a/...alog-crawler-e2e-test/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerE2eTest.java b/...alog-crawler-e2e-test/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerE2eTest.java
@@ -106,6 +106,7 @@ class CrawlerE2eTest {
                 props.put(CrawlerExtension.CRON_DEAD_CONNECTOR_REFRESH, everySeconds);
                 props.put(CrawlerExtension.SCHEDULED_KILL_OFFLINE_CONNECTORS, everySeconds);
                 props.put(CrawlerExtension.KILL_OFFLINE_CONNECTORS_AFTER, "P1D");
+                props.put("my.edc.datasource.placeholder.baseurl", "http://example.com/edc/backend");
 
                 return props;
             }

diff --git a/...ler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java b/...ler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java
@@ -14,6 +14,7 @@
 
 package de.sovity.edc.ext.catalog.crawler;
 
+import de.sovity.edc.ext.wrapper.api.common.mappers.PlaceholderEndpointService;
 import org.eclipse.edc.connector.api.management.configuration.transform.ManagementApiTypeTransformerRegistry;
 import org.eclipse.edc.connector.spi.catalog.CatalogService;
 import org.eclipse.edc.jsonld.spi.JsonLd;
@@ -117,12 +118,13 @@ public void initialize(ServiceExtensionContext context) {
         }
 
         services = CrawlerExtensionContextBuilder.buildContext(
-                context.getConfig(),
-                context.getMonitor(),
-                typeManager,
-                typeTransformerRegistry,
-                jsonLd,
-                catalogService
+            context.getConfig(),
+            context.getMonitor(),
+            typeManager,
+            typeTransformerRegistry,
+            jsonLd,
+            catalogService,
+            new PlaceholderEndpointService("http://0.0.0.0/")
         );
 
         // Provide access for the tests

diff --git a/...awler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java b/...awler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java
@@ -52,6 +52,7 @@
 import de.sovity.edc.ext.catalog.crawler.orchestration.schedules.QuartzScheduleInitializer;
 import de.sovity.edc.ext.catalog.crawler.orchestration.schedules.utils.CronJobRef;
 import de.sovity.edc.ext.wrapper.api.common.mappers.AssetMapper;
+import de.sovity.edc.ext.wrapper.api.common.mappers.PlaceholderEndpointService;
 import de.sovity.edc.ext.wrapper.api.common.mappers.PolicyMapper;
 import de.sovity.edc.ext.wrapper.api.common.mappers.asset.AssetEditRequestMapper;
 import de.sovity.edc.ext.wrapper.api.common.mappers.asset.AssetJsonLdBuilder;
@@ -101,7 +102,8 @@ public static CrawlerExtensionContext buildContext(
         TypeManager typeManager,
         TypeTransformerRegistry typeTransformerRegistry,
         JsonLd jsonLd,
-        CatalogService catalogService
+        CatalogService catalogService,
+        PlaceholderEndpointService placeholderEndpointService
     ) {
         // Config
         var crawlerConfigFactory = new CrawlerConfigFactory(config);
@@ -120,7 +122,7 @@ public static CrawlerExtensionContext buildContext(
 
         // Services
         var objectMapperJsonLd = getJsonLdObjectMapper(typeManager);
-        var assetMapper = newAssetMapper(typeTransformerRegistry, jsonLd);
+        var assetMapper = newAssetMapper(typeTransformerRegistry, jsonLd, placeholderEndpointService);
         var policyMapper = newPolicyMapper(typeTransformerRegistry, objectMapperJsonLd);
         var crawlerEventLogger = new CrawlerEventLogger();
         var crawlerExecutionTimeLogger = new CrawlerExecutionTimeLogger();
@@ -229,7 +231,8 @@ private static PolicyMapper newPolicyMapper(
     @NotNull
     private static AssetMapper newAssetMapper(
         TypeTransformerRegistry typeTransformerRegistry,
-        JsonLd jsonLd
+        JsonLd jsonLd,
+        PlaceholderEndpointService placeholderEndpointService
     ) {
         var edcPropertyUtils = new EdcPropertyUtils();
         var assetJsonLdUtils = new AssetJsonLdUtils();
@@ -241,7 +244,7 @@ private static AssetMapper newAssetMapper(
             endpoint -> false
         );
         var httpHeaderMapper = new HttpHeaderMapper();
-        var httpDataSourceMapper = new HttpDataSourceMapper(httpHeaderMapper);
+        var httpDataSourceMapper = new HttpDataSourceMapper(httpHeaderMapper, placeholderEndpointService);
         var dataSourceMapper = new DataSourceMapper(
             edcPropertyUtils,
             httpDataSourceMapper

diff --git a/extensions/contract-termination/build.gradle.kts b/extensions/contract-termination/build.gradle.kts
@@ -13,8 +13,8 @@ dependencies {
     implementation(project(":extensions:sovity-messenger"))
 
     implementation(libs.edc.coreSpi)
-    implementation(libs.edc.transferSpi)
     implementation(libs.edc.dspNegotiationTransform)
+    implementation(libs.edc.transferSpi)
 
     implementation(libs.jakarta.rsApi)
 

diff --git a/...est/java/de/sovity/edc/extension/contacttermination/query/TerminateContractQueryTest.java b/...est/java/de/sovity/edc/extension/contacttermination/query/TerminateContractQueryTest.java
@@ -21,6 +21,7 @@
 import de.sovity.edc.extension.e2e.extension.E2eScenario;
 import de.sovity.edc.extension.e2e.extension.E2eTestExtension;
 import de.sovity.edc.extension.e2e.extension.Provider;
+import de.sovity.edc.extension.utils.junit.DisabledOnGithub;
 import lombok.val;
 import org.eclipse.edc.connector.contract.spi.types.negotiation.ContractNegotiation;
 import org.junit.jupiter.api.Test;
@@ -34,6 +35,7 @@
 @ExtendWith(E2eTestExtension.class)
 class TerminateContractQueryTest {
 
+    @DisabledOnGithub
     @Test
     void terminateConsumerAgreementOrThrow_shouldInsertRowInTerminationTable(
         E2eScenario scenario,

diff --git a/extensions/policy-referring-connector/build.gradle.kts b/extensions/policy-referring-connector/build.gradle.kts
@@ -8,6 +8,8 @@ dependencies {
     api(libs.edc.authSpi)
     api(libs.edc.policyEngineSpi)
     api(libs.edc.contractSpi)
+
+
     testImplementation(libs.edc.junit)
 
     testImplementation(libs.mockito.core)

diff --git a/extensions/policy-time-interval/build.gradle.kts b/extensions/policy-time-interval/build.gradle.kts
@@ -7,6 +7,8 @@ plugins {
 dependencies {
     api(libs.edc.authSpi)
     api(libs.edc.policyEngineSpi)
+
+
     testImplementation(libs.edc.junit)
 }
 

diff --git a/extensions/postgres-flyway/build.gradle.kts b/extensions/postgres-flyway/build.gradle.kts
@@ -14,6 +14,7 @@ dependencies {
     // Adds Database-Related EDC-Extensions (EDC-SQL-Stores, JDBC-Driver, Pool and Transactions)
     implementation(libs.edc.controlPlaneSql)
     implementation(libs.edc.transactionLocal)
+
     implementation(libs.tractus.sqlPool)
 
     implementation(libs.apache.commonsLang)

diff --git a/extensions/sovity-messenger/build.gradle.kts b/extensions/sovity-messenger/build.gradle.kts
@@ -17,7 +17,6 @@ dependencies {
     implementation(libs.edc.managementApiConfiguration)
     implementation(libs.edc.transformCore)
 
-
     testAnnotationProcessor(libs.lombok)
 
     testCompileOnly(libs.lombok)

diff --git a/...-api/src/main/java/de/sovity/edc/ext/wrapper/api/ui/model/ContractAgreementPageQuery.java b/...-api/src/main/java/de/sovity/edc/ext/wrapper/api/ui/model/ContractAgreementPageQuery.java
@@ -14,7 +14,6 @@
 
 package de.sovity.edc.ext.wrapper.api.ui.model;
 
-import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.AllArgsConstructor;

diff --git a/...pper/wrapper-api/src/main/java/de/sovity/edc/ext/wrapper/api/usecase/UseCaseResource.java b/...pper/wrapper-api/src/main/java/de/sovity/edc/ext/wrapper/api/usecase/UseCaseResource.java
@@ -19,7 +19,6 @@
 import de.sovity.edc.ext.wrapper.api.usecase.model.KpiResult;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.tags.Tag;
-import jakarta.validation.Valid;
 import jakarta.validation.constraints.NotNull;
 import jakarta.ws.rs.Consumes;
 import jakarta.ws.rs.GET;

diff --git a/extensions/wrapper/wrapper-common-mappers/build.gradle.kts b/extensions/wrapper/wrapper-common-mappers/build.gradle.kts
@@ -12,11 +12,15 @@ dependencies {
     api(libs.edc.coreSpi)
     api(libs.edc.transformCore)
     api(libs.edc.transformSpi)
+
     api(project(":extensions:wrapper:wrapper-common-api"))
     api(project(":utils:json-and-jsonld-utils"))
+
     implementation(libs.apache.commonsLang)
     implementation(libs.apache.commonsCollections)
     implementation(libs.flexmark.all)
+    implementation(libs.okhttp.okhttp)
+
 
     testAnnotationProcessor(libs.lombok)
     testCompileOnly(libs.lombok)

diff --git a/...rc/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/PlaceholderEndpointService.java b/...rc/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/PlaceholderEndpointService.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2024 sovity GmbH
+ *
+ *  This program and the accompanying materials are made available under the
+ *  terms of the Apache License, Version 2.0 which is available at
+ *  https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *
+ *  Contributors:
+ *       sovity GmbH - initial API and implementation
+ *
+ */
+
+package de.sovity.edc.ext.wrapper.api.common.mappers;
+
+import lombok.RequiredArgsConstructor;
+import okhttp3.HttpUrl;
+
+@RequiredArgsConstructor
+public class PlaceholderEndpointService {
+
+    private final String baseUrl;
+
+    public static final String DUMMY_ENDPOINT_URL = "/data-source/placeholder/asset";
+
+    public String getPlaceholderEndpointForAsset(String email, String subject) {
+        return HttpUrl.parse(baseUrl + DUMMY_ENDPOINT_URL)
+            .newBuilder()
+            .addQueryParameter("email", email)
+            .addQueryParameter("subject", subject)
+            .build()
+            .toString();
+    }
+}
diff --git a/...a/de/sovity/edc/ext/wrapper/api/common/mappers/dataaddress/http/HttpDataSourceMapper.java b/...a/de/sovity/edc/ext/wrapper/api/common/mappers/dataaddress/http/HttpDataSourceMapper.java
@@ -14,6 +14,7 @@
 
 package de.sovity.edc.ext.wrapper.api.common.mappers.dataaddress.http;
 
+import de.sovity.edc.ext.wrapper.api.common.mappers.PlaceholderEndpointService;
 import de.sovity.edc.ext.wrapper.api.common.model.UiDataSourceHttpData;
 import de.sovity.edc.ext.wrapper.api.common.model.UiDataSourceOnRequest;
 import de.sovity.edc.utils.jsonld.vocab.Prop;
@@ -33,6 +34,7 @@
 @RequiredArgsConstructor
 public class HttpDataSourceMapper {
     private final HttpHeaderMapper httpHeaderMapper;
+    private final PlaceholderEndpointService placeholderEndpointService;
 
     /**
      * Data Address for type HTTP_DATA
@@ -92,8 +94,12 @@ public Map<String, String> buildOnRequestDataAddress(@NonNull UiDataSourceOnRequ
             "Need contactPreferredEmailSubject"
         );
 
+        var placeholderEndpointForAsset = placeholderEndpointService.getPlaceholderEndpointForAsset(
+            onRequest.getContactEmail(),
+            onRequest.getContactPreferredEmailSubject());
+
         var actualDataSource = UiDataSourceHttpData.builder()
-            .baseUrl("http://0.0.0.0")
+            .baseUrl(placeholderEndpointForAsset)
             .build();
 
         var props = buildDataAddress(actualDataSource);

diff --git a/...er-common-mappers/src/test/java/de/sovity/edc/ext/wrapper/api/common/mappers/Factory.java b/...er-common-mappers/src/test/java/de/sovity/edc/ext/wrapper/api/common/mappers/Factory.java
@@ -50,7 +50,7 @@ public static AssetJsonLdBuilder newAssetJsonLdBuilder(OwnConnectorEndpointServi
         return new AssetJsonLdBuilder(
             new DataSourceMapper(
                 new EdcPropertyUtils(),
-                new HttpDataSourceMapper(new HttpHeaderMapper())
+                new HttpDataSourceMapper(new HttpHeaderMapper(), new PlaceholderEndpointService("http://example.com/dummy/baseUrl"))
             ),
             newAssetJsonLdParser(ownConnectorEndpointService),
             new AssetEditRequestMapper()

diff --git a/.../test/java/de/sovity/edc/ext/wrapper/api/common/mappers/asset/AssetJsonLdBuilderTest.java b/.../test/java/de/sovity/edc/ext/wrapper/api/common/mappers/asset/AssetJsonLdBuilderTest.java
@@ -534,7 +534,7 @@ void test_create_onRequest() {
         var dataAddress = Json.createObjectBuilder()
             .add(Prop.TYPE, Prop.Edc.TYPE_DATA_ADDRESS)
             .add(Prop.Edc.TYPE, Prop.Edc.DATA_ADDRESS_TYPE_HTTP_DATA)
-            .add(Prop.Edc.BASE_URL, "http://0.0.0.0")
+            .add(Prop.Edc.BASE_URL, "http://example.com/dummy/baseUrl/data-source/placeholder/asset?email=contact%40example.com&subject=Test")
             .add(Prop.SovityDcatExt.DATA_SOURCE_AVAILABILITY, Prop.SovityDcatExt.DATA_SOURCE_AVAILABILITY_ON_REQUEST)
             .add(Prop.SovityDcatExt.CONTACT_EMAIL, "[email protected]")
             .add(Prop.SovityDcatExt.CONTACT_PREFERRED_EMAIL_SUBJECT, "Test");

diff --git a/extensions/wrapper/wrapper/build.gradle.kts b/extensions/wrapper/wrapper/build.gradle.kts
@@ -27,6 +27,7 @@ dependencies {
     implementation(libs.apache.commonsLang)
     implementation(libs.edc.apiCore)
     implementation(libs.edc.managementApiConfiguration)
+    implementation(libs.edc.dspApiConfiguration)
     implementation(libs.edc.dspHttpSpi)
     implementation(libs.jooq.jooq)
     implementation(libs.hibernate.validation)

diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtension.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtension.java
@@ -19,7 +19,6 @@
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import de.sovity.edc.extension.contacttermination.ContractAgreementTerminationService;
 import de.sovity.edc.extension.db.directaccess.DslContextFactory;
-import de.sovity.edc.extension.messenger.SovityMessenger;
 import org.eclipse.edc.connector.api.management.configuration.ManagementApiConfiguration;
 import org.eclipse.edc.connector.api.management.configuration.transform.ManagementApiTypeTransformerRegistry;
 import org.eclipse.edc.connector.contract.spi.negotiation.store.ContractNegotiationStore;
@@ -35,7 +34,9 @@
 import org.eclipse.edc.connector.transfer.spi.store.TransferProcessStore;
 import org.eclipse.edc.jsonld.spi.JsonLd;
 import org.eclipse.edc.policy.engine.spi.PolicyEngine;
+import org.eclipse.edc.protocol.dsp.api.configuration.DspApiConfiguration;
 import org.eclipse.edc.runtime.metamodel.annotation.Inject;
+import org.eclipse.edc.runtime.metamodel.annotation.Setting;
 import org.eclipse.edc.spi.CoreConstants;
 import org.eclipse.edc.spi.asset.AssetIndex;
 import org.eclipse.edc.spi.system.ServiceExtension;
@@ -45,6 +46,8 @@
 
 public class WrapperExtension implements ServiceExtension {
 
+    @Setting(value = "Base URL for the On Request asset datasource, as reachable by the data plane")
+    public static final String MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL = "my.edc.datasource.placeholder.baseurl";
 
     public static final String EXTENSION_NAME = "WrapperExtension";
 
@@ -67,14 +70,14 @@ public class WrapperExtension implements ServiceExtension {
     @Inject
     private DslContextFactory dslContextFactory;
     @Inject
+    private DspApiConfiguration dspApiConfiguration;
+    @Inject
     private ManagementApiConfiguration dataManagementApiConfiguration;
     @Inject
     private PolicyDefinitionStore policyDefinitionStore;
     @Inject
     private PolicyEngine policyEngine;
     @Inject
-    private SovityMessenger sovityMessenger;
-    @Inject
     private TransferProcessService transferProcessService;
     @Inject
     private TransferProcessStore transferProcessStore;
@@ -119,16 +122,18 @@ public void initialize(ServiceExtensionContext context) {
             policyDefinitionService,
             policyDefinitionStore,
             policyEngine,
-            sovityMessenger,
             transferProcessService,
             transferProcessStore,
             typeTransformerRegistry
         );
 
         wrapperExtensionContext.selfDescriptionService().validateSelfDescriptionConfig();
 
-        wrapperExtensionContext.jaxRsResources().forEach(resource ->
+        wrapperExtensionContext.managementApiResources().forEach(resource ->
             webService.registerResource(dataManagementApiConfiguration.getContextAlias(), resource));
+
+        wrapperExtensionContext.dspApiResources().forEach(resource ->
+            webService.registerResource(dspApiConfiguration.getContextAlias(), resource));
     }
 
     private void fixObjectMapperDateSerialization(ObjectMapper objectMapper) {

diff --git a/...ions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContext.java b/...ions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContext.java
@@ -22,13 +22,15 @@
 /**
  * Manual Dependency Injection result
  *
- * @param jaxRsResources         Jax RS Resource implementations to register. Implementations of
+ * @param managementApiResources Jax RS Resource implementations to register. Implementations of
  *                               APIs supported by our EDC API Client that don't have their own
  *                               extension should land here.
+ * @param dspApiResources     Jax RS Resource implementations to register. Rare DSP API
  * @param selfDescriptionService Required here for validation on start-up
  */
 public record WrapperExtensionContext(
-        List<Object> jaxRsResources,
+        List<Object> managementApiResources,
+        List<Object> dspApiResources,
         SelfDescriptionService selfDescriptionService
 ) {
 }