feat: Implement placeholder endpoint

CHANGELOG.md

@@ -29,6 +29,9 @@ New configuration to access the database:
 - `EDC_SERVER_DB_CONNECTION_POOL_SIZE`
   - The property controls the maximum size that the pool is allowed to reach, including both idle and in-use connections. Basically this value will determine the maximum number of actual connections to the database backend.
   - Defaults to `3`
+- `MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL`
+  - Base URL of an asset as seen from an external source, from the asset's consumer (typically from the internet, ...)
+  - It is documented in the asset data placeholder [README.md](extensions/placeholder-data-source/README.md)
 
 #### Compatible Versions
 

docs/api/sovity-edc-api-wrapper.yaml

@@ -584,11 +584,11 @@ components:
     DataSourceType:
       type: string
       description: Supported Data Source Types by UiDataSource
-      default: CUSTOM
       enum:
       - HTTP_DATA
       - ON_REQUEST
       - CUSTOM
+      default: CUSTOM
     SecretValue:
       type: object
       properties:
@@ -796,14 +796,14 @@ components:
     UiDataSourceHttpDataMethod:
       type: string
       description: Supported HTTP Methods by UiDataSource
-      default: GET
       enum:
       - GET
       - POST
       - PUT
       - PATCH
       - DELETE
       - OPTIONS
+      default: GET
     UiDataSourceOnRequest:
       required:
       - contactEmail

extensions/catalog-crawler/catalog-crawler-e2e-test/build.gradle.kts

@@ -19,6 +19,7 @@ dependencies {
     testImplementation(libs.assertj.core)
     testImplementation(libs.mockito.core)
     testImplementation(libs.edc.junit)
+    testImplementation(libs.edc.runtimeMetamodel)
     testImplementation(libs.restAssured.restAssured)
     testImplementation(libs.testcontainers.testcontainers)
     testImplementation(libs.flyway.core)

extensions/catalog-crawler/catalog-crawler-e2e-test/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerE2eTest.java

@@ -38,6 +38,7 @@
 import de.sovity.edc.ext.catalog.crawler.utils.TestData;
 import de.sovity.edc.extension.e2e.connector.config.ConnectorConfig;
 import de.sovity.edc.extension.e2e.db.EdcRuntimeExtensionWithTestDatabase;
+import de.sovity.edc.extension.placeholderdatasource.PlaceholderDataSourceExtension;
 import de.sovity.edc.utils.jsonld.vocab.Prop;
 import org.awaitility.Awaitility;
 import org.awaitility.core.ThrowingRunnable;
@@ -106,6 +107,7 @@ class CrawlerE2eTest {
                 props.put(CrawlerExtension.CRON_DEAD_CONNECTOR_REFRESH, everySeconds);
                 props.put(CrawlerExtension.SCHEDULED_KILL_OFFLINE_CONNECTORS, everySeconds);
                 props.put(CrawlerExtension.KILL_OFFLINE_CONNECTORS_AFTER, "P1D");
+                props.put(PlaceholderDataSourceExtension.MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL, "http://example.com/edc/backend");
 
                 return props;
             }

extensions/catalog-crawler/catalog-crawler-launcher-base/build.gradle.kts

@@ -11,6 +11,7 @@ dependencies {
     api(libs.edc.http)
     api(libs.edc.dsp)
     api(libs.edc.jsonLd)
+    implementation(libs.edc.runtimeMetamodel)
 
     // Data Catalog Crawler
     api(project(":extensions:catalog-crawler:catalog-crawler"))

extensions/catalog-crawler/catalog-crawler/build.gradle.kts

@@ -8,6 +8,7 @@ dependencies {
 
     implementation(libs.edc.controlPlaneSpi)
     implementation(libs.edc.managementApiConfiguration)
+    implementation(libs.edc.runtimeMetamodel)
 
     implementation(libs.quartz.quartz)
     implementation(libs.apache.commonsLang)
@@ -17,6 +18,7 @@ dependencies {
     api(project(":utils:json-and-jsonld-utils"))
     api(project(":extensions:wrapper:wrapper-common-mappers"))
     api(project(":extensions:catalog-crawler:catalog-crawler-db"))
+    api(project(":extensions:placeholder-data-source"))
     api(project(":extensions:postgres-flyway-core"))
 
     testAnnotationProcessor(libs.lombok)

extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java

@@ -14,6 +14,7 @@
 
 package de.sovity.edc.ext.catalog.crawler;
 
+import de.sovity.edc.extension.placeholderdatasource.PlaceholderEndpointService;
 import org.eclipse.edc.connector.api.management.configuration.transform.ManagementApiTypeTransformerRegistry;
 import org.eclipse.edc.connector.spi.catalog.CatalogService;
 import org.eclipse.edc.jsonld.spi.JsonLd;
@@ -90,6 +91,9 @@ public class CrawlerExtension implements ServiceExtension {
     @Inject
     private TypeManager typeManager;
 
+    @Inject
+    private PlaceholderEndpointService placeholderEndpointService;
+
     @Inject
     private ManagementApiTypeTransformerRegistry typeTransformerRegistry;
 
@@ -117,12 +121,13 @@ public void initialize(ServiceExtensionContext context) {
         }
 
         services = CrawlerExtensionContextBuilder.buildContext(
-                context.getConfig(),
-                context.getMonitor(),
-                typeManager,
-                typeTransformerRegistry,
-                jsonLd,
-                catalogService
+            context.getConfig(),
+            context.getMonitor(),
+            typeManager,
+            typeTransformerRegistry,
+            jsonLd,
+            catalogService,
+            placeholderEndpointService
         );
 
         // Provide access for the tests

extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java

@@ -68,6 +68,7 @@
 import de.sovity.edc.ext.wrapper.api.common.mappers.policy.LiteralMapper;
 import de.sovity.edc.ext.wrapper.api.common.mappers.policy.OperatorMapper;
 import de.sovity.edc.ext.wrapper.api.common.mappers.policy.PolicyValidator;
+import de.sovity.edc.extension.placeholderdatasource.PlaceholderEndpointService;
 import de.sovity.edc.utils.catalog.DspCatalogService;
 import de.sovity.edc.utils.catalog.mapper.DspDataOfferBuilder;
 import lombok.NoArgsConstructor;
@@ -101,7 +102,8 @@ public static CrawlerExtensionContext buildContext(
         TypeManager typeManager,
         TypeTransformerRegistry typeTransformerRegistry,
         JsonLd jsonLd,
-        CatalogService catalogService
+        CatalogService catalogService,
+        PlaceholderEndpointService placeholderEndpointService
     ) {
         // Config
         var crawlerConfigFactory = new CrawlerConfigFactory(config);
@@ -120,7 +122,7 @@ public static CrawlerExtensionContext buildContext(
 
         // Services
         var objectMapperJsonLd = getJsonLdObjectMapper(typeManager);
-        var assetMapper = newAssetMapper(typeTransformerRegistry, jsonLd);
+        var assetMapper = newAssetMapper(typeTransformerRegistry, jsonLd, placeholderEndpointService);
         var policyMapper = newPolicyMapper(typeTransformerRegistry, objectMapperJsonLd);
         var crawlerEventLogger = new CrawlerEventLogger();
         var crawlerExecutionTimeLogger = new CrawlerExecutionTimeLogger();
@@ -229,7 +231,8 @@ private static PolicyMapper newPolicyMapper(
     @NotNull
     private static AssetMapper newAssetMapper(
         TypeTransformerRegistry typeTransformerRegistry,
-        JsonLd jsonLd
+        JsonLd jsonLd,
+        PlaceholderEndpointService placeholderEndpointService
     ) {
         var edcPropertyUtils = new EdcPropertyUtils();
         var assetJsonLdUtils = new AssetJsonLdUtils();
@@ -241,7 +244,7 @@ private static AssetMapper newAssetMapper(
             endpoint -> false
         );
         var httpHeaderMapper = new HttpHeaderMapper();
-        var httpDataSourceMapper = new HttpDataSourceMapper(httpHeaderMapper);
+        var httpDataSourceMapper = new HttpDataSourceMapper(httpHeaderMapper, placeholderEndpointService);
         var dataSourceMapper = new DataSourceMapper(
             edcPropertyUtils,
             httpDataSourceMapper

extensions/contract-termination/build.gradle.kts

@@ -13,8 +13,9 @@ dependencies {
     implementation(project(":extensions:sovity-messenger"))
 
     implementation(libs.edc.coreSpi)
-    implementation(libs.edc.transferSpi)
     implementation(libs.edc.dspNegotiationTransform)
+    implementation(libs.edc.runtimeMetamodel)
+    implementation(libs.edc.transferSpi)
 
     implementation(libs.jakarta.rsApi)
 

extensions/contract-termination/src/test/java/de/sovity/edc/extension/contacttermination/query/TerminateContractQueryTest.java

@@ -21,6 +21,7 @@
 import de.sovity.edc.extension.e2e.extension.E2eScenario;
 import de.sovity.edc.extension.e2e.extension.E2eTestExtension;
 import de.sovity.edc.extension.e2e.extension.Provider;
+import de.sovity.edc.extension.utils.junit.DisabledOnGithub;
 import lombok.val;
 import org.eclipse.edc.connector.contract.spi.types.negotiation.ContractNegotiation;
 import org.junit.jupiter.api.Test;
@@ -34,6 +35,7 @@
 @ExtendWith(E2eTestExtension.class)
 class TerminateContractQueryTest {
 
+    @DisabledOnGithub
     @Test
     void terminateConsumerAgreementOrThrow_shouldInsertRowInTerminationTable(
         E2eScenario scenario,

extensions/database-direct-access/build.gradle.kts

@@ -9,6 +9,7 @@ dependencies {
     compileOnly(libs.lombok)
 
     implementation(libs.edc.coreSpi)
+    implementation(libs.edc.runtimeMetamodel)
 
     implementation(libs.jooq.jooq)
     implementation(libs.hikari)

extensions/edc-ui-config/build.gradle.kts

@@ -9,6 +9,7 @@ dependencies {
     api(libs.edc.controlPlaneSpi)
     implementation(libs.edc.apiCore)
     implementation(libs.edc.managementApiConfiguration)
+    implementation(libs.edc.runtimeMetamodel)
 
     implementation(libs.jakarta.rsApi)
     implementation(libs.jakarta.validationApi)

extensions/last-commit-info/build.gradle.kts

@@ -12,6 +12,7 @@ dependencies {
     api(libs.edc.controlPlaneSpi)
     implementation(libs.edc.apiCore)
     implementation(libs.edc.managementApiConfiguration)
+    implementation(libs.edc.runtimeMetamodel)
 
     implementation(libs.jakarta.rsApi)
     implementation(libs.jakarta.validationApi)

extensions/placeholder-data-source/README.md

@@ -0,0 +1,73 @@
+<!-- PROJECT LOGO -->
+<br />
+<div align="center">
+  <a href="https://github.com/sovity/edc-ce">
+    <img src="https://raw.githubusercontent.com/sovity/edc-ui/main/src/assets/images/sovity_logo.svg" alt="Logo" width="300">
+  </a>
+
+<h3 align="center">EDC-Connector Extension:<br />Sovity Messenger</h3>
+
+  <p align="center">
+    <a href="https://github.com/sovity/edc-ce/issues/new?template=bug_report.md">Report Bug</a>
+    ·
+    <a href="https://github.com/sovity/edc-ce/issues/new?template=feature_request.md">Asset Data Source Placeholder</a>
+  </p>
+</div>
+
+## About this Extension
+
+Provides a placeholder endpoint for on-request offers.
+
+## Why does this extension exist?
+
+This extension exists to inform the asset's consumer upon data retrieval that they should contact the provider and take extra steps to access the data.
+
+## Configuration
+
+`MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL` / `my.edc.datasource.placeholder.baseurl` must be set to point to the placeholder endpoint's base URL.
+`/data-source/placeholder/asset/` will be appended to this base URL to make it the placeholder data source endpoint. This will be the address as seen by the consumer.
+
+---
+
+On a production system, the base URL it could be:
+
+`https://mycompany.com/path/to/backend`
+
+with a placeholder value:
+
+`https://mycompany.com/path/to/backend/data-source/placeholder/asset`
+
+and a full path to the asset's data:
+
+`https://mycompany.com/path/to/backend/data-source/placeholder/asset?email=foo%40example.com&subject=Contact+us+now`.
+
+---
+
+On a system started with docker-compose, it will be pointing to the DSP port on the provider's EDC
+
+`http://edc:11003/`
+
+`http://edc:11003/data-source/placeholder/asset`
+
+`http://edc:11003/data-source/placeholder/asset?email=foo%40example.com&subject=Contact+us+now`
+
+---
+
+During local/dev/unit test execution, it will be pointing to the DSP port on the provider's EDC
+
+`http://localhost:12345/`
+
+where the port that is here `12345` would actually be chosen at random
+
+`http://localhost:12345/data-source/placeholder/asset`
+
+`http://localhost:12345/data-source/placeholder/asset?email=foo%40example.com&subject=Contact+us+now`
+
+## License
+
+Apache License 2.0 - see [LICENSE](../../LICENSE)
+
+## Contact
+
+sovity GmbH - [email protected]
+

extensions/placeholder-data-source/build.gradle.kts

@@ -0,0 +1,35 @@
+
+plugins {
+    `java-library`
+    `maven-publish`
+}
+
+dependencies {
+    annotationProcessor(libs.lombok)
+    compileOnly(libs.lombok)
+
+    implementation(libs.edc.apiCore)
+    implementation(libs.edc.coreSpi)
+    implementation(libs.edc.dspApiConfiguration)
+    implementation(libs.edc.runtimeMetamodel)
+    implementation(libs.okhttp.okhttp)
+
+
+    testAnnotationProcessor(libs.lombok)
+    testCompileOnly(libs.lombok)
+
+    testImplementation(project(":utils:test-utils"))
+    testImplementation(libs.assertj.core)
+    testImplementation(libs.junit.api)
+    testRuntimeOnly(libs.junit.engine)
+}
+
+group = libs.versions.sovityEdcExtensionGroup.get()
+
+publishing {
+    publications {
+        create<MavenPublication>(project.name) {
+            from(components["java"])
+        }
+    }
+}

extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderDataSourceExtension.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2024 sovity GmbH
+ *
+ *  This program and the accompanying materials are made available under the
+ *  terms of the Apache License, Version 2.0 which is available at
+ *  https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *
+ *  Contributors:
+ *       sovity GmbH - initial API and implementation
+ *
+ */
+
+package de.sovity.edc.extension.placeholderdatasource;
+
+import lombok.val;
+import org.eclipse.edc.protocol.dsp.api.configuration.DspApiConfiguration;
+import org.eclipse.edc.runtime.metamodel.annotation.Extension;
+import org.eclipse.edc.runtime.metamodel.annotation.Inject;
+import org.eclipse.edc.runtime.metamodel.annotation.Provides;
+import org.eclipse.edc.runtime.metamodel.annotation.Setting;
+import org.eclipse.edc.spi.system.ServiceExtension;
+import org.eclipse.edc.spi.system.ServiceExtensionContext;
+import org.eclipse.edc.web.spi.WebService;
+
+import static de.sovity.edc.extension.placeholderdatasource.PlaceholderDataSourceExtension.NAME;
+
+@Extension(value = NAME)
+@Provides(PlaceholderEndpointService.class)
+public class PlaceholderDataSourceExtension implements ServiceExtension {
+
+    public static final String NAME = "Placeholder Data Source";
+
+    @Setting(required = true)
+    public static final String MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL = "my.edc.datasource.placeholder.baseurl";
+
+    @Inject
+    private DspApiConfiguration dspApiConfiguration;
+
+    @Inject
+    private WebService webService;
+
+    @Override
+    public void initialize(ServiceExtensionContext context) {
+        val controller = new PlaceholderEndpointController();
+        webService.registerResource(dspApiConfiguration.getContextAlias(), controller);
+
+        val baseUrl = context.getConfig().getString(MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL);
+        context.registerService(PlaceholderEndpointService.class, new PlaceholderEndpointService(baseUrl));
+    }
+}