Self review

CHANGELOG.md

@@ -24,6 +24,9 @@ New configuration to access the database:
 - `EDC_SERVER_DB_CONNECTION_POOL_SIZE`
   - The property controls the maximum size that the pool is allowed to reach, including both idle and in-use connections. Basically this value will determine the maximum number of actual connections to the database backend.
   - Defaults to `3`
+- `MY_EDC_DATASOURCE_PLACEHOLDER_BASEURL` 
+  - The property on which URL an asset is seen from an external source (internet, VPN, ...)
+  - It is documented in the asset data placeholder [README.md](extensions/placeholder-data-source/README.md)
 
 #### Compatible Versions
 

extensions/catalog-crawler/catalog-crawler/build.gradle.kts

@@ -17,6 +17,7 @@ dependencies {
     api(project(":utils:json-and-jsonld-utils"))
     api(project(":extensions:wrapper:wrapper-common-mappers"))
     api(project(":extensions:catalog-crawler:catalog-crawler-db"))
+    api(project(":extensions:placeholder-data-source"))
     api(project(":extensions:postgres-flyway-core"))
 
     testAnnotationProcessor(libs.lombok)

extensions/placeholder-data-source/README.md

@@ -10,7 +10,7 @@
   <p align="center">
     <a href="https://github.com/sovity/edc-ce/issues/new?template=bug_report.md">Report Bug</a>
     ·
-    <a href="https://github.com/sovity/edc-ce/issues/new?template=feature_request.md">Request Feature</a>
+    <a href="https://github.com/sovity/edc-ce/issues/new?template=feature_request.md">Asset Data Source Placeholder</a>
   </p>
 </div>
 
@@ -35,7 +35,7 @@ On a production system, the base URL it could be:
 
 with a placeholder value:
 
-`https://mycompany.com/path/to/backend/data-source/placeholder/asset/` 
+`https://mycompany.com/path/to/backend/data-source/placeholder/asset` 
 
 and a full path to the asset's data:
 
@@ -47,7 +47,7 @@ On a system started with docker-compose, it will be pointing to the DSP port on
 
 `http://edc:11003/`
 
-`http://edc:11003/data-source/placeholder/asset/`
+`http://edc:11003/data-source/placeholder/asset`
 
 `http://edc:11003/data-source/placeholder/asset?email=foo%40example.com&subject=Contact+us+now`
 
@@ -57,9 +57,9 @@ During local/dev/unit test execution, it will be pointing to the DSP port on the
 
 `http://localhost:12345/`
 
-where `12345` would be chosen at random
+where the port that is here `12345` would actually be chosen at random
 
-`http://localhost:12345/data-source/placeholder/asset/`
+`http://localhost:12345/data-source/placeholder/asset`
 
 `http://localhost:12345/data-source/placeholder/asset?email=foo%40example.com&subject=Contact+us+now`
 

extensions/placeholder-data-source/build.gradle.kts

@@ -8,13 +8,17 @@ dependencies {
     annotationProcessor(libs.lombok)
     compileOnly(libs.lombok)
 
-    api(libs.edc.coreSpi)
-    api(libs.edc.controlPlaneSpi)
+    implementation(libs.edc.coreSpi)
     implementation(libs.edc.apiCore)
     implementation(libs.edc.dspApiConfiguration)
     implementation(libs.okhttp.okhttp)
 
-    testImplementation(libs.mockito.core)
+
+    testAnnotationProcessor(libs.lombok)
+    testCompileOnly(libs.lombok)
+
+    testImplementation(project(":utils:test-utils"))
+    testImplementation(libs.assertj.core)
     testImplementation(libs.junit.api)
     testRuntimeOnly(libs.junit.engine)
 }

extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderEndpointController.java

@@ -34,9 +34,11 @@ public class PlaceholderEndpointController {
     @Consumes("*/*")
     public Response get(@QueryParam("email") String email, @QueryParam("subject") String subject) {
         return Response.ok("""
-            This is not a real data offer.
+            This is not real data.
 
-            The offer you are trying to use only has this placeholder as a dummy endpoint and requires you top take extra actions to access it.
+            This asset is accessible on request.
+
+            The offer you are trying to use only has this placeholder as a dummy endpoint and requires you to take extra steps to access it.
 
             Please contact the data provider for more information about how to access it.
             """ + "\n\n" + "Email: " + email + "\n" + "Subject: " + subject + "\n"

extensions/placeholder-data-source/src/main/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderEndpointService.java

@@ -24,7 +24,6 @@ public class PlaceholderEndpointService {
 
     public static final String DUMMY_ENDPOINT_URL = "/data-source/placeholder/asset";
 
-    // TODO: maybe add String contactEmail, String contactPreferredEmailSubject and show them in the placeholder
     public String getPlaceholderEndpointForAsset(String email, String subject) {
         return HttpUrl.parse(baseUrl + DUMMY_ENDPOINT_URL)
             .newBuilder()

tests/src/test/java/de/sovity/edc/e2e/PlaceholderDataSourceTest.java → extensions/placeholder-data-source/src/test/java/de/sovity/edc/extension/placeholderdatasource/PlaceholderDataSourceExtensionTest.java

@@ -12,15 +12,13 @@
  *
  */
 
-package de.sovity.edc.e2e;
+package de.sovity.edc.extension.placeholderdatasource;
 
 import de.sovity.edc.client.EdcClient;
 import de.sovity.edc.client.gen.model.DataSourceType;
 import de.sovity.edc.client.gen.model.UiAssetCreateRequest;
 import de.sovity.edc.client.gen.model.UiDataSource;
 import de.sovity.edc.client.gen.model.UiDataSourceOnRequest;
-import de.sovity.edc.extension.placeholderdatasource.PlaceholderDataSourceExtension;
-import de.sovity.edc.extension.placeholderdatasource.PlaceholderEndpointService;
 import de.sovity.edc.extension.e2e.connector.config.ConnectorConfig;
 import de.sovity.edc.extension.e2e.extension.E2eTestExtension;
 import de.sovity.edc.extension.e2e.extension.Provider;
@@ -38,7 +36,7 @@
 import static org.assertj.core.api.Assertions.assertThat;
 
 @ExtendWith(E2eTestExtension.class)
-public class PlaceholderDataSourceTest {
+class PlaceholderDataSourceExtensionTest {
 
     @SneakyThrows
     @Test
@@ -85,7 +83,7 @@ void shouldAccessDummyEndpoint(
         val client = new OkHttpClient();
         val content = client.newCall(request).execute().body().string();
 
-        assertThat(content).contains("This is not a real data offer.");
+        assertThat(content).contains("This is not real data.");
         assertThat(content).contains(email);
         assertThat(content).contains(subject);
     }

extensions/wrapper/wrapper-api/src/main/java/de/sovity/edc/ext/wrapper/api/ui/UiResource.java

@@ -34,10 +34,8 @@
 import de.sovity.edc.ext.wrapper.api.ui.model.UiContractNegotiation;
 import de.sovity.edc.ext.wrapper.api.ui.model.UiDataOffer;
 import io.swagger.v3.oas.annotations.Operation;
-import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.parameters.RequestBody;
 import io.swagger.v3.oas.annotations.tags.Tag;
-import jakarta.validation.Valid;
 import jakarta.ws.rs.Consumes;
 import jakarta.ws.rs.DELETE;
 import jakarta.ws.rs.GET;

extensions/wrapper/wrapper-api/src/main/java/de/sovity/edc/ext/wrapper/api/ui/model/ContractAgreementPageQuery.java

@@ -14,7 +14,6 @@
 
 package de.sovity.edc.ext.wrapper.api.ui.model;
 
-import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.AllArgsConstructor;

extensions/wrapper/wrapper-api/src/main/java/de/sovity/edc/ext/wrapper/api/usecase/UseCaseResource.java

@@ -21,7 +21,6 @@
 import de.sovity.edc.ext.wrapper.api.usecase.model.PolicyCreateRequest;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.tags.Tag;
-import jakarta.validation.Valid;
 import jakarta.validation.constraints.NotNull;
 import jakarta.ws.rs.Consumes;
 import jakarta.ws.rs.GET;

extensions/wrapper/wrapper-common-mappers/build.gradle.kts

@@ -12,7 +12,7 @@ dependencies {
     api(libs.edc.coreSpi)
     api(libs.edc.transformCore)
     api(libs.edc.transformSpi)
-    api(project(":extensions:placeholder-data-source"))
+    implementation(project(":extensions:placeholder-data-source"))
     api(project(":extensions:wrapper:wrapper-common-api"))
     api(project(":utils:json-and-jsonld-utils"))
     implementation(libs.apache.commonsLang)

extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/AssetMapper.java

@@ -53,8 +53,6 @@ public Asset editAsset(
         @NonNull UiAssetEditRequest editRequest
     ) {
         var assetJsonLd = buildAssetJsonLd(asset);
-        // edit request is all empty
-        // assetJsonLd contains the manually set baseUrl
         var editedJsonLd = assetJsonLdBuilder.editAssetJsonLd(assetJsonLd, editRequest);
         return buildAsset(editedJsonLd);
     }

extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/asset/AssetEditRequestMapper.java

@@ -32,10 +32,7 @@ public class AssetEditRequestMapper {
      * @param editRequest {@link UiAssetEditRequest}
      * @return {@link UiAssetCreateRequest}
      */
-    public UiAssetCreateRequest buildCreateRequest(
-        @NonNull UiAssetEditRequest editRequest,
-        @NonNull String assetId
-    ) {
+    public UiAssetCreateRequest buildCreateRequest(@NonNull UiAssetEditRequest editRequest, @NonNull String assetId) {
         var dataSource = editRequest.getDataSourceOverrideOrNull();
         if (dataSource == null) {
             dataSource = dummyDataSource();
@@ -76,8 +73,6 @@ public UiAssetCreateRequest buildCreateRequest(
     }
 
     private UiDataSource dummyDataSource() {
-        return UiDataSource.builder()
-            .type(DataSourceType.CUSTOM)
-            .build();
+        return UiDataSource.builder().type(DataSourceType.CUSTOM).build();
     }
 }

extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/asset/AssetJsonLdBuilder.java

@@ -63,41 +63,20 @@ public JsonObject createAssetJsonLd(
         );
     }
 
+
     @SneakyThrows
     @Nullable
     public JsonObject editAssetJsonLd(
-        JsonObject assetJsonLd, // 3
-        UiAssetEditRequest editRequest // 2
+        JsonObject assetJsonLd,
+        UiAssetEditRequest editRequest
     ) {
-
-        /*
-         * Data source selection order:
-         * override?
-         * edit request
-         * jsonLd
-         * placeholder
-         */
-
-        // request base URL comes here
         var dataAddress = getDataAddressJsonLd(assetJsonLd);
-        // override is null
         if (editRequest.getDataSourceOverrideOrNull() != null) {
-            // this part of the code is not asserted
-            // what is this override used for?
-            // TODO should this part have higher or lower prio than the automatic assignment to the dummy data source?
-            // 1
-            // there is another handling of this field in de.sovity.edc.ext.wrapper.api.common.mappers.asset.AssetEditRequestMapper.buildCreateRequest
-            // TODO: this is not a clear name on the API side: is `orNull` the Java null of the JSON null?
             dataAddress = dataSourceMapper.buildDataSourceJsonLd(editRequest.getDataSourceOverrideOrNull());
         }
 
-        // the dataAddress that is used should be the one that's part of the createRequest, not a separate element
-
         var assetId = Objects.requireNonNull(JsonLdUtils.string(assetJsonLd, Prop.ID), "Asset JSON-LD had no @id");
         var organizationName = assetJsonLdParser.getCreatorOrganizationName(assetJsonLd);
-        // set dummy source to placeholder URL inside this createRequest
-        // this `createRequest` contains the correct dataSource
-        // 4: dummy
         var createRequest = assetEditRequestMapper.buildCreateRequest(editRequest, assetId);
         var properties = getAssetProperties(createRequest, dataAddress, organizationName);
         var privateProperties = getAssetPrivateProperties(createRequest);
@@ -154,8 +133,6 @@ private JsonObject getAssetProperties(
         addMobilityTheme(properties, request);
 
         addCustomJsonLd(properties, request);
-//         request.getDataSource().
-        // TODO: replace with request.getDataSource()
         addDataSourceHints(properties, dataAddressJsonLd);
         return properties.build();
     }

extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/dataaddress/DataSourceMapper.java

@@ -29,7 +29,6 @@
 import lombok.RequiredArgsConstructor;
 
 import java.util.Map;
-import java.util.function.BiFunction;
 import java.util.function.Function;
 import java.util.function.Supplier;
 
@@ -42,7 +41,6 @@ public class DataSourceMapper {
     private final HttpDataSourceMapper httpDataSourceMapper;
 
     public JsonObject buildDataSourceJsonLd(@NonNull UiDataSource dataSource) {
-        // this datasource has the URL set in the creation request
         var props = this.matchDataSource(
             dataSource,
             httpDataSourceMapper::buildDataAddress,

extensions/wrapper/wrapper-common-mappers/src/main/java/de/sovity/edc/ext/wrapper/api/common/mappers/dataaddress/http/HttpDataSourceMapper.java

@@ -94,7 +94,6 @@ public Map<String, String> buildOnRequestDataAddress(@NonNull UiDataSourceOnRequ
             "Need contactPreferredEmailSubject"
         );
 
-        // this seems to be in use
         String placeholderEndpointForAsset = placeholderEndpointService.getPlaceholderEndpointForAsset(
             onRequest.getContactEmail(),
             onRequest.getContactPreferredEmailSubject());
@@ -130,7 +129,6 @@ public JsonObject enhanceAssetWithDataSourceHints(Map<String, String> dataAddres
                 Prop.Edc.PROXY_BODY, Prop.SovityDcatExt.HttpDatasourceHints.BODY
             ).forEach((prop, hint) ->
                 // Will add hints as "true" or "false"
-                // TODO: what are those hints for? External info about the content of those fields??
                 json.add(hint, String.valueOf("true".equals(dataAddress.get(prop))))
             );
         }

extensions/wrapper/wrapper/build.gradle.kts

@@ -21,6 +21,7 @@ dependencies {
 
     implementation(project(":extensions:contract-termination"))
     implementation(project(":extensions:database-direct-access"))
+    implementation(project(":extensions:placeholder-data-source"))
     implementation(project(":extensions:sovity-messenger"))
     implementation(project(":utils:jooq-database-access"))
 

extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtension.java

@@ -19,8 +19,8 @@
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import de.sovity.edc.extension.contacttermination.ContractAgreementTerminationService;
 import de.sovity.edc.extension.db.directaccess.DslContextFactory;
-import de.sovity.edc.extension.placeholderdatasource.PlaceholderEndpointService;
 import de.sovity.edc.extension.messenger.SovityMessenger;
+import de.sovity.edc.extension.placeholderdatasource.PlaceholderEndpointService;
 import org.eclipse.edc.connector.api.management.configuration.ManagementApiConfiguration;
 import org.eclipse.edc.connector.api.management.configuration.transform.ManagementApiTypeTransformerRegistry;
 import org.eclipse.edc.connector.contract.spi.negotiation.store.ContractNegotiationStore;

extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java

@@ -76,8 +76,8 @@
 import de.sovity.edc.extension.contacttermination.query.ContractAgreementTerminationDetailsQuery;
 import de.sovity.edc.extension.contacttermination.query.TerminateContractQuery;
 import de.sovity.edc.extension.db.directaccess.DslContextFactory;
-import de.sovity.edc.extension.placeholderdatasource.PlaceholderEndpointService;
 import de.sovity.edc.extension.messenger.SovityMessenger;
+import de.sovity.edc.extension.placeholderdatasource.PlaceholderEndpointService;
 import de.sovity.edc.utils.catalog.DspCatalogService;
 import de.sovity.edc.utils.catalog.mapper.DspDataOfferBuilder;
 import lombok.NoArgsConstructor;

extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/UiResourceImpl.java

@@ -45,11 +45,7 @@
 import de.sovity.edc.ext.wrapper.api.ui.pages.transferhistory.TransferHistoryPageApiService;
 import de.sovity.edc.ext.wrapper.api.ui.pages.transferhistory.TransferHistoryPageAssetFetcherService;
 import de.sovity.edc.extension.db.directaccess.DslContextFactory;
-import jakarta.validation.ConstraintViolationException;
-import jakarta.validation.Validation;
-import jakarta.validation.ValidatorFactory;
 import lombok.RequiredArgsConstructor;
-import lombok.val;
 import org.jetbrains.annotations.Nullable;
 
 import java.util.List;

extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/asset/AssetApiService.java

@@ -48,7 +48,6 @@ public List<UiAsset> getAssets() {
             .toList();
     }
 
-    // TODO: placeholder endpoint is not supported when creating the asset?
     @NotNull
     public IdResponseDto createAsset(UiAssetCreateRequest request) {
         assetIdValidator.assertValid(request.getId());

extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/contract_agreements/services/ContractAgreementData.java

@@ -21,7 +21,6 @@
 import org.eclipse.edc.spi.types.domain.asset.Asset;
 
 import java.util.List;
-import java.util.Map;
 
 
 /**

extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/contract_agreements/services/ContractAgreementPageCardBuilder.java

@@ -23,7 +23,6 @@
 import de.sovity.edc.ext.wrapper.api.ui.model.ContractAgreementTransferProcess;
 import de.sovity.edc.ext.wrapper.api.ui.model.ContractTerminatedBy;
 import de.sovity.edc.ext.wrapper.api.ui.pages.transferhistory.TransferProcessStateService;
-import jakarta.validation.constraints.Null;
 import lombok.NonNull;
 import lombok.RequiredArgsConstructor;
 import org.eclipse.edc.connector.contract.spi.types.agreement.ContractAgreement;
@@ -34,7 +33,6 @@
 
 import java.util.Comparator;
 import java.util.List;
-import java.util.Map;
 
 import static de.sovity.edc.ext.wrapper.api.ui.model.ContractTerminationStatus.ONGOING;
 import static de.sovity.edc.ext.wrapper.api.ui.model.ContractTerminationStatus.TERMINATED;

extensions/wrapper/wrapper/src/test/java/de/sovity/edc/ext/wrapper/api/ui/pages/policy/PolicyDefinitionApiServiceTest.java

@@ -27,7 +27,6 @@
 import de.sovity.edc.extension.db.directaccess.DslContextFactory;
 import de.sovity.edc.extension.e2e.connector.config.ConnectorConfig;
 import de.sovity.edc.extension.e2e.db.EdcRuntimeExtensionWithTestDatabase;
-import lombok.SneakyThrows;
 import lombok.val;
 import org.eclipse.edc.connector.spi.policydefinition.PolicyDefinitionService;
 import org.eclipse.edc.junit.annotations.ApiTest;

utils/test-utils/build.gradle.kts

@@ -12,6 +12,7 @@ dependencies {
 
     api(libs.edc.junit)
     api(libs.awaitility.java)
+    api(libs.postgres)
     api(project(":extensions:wrapper:clients:java-client"))
     api(project(":utils:json-and-jsonld-utils"))
 

utils/test-utils/src/main/java/de/sovity/edc/extension/e2e/connector/config/ConnectorConfigFactory.java

@@ -85,9 +85,6 @@ public static ConnectorConfig basicEdcConfig(String participantId, int firstPort
         properties.put("edc.last.commit.info", "test env commit message");
         properties.put("edc.build.date", "2023-05-08T15:30:00Z");
 
-        // TODO: rm
-        properties.put("edc.server.db.connection.timeout.in.ms", "5000");
-
         properties.put("my.edc.participant.id", participantId);
         properties.put("my.edc.title", "Connector Title %s".formatted(participantId));
         properties.put("my.edc.description", "Connector Description %s".formatted(participantId));
@@ -98,10 +95,6 @@ public static ConnectorConfig basicEdcConfig(String participantId, int firstPort
 
         properties.put("my.edc.datasource.placeholder.baseurl", apiConfig.getProtocolApiGroup().getUri().toString());
 
-        // TODO: rm
-        properties.put("edc.server.db.connection.pool.size", "3");
-
-        // TODO: rm?
         properties.put("web.http.port", String.valueOf(apiConfig.getDefaultApiGroup().port()));
         properties.put("web.http.path", String.valueOf(apiConfig.getDefaultApiGroup().path()));
         properties.put("web.http.protocol.port", String.valueOf(apiConfig.getProtocolApiGroup().port()));