Merge pull request #119 from FraunhoferIOSB/development

Development changes (see changelog)

changelog.md

@@ -6,10 +6,31 @@ Compatibility: **Eclipse Dataspace Connector v0.7.0**
 
 **New Features**
 
+* Dynamical building of AAS access URLs using Reference Chains [Submodel x, Collection y, Element z]
+* Accepting self-signed certificates (only possible for own AAS services) now
+  optional (`edc.dataplane.aas.acceptSelfSignedCertificates`)
+* SubmodelElementList is now a model element inside the SelfDescription (AASv3 change)
+
+* New extension: data-plane-aas
+    * Provides custom AAS data source and data sink
+
 **Bugfixes**
 
+* Adding external AAS services with self-signed certificates using configuration value edc.aas.remoteAASLocation is now
+  possible
+* Unregistering an external AAS service no longer throws IllegalArgumentException
+* DataTransfer with AAS DataSource no longer throws IOException: closed
+
 **Miscellaneous**
 
+* In the development version, HTTP Push data transfer to AAS services with self-signed certificates is possible
+    * This is only for testing
+    * Will be removed in release version
+    * Only works if the provider "knows" the consumer AAS service
+* EDC4AAS extension now uses EDC monitor directly instead of wrapping it with custom logger
+* Since the AAS model parsing got more complex, it was extracted out of the AAS agent
+* Added tests for self-signed certificate retriever
+
 ## V2.0.0
 
 This version is compatible to **Eclipse Dataspace Connector v0.6.2**
@@ -29,7 +50,6 @@ This version is compatible to **Eclipse Dataspace Connector v0.6.2**
 * Update postman requests
 * !Change API port for self-description to 8281/9291 in examples!
 
-
 ## V1.0.0-alpha5
 
 This version is compatible to **Eclipse Dataspace Connector v0.4.1**
@@ -38,7 +58,7 @@ This version is compatible to **Eclipse Dataspace Connector v0.4.1**
 
 * Support of **Eclipse Dataspace Connector v0.4.1**
 * Extracted automated client negotiation into own extension (/client)
- 
+
 **Bugfixes**
 
 * Fix: Case where submodels not offered by provider
@@ -56,7 +76,7 @@ This version is compatible to **Eclipse Dataspace Connector v0.4.1**
 This version is compatible to **Eclipse Dataspace Connector v0.3.0**
 
 **New Features**
-  
+
 * Support of **Eclipse Dataspace Connector v0.3.0**
 
 **Bugfixes**

client/build.gradle.kts

@@ -18,6 +18,7 @@ java {
 dependencies {
     // See this project's README.MD for explanations
     implementation(project(":public-api-management"))
+    implementation(project(":data-plane-aas"))
 
     implementation("$group:connector-core:$edcVersion") // PolicyService
     implementation("$group:control-plane-contract:$edcVersion") // Observe contract negotiations

client/src/main/java/de/fraunhofer/iosb/client/ClientEndpoint.java

@@ -39,6 +39,7 @@
 import org.eclipse.edc.connector.controlplane.policy.spi.PolicyDefinition;
 import org.eclipse.edc.policy.model.Policy;
 import org.eclipse.edc.spi.monitor.Monitor;
+import org.eclipse.edc.spi.types.domain.DataAddress;
 
 import java.io.StringReader;
 import java.net.URL;
@@ -106,7 +107,10 @@ public ClientEndpoint(Monitor monitor,
     @GET
     @Path(OFFER_PATH)
     public Response getOffer(@QueryParam("providerUrl") URL providerUrl, @QueryParam("assetId") String assetId, @QueryParam("providerId") String counterPartyId) {
-        monitor.debug(format("[Client] Received a %s GET request", OFFER_PATH));
+        Objects.requireNonNull(assetId, "Asset ID must not be null");
+
+
+        monitor.debug(format("[Client] Received an %s GET request", OFFER_PATH));
 
         if (Objects.isNull(providerUrl)) {
             return Response.status(Response.Status.BAD_REQUEST).entity("Provider URL must not be null").build();
@@ -152,18 +156,18 @@ private String buildResponseFrom(Dataset dataset) throws JsonProcessingException
      * Negotiate a contract agreement using the given contract offer if no agreement
      * exists for this constellation.
      *
-     * @param counterPartyUrl    Provider EDCs URL (DSP endpoint)
-     * @param counterPartyId     Provider EDCs ID
-     * @param assetId            ID of the asset to be retrieved
-     * @param dataDestinationUrl URL of destination data sink.
+     * @param counterPartyUrl Provider EDCs URL (DSP endpoint)
+     * @param counterPartyId  Provider EDCs ID
+     * @param assetId         ID of the asset to be retrieved
+     * @param dataAddress     DataAddress of destination data sink.
      * @return Asset data
      */
     @POST
     @Path(NEGOTIATE_PATH)
     public Response negotiateContract(@QueryParam("providerUrl") URL counterPartyUrl,
                                       @QueryParam("providerId") String counterPartyId,
                                       @QueryParam("assetId") String assetId,
-                                      @QueryParam("dataDestinationUrl") URL dataDestinationUrl) {
+                                      DataAddress dataAddress) {
         monitor.debug(format("[Client] Received a %s POST request", NEGOTIATE_PATH));
         Objects.requireNonNull(counterPartyUrl, "Provider URL must not be null");
         Objects.requireNonNull(counterPartyId, "Provider ID must not be null");
@@ -201,7 +205,7 @@ public Response negotiateContract(@QueryParam("providerUrl") URL counterPartyUrl
                     .build();
         }
 
-        return getData(counterPartyUrl, agreement.getId(), assetId, dataDestinationUrl);
+        return getData(counterPartyUrl, agreement.getId(), assetId, dataAddress);
     }
 
     /**
@@ -237,31 +241,33 @@ public Response negotiateContract(ContractRequest contractRequest) {
 
     /**
      * Submits a data transfer request to the providerUrl.
+     * In the future this could be replaced with the
+     * <a href="https://www.ietf.org/archive/id/draft-ietf-httpbis-safe-method-w-body-02.html">HTTP QUERY method</a>
      *
-     * @param providerUrl        The data provider's url
-     * @param agreementId        The basis of the data transfer.
-     * @param assetId            The asset of which the data should be transferred
-     * @param dataDestinationUrl URL of destination data sink.
+     * @param providerUrl The data provider's url
+     * @param agreementId The basis of the data transfer.
+     * @param assetId     The asset of which the data should be transferred
+     * @param dataAddress URL of destination data sink.
      * @return On success, the data of the desired asset. Else, returns an error message.
      */
-    @GET
+    @POST
     @Path(TRANSFER_PATH)
     public Response getData(@QueryParam("providerUrl") URL providerUrl,
                             @QueryParam("agreementId") String agreementId,
                             @QueryParam("assetId") String assetId,
-                            @QueryParam("dataDestinationUrl") URL dataDestinationUrl) {
+                            DataAddress dataAddress) {
         monitor.debug(format("[Client] Received a %s GET request", TRANSFER_PATH));
         Objects.requireNonNull(providerUrl, "providerUrl must not be null");
         Objects.requireNonNull(agreementId, "agreementId must not be null");
         Objects.requireNonNull(assetId, "assetId must not be null");
 
         try {
             var data = transferController.initiateTransferProcess(providerUrl, agreementId, assetId,
-                    dataDestinationUrl);
-            if (Objects.isNull(dataDestinationUrl)) {
+                    dataAddress);
+            if (Objects.isNull(dataAddress)) {
                 return Response.ok(data).build();
             } else {
-                return Response.ok(format("Data transfer request to URL %s sent.", dataDestinationUrl)).build();
+                return Response.ok("Data transfer request sent.").build();
             }
         } catch (InterruptedException | ExecutionException negotiationException) {
             monitor.severe(format("[Client] Data transfer failed for provider %s and agreementId %s", providerUrl,

client/src/main/java/de/fraunhofer/iosb/client/authentication/DataTransferEndpointManager.java

@@ -38,10 +38,10 @@ public DataTransferEndpointManager(PublicApiManagementService publicApiManagemen
     /**
      * Add key,value pair for a request. This key will only be available for one
      * request.
-
+     *
      * @param agreementId Agreement to build the endpoint path suffix
-     * @param key   The key name
-     * @param value The value
+     * @param key         The key name
+     * @param value       The value
      */
     public void addTemporaryEndpoint(String agreementId, String key, String value) {
         var endpointSuffix = ClientEndpoint.AUTOMATED_PATH + "/receiveData/" + agreementId;

client/src/main/java/de/fraunhofer/iosb/client/datatransfer/DataTransferController.java

@@ -18,10 +18,10 @@
 import de.fraunhofer.iosb.api.PublicApiManagementService;
 import de.fraunhofer.iosb.client.authentication.DataTransferEndpointManager;
 import org.eclipse.edc.connector.controlplane.transfer.spi.TransferProcessManager;
-import org.eclipse.edc.connector.dataplane.http.spi.HttpDataAddress;
 import org.eclipse.edc.spi.EdcException;
 import org.eclipse.edc.spi.monitor.Monitor;
 import org.eclipse.edc.spi.system.configuration.Config;
+import org.eclipse.edc.spi.types.domain.DataAddress;
 import org.eclipse.edc.web.spi.WebService;
 
 import java.net.URL;
@@ -73,32 +73,28 @@ public DataTransferController(Monitor monitor, Config config, WebService webServ
      * Initiates the transfer process defined by the arguments. The data of the
      * transfer will be sent to {@link DataTransferEndpoint#RECEIVE_DATA_PATH}.
      *
-     * @param providerUrl        The provider from whom the data is to be fetched.
-     * @param agreementId        Non-null ContractAgreement of the negotiation process.
-     * @param assetId            The asset to be fetched.
-     * @param dataDestinationUrl HTTPDataAddress the result of the transfer should be
-     *                           sent to. (If null, send to extension and print in log)
+     * @param providerUrl     The provider from whom the data is to be fetched.
+     * @param agreementId     Non-null ContractAgreement of the negotiation process.
+     * @param assetId         The asset to be fetched.
+     * @param dataSinkAddress HTTPDataAddress the result of the transfer should be
+     *                        sent to. (If null, send to extension and print in log)
      * @return A completable future whose result will be the data or an error message.
      * @throws InterruptedException If the data transfer was interrupted
      * @throws ExecutionException   If the data transfer process failed
      */
     public String initiateTransferProcess(URL providerUrl, String agreementId, String assetId,
-                                          URL dataDestinationUrl) throws InterruptedException, ExecutionException {
+                                          DataAddress dataSinkAddress) throws InterruptedException, ExecutionException {
         // Prepare for incoming data
         var dataFuture = dataTransferObservable.register(agreementId);
 
-        if (Objects.isNull(dataDestinationUrl)) {
+        if (Objects.isNull(dataSinkAddress)) {
             var apiKey = UUID.randomUUID().toString();
             dataTransferEndpointManager.addTemporaryEndpoint(agreementId, DATA_TRANSFER_API_KEY, apiKey);
 
             this.transferInitiator.initiateTransferProcess(providerUrl, agreementId, assetId, apiKey);
             return waitForData(dataFuture, agreementId);
         } else {
             // Send data to custom target url
-            var dataSinkAddress = HttpDataAddress.Builder.newInstance()
-                    .baseUrl(dataDestinationUrl.toString())
-                    .build();
-
             this.transferInitiator.initiateTransferProcess(providerUrl, agreementId, assetId, dataSinkAddress);
             // Don't have to wait for data
             return null;

client/src/main/java/de/fraunhofer/iosb/client/datatransfer/TransferInitiator.java

@@ -20,6 +20,7 @@
 import jakarta.ws.rs.core.UriBuilderException;
 import org.eclipse.edc.connector.controlplane.transfer.spi.TransferProcessManager;
 import org.eclipse.edc.connector.controlplane.transfer.spi.types.TransferRequest;
+import org.eclipse.edc.connector.dataplane.http.spi.HttpDataAddress;
 import org.eclipse.edc.spi.EdcException;
 import org.eclipse.edc.spi.monitor.Monitor;
 import org.eclipse.edc.spi.system.configuration.Config;
@@ -33,7 +34,6 @@
 import static de.fraunhofer.iosb.client.datatransfer.DataTransferController.DATA_TRANSFER_API_KEY;
 import static java.lang.String.format;
 import static org.eclipse.edc.protocol.dsp.http.spi.types.HttpMessageProtocol.DATASPACE_PROTOCOL_HTTP;
-import static org.eclipse.edc.spi.constants.CoreConstants.EDC_NAMESPACE;
 
 /**
  * Initiate transfer requests
@@ -57,11 +57,10 @@ void initiateTransferProcess(URL providerUrl, String agreementId, String assetId
                     "Cannot transfer to own EDC since own URI could not be built while initializing client extension. Not continuing...");
             return;
         }
-        var dataDestination = DataAddress.Builder.newInstance()
-                .type("HttpData")
-                .property(EDC_NAMESPACE + "baseUrl", ownUri.toString())
-                .property(EDC_NAMESPACE + "path", agreementId)
-                .property("header:" + DATA_TRANSFER_API_KEY, apiKey) // API key for validation on consumer side
+        var dataDestination = HttpDataAddress.Builder.newInstance()
+                .baseUrl(ownUri.toString())
+                .path(agreementId)
+                .addAdditionalHeader(DATA_TRANSFER_API_KEY, apiKey) // API key for validation on consumer side
                 .build();
 
         initiateTransferProcess(providerUrl, agreementId, assetId, dataDestination);

client/src/main/java/de/fraunhofer/iosb/client/policy/PolicyDefinitionStore.java

@@ -81,7 +81,7 @@ Optional<PolicyDefinition> removePolicyDefinition(String policyDefinitionId) {
     /**
      * Update a policyDefinition
      *
-     * @param policyDefinition   The updated policyDefinition
+     * @param policyDefinition The updated policyDefinition
      * @return Optional containing updated policy definition or null
      */
     Optional<PolicyDefinition> updatePolicyDefinitions(PolicyDefinition policyDefinition) {

client/src/main/java/de/fraunhofer/iosb/client/policy/PolicyService.java

@@ -90,7 +90,6 @@ class PolicyService {
     }
 
     Dataset getDatasetForAssetId(@NotNull String counterPartyId, @NotNull URL counterPartyUrl, @NotNull String assetId) throws InterruptedException {
-
         var catalogFuture = catalogService.requestCatalog(
                 counterPartyId, // why do we even need a provider id when we have the url...
                 counterPartyUrl.toString(),

client/src/main/java/de/fraunhofer/iosb/client/policy/transform/JsonObjectToCatalogTransformer.java

@@ -33,7 +33,6 @@
  * Transformer for Catalog in JSON representation to EDC Catalog class.
  * Modified code from <a href="https://github.com/eclipse-edc/Connector/tree/v0.5.1/core/common/transform-core">github.com</a>
  * <br>See also: <a href="https://github.com/eclipse-edc/Connector/pull/3916">https://github.com/eclipse-edc/Connector/pull/3916</a>
- *
  */
 public class JsonObjectToCatalogTransformer extends AbstractJsonLdTransformer<JsonObject, Catalog> {
 

client/src/main/java/de/fraunhofer/iosb/client/util/Pair.java

@@ -31,14 +31,14 @@ public T getFirst() {
         return first;
     }
 
-    public U getSecond() {
-        return second;
-    }
-
     public void setFirst(T t) {
         this.first = t;
     }
 
+    public U getSecond() {
+        return second;
+    }
+
     public void setSecond(U u) {
         this.second = u;
     }

client/src/test/java/de/fraunhofer/iosb/client/ClientEndpointTest.java

@@ -20,8 +20,8 @@
 import de.fraunhofer.iosb.client.datatransfer.DataTransferController;
 import de.fraunhofer.iosb.client.negotiation.NegotiationController;
 import de.fraunhofer.iosb.client.policy.PolicyController;
+import de.fraunhofer.iosb.dataplane.aas.spi.AasDataAddress;
 import jakarta.ws.rs.core.Response;
-
 import org.eclipse.edc.connector.controlplane.asset.spi.domain.Asset;
 import org.eclipse.edc.connector.controlplane.catalog.spi.Catalog;
 import org.eclipse.edc.connector.controlplane.catalog.spi.Dataset;
@@ -73,11 +73,9 @@ public class ClientEndpointTest {
 
     private static URL url;
     private static ClientAndServer mockServer;
-
-    private ClientEndpoint clientEndpoint;
-
     private static PolicyDefinition mockPolicyDefinition;
     private static Catalog mockCatalog;
+    private ClientEndpoint clientEndpoint;
 
     @BeforeAll
     public static void initialize() throws IOException {
@@ -203,7 +201,7 @@ public void negotiateContractAndTransferTest() {
     @Test
     public void getDataTest() {
         try {
-            clientEndpoint.getData(url, "test-agreement-id", "test-asset-id", url);
+            clientEndpoint.getData(url, "test-agreement-id", "test-asset-id", AasDataAddress.Builder.newInstance().baseUrl(url.toString()).build());
             fail();
         } catch (EdcException expected) {
         }
@@ -218,7 +216,7 @@ public void getAcceptedContractOffersTest() {
     public void addAcceptedContractOffersTest() {
         var mockPolicyDefinitionsAsList = new ArrayList<PolicyDefinition>();
         mockPolicyDefinitionsAsList.add(mockPolicyDefinition); // ClientEndpoint creates ArrayList
-        var offers = new PolicyDefinition[]{mockPolicyDefinition};
+        var offers = new PolicyDefinition[] {mockPolicyDefinition};
 
         clientEndpoint.addAcceptedPolicyDefinitions(offers);
 
@@ -227,7 +225,7 @@ public void addAcceptedContractOffersTest() {
 
     @Test
     public void updateAcceptedContractOfferTest() {
-        var offers = new PolicyDefinition[]{mockPolicyDefinition};
+        var offers = new PolicyDefinition[] {mockPolicyDefinition};
 
         clientEndpoint.addAcceptedPolicyDefinitions(offers);
 

client/src/test/java/de/fraunhofer/iosb/client/policy/PolicyServiceTest.java

@@ -59,11 +59,11 @@
  */
 public class PolicyServiceTest {
 
+    private final URL testUrl;
     private CatalogService catalogService;
     private TypeTransformerRegistry typeTransformerRegistry;
     private PolicyServiceConfig config;
     private PolicyService policyService;
-    private final URL testUrl;
 
     public PolicyServiceTest() throws MalformedURLException {
         int providerPort = 54321;

client/src/test/java/de/fraunhofer/iosb/client/testutils/FileManager.java

@@ -27,11 +27,11 @@
 
 public class FileManager {
 
+    private static final File RESOURCES_DIRECTORY = new File("src/test/resources");
+
     private FileManager() {
     }
 
-    private static final File RESOURCES_DIRECTORY = new File("src/test/resources");
-
     public static String loadResource(String fileName) {
         try (FileInputStream x = new FileInputStream(new File(RESOURCES_DIRECTORY, fileName))) {
             return IOUtils.toString(x, StandardCharsets.UTF_8);