diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml
index 47cc8e07..7ec95923 100644
--- a/.github/workflows/build-release.yml
+++ b/.github/workflows/build-release.yml
@@ -33,16 +33,18 @@ jobs:
- name: deploy
uses: actions/setup-node@v2
with:
- node-version: 14.x
+ node-version: 18.x
- run: npm i
working-directory: ./dynamic-mapping-ui
+ - run: npm install -g @angular/cli > /dev/null
+ working-directory: ./dynamic-mapping-ui
- run: npm run build --if-present
working-directory: ./dynamic-mapping-ui
- name: Zip Frontend
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
run: |
- cd dynamic-mapping-ui/dist/apps/sag-ps-pkg-dynamic-mapping
- zip -r -q ../../../dynamic-mapping-ui.zip *
+ cd dynamic-mapping-ui/dist/dynamic-mapping-ui
+ zip -r -q ../../dynamic-mapping-ui.zip *
- name: Create Release
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
id: create_release
@@ -52,7 +54,7 @@ jobs:
with:
tag_name: ${{ github.ref }}
release_name: Release ${{ github.ref }}
- draft: false
+ draft: true
prerelease: false
- name: Upload Release Asset Frontend
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
diff --git a/.gitignore b/.gitignore
index e6980995..2e91c8e2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,17 +35,17 @@ target/
logs/
bin/
attic/*
-dist/
-
.factorypath
set.sh
dynamic-mapping-ui/.env
-
-**/.flattened-pom.xml
+.flattened-pom.xml
+dist
node_modules
.angular
dynamic-mapping-ui/cypress.env.js
dynamic-mapping-ui/cypress/videos
dynamic-mapping-ui/cypress/screenshots
dynamic-mapping-ui/cypress/fixtures/mqttConnectionPostRequest.json
+dynamic-mapping-service/src/main/configuration/dynamic-mapping-service-logging.xml
+JSONata4Java/dependency-reduced-pom.xml
diff --git a/README.md b/README.md
index 68135513..a5a12b2e 100644
--- a/README.md
+++ b/README.md
@@ -3,7 +3,7 @@
## Table of Contents
- [Overview](#overview)
- [Installation Guide ](#installation-guide)
-- [Build & Deploye](#build--deploy)
+- [Build & Deploy](#build--deploy)
- [User Guide](#user-guide)
- [API Documentation](#api-documentation)
- [Tests & Sample Data](#tests--sample-data)
@@ -13,34 +13,36 @@
## Overview
The Cumulocity Dynamic Mapper addresses the need to get **any** data provided by a message broker mapped to the Cumulocity IoT Domain model in a zero-code approach.
-It can connect to multiple message brokers likes **MQTT**, **MQTT Connect** and others, subscribes to specific topics and maps the data in a graphical way to the domain model of Cumulocity.
+It can connect to multiple message brokers likes **MQTT**, **MQTT Service** , **Kafka** and others, subscribes to specific topics and maps the data in a graphical editor to the domain model of Cumulocity.
Per default the followings connectors are supported
* **MQTT** - any MQTT Broker
-* **MQTT Connect** - MQTT Broker provided by Cumulocity IoT (in development)
+* **MQTT Service** - MQTT Broker
+* **Kafka** - Kafka Broker
-It contains two major components:
+The solution is compposed of two major components:
-* A **Microservice** - exposes REST endpoints, provides a generic connector interface which can be used by broker clients to
+* A **microservice** - exposes REST endpoints, provides a generic connector interface which can be used by broker clients to
connect to a message broker, a generic data mapper, a comprehensive expression language for data mapping and the
[Cumulocity Microservice SDK](https://cumulocity.com/guides/microservice-sdk/introduction/) to connect to Cumulocity. It also supports multi tenancy.
-* A **Frontend Plugin** - uses the exposed endpoints of the microservice to configure a broker connection & to perform
+* A **frontend (plugin)** - uses the exposed endpoints of the microservice to configure a broker connection & to perform
graphical data mappings within the Cumumlocity IoT UI.
Using the Cumulocity Dynamic Mapper you are able to connect to almost any message broker and map any payload on any topic dynamically to
-the Cumulocity IoT Domain Model in a graphical way.
+the Cumulocity IoT Domain Model in a graphical editor.
Here are the **core features** summarized:
* **Connect** to multiple message broker of your choice at the same time.
* **Map** any data to/from the Cumulocity IoT Domain Model in a graphical way.
-* **Bidirectional mappings** are supported - so you can forward data to Cumulocity or subscribe on Cumulocity data and forward it to the broker.
+* **Bidirectional mappings** are supported - so you can forward data to Cumulocity or subscribe on Cumulocity data and forward it to the broker
* **Transform** data with a comprehensive expression language supported by [JSONata](https://jsonata.org/)
* **Multiple payload formats** are supported, starting with **JSON**, **Protobuf**, **Binary**, **CSV**.
* **Extend** the mapper easily by using payload extensions or the provided connector interface
* Full support of **multi-tenancy** - deploy it in your enterprise tenant and subscribe it to sub-tenants.
+
@@ -50,27 +52,28 @@ Here are the **core features** summarized:
The architecture of the components consists of the following components:
-
+
-The orange components are part of this project which are:
+The light blue components are part of this project which are:
-* 2 Default connectors for..
- * **MQTT Client** - using [PAHO MQTT Client](https://github.com/eclipse/paho.mqtt.java) to connect and subscribe to MQTT brokers
- * **MQTT Connect (in development)** - using the MQTT Connect Client to connect to MQTT Connect
-* **Data Mapper** - handling of received messages via connector and mapping them to a target data format for Cumulocity IoT.
+* three default connectors for..
+ * **MQTT client** - using [hivemq-mqtt-client](https://github.com/hivemq/hivemq-mqtt-client) to connect and subscribe to MQTT brokers
+ * **MQTT Service client** - using hivemq-mqtt-client to connect to MQTT Service
+ * **Kafka connector** - to connect to Kafka brokers
+* **Data mapper** - handling of received messages via connector and mapping them to a target data format for Cumulocity IoT.
Also includes an expression runtime [JSONata](https://jsonata.org) to execute expressions
-* **C8Y Client** - implements part of the Cumulocity IoT REST API to integrate data
-* **REST Endpoints** - custom endpoints which are used by the MQTT Frontend or can be used to add mappings programmatically
-* **Mapper Frontend** - A plugin for Cumulocity IoT to provide an UI for MQTT Configuration & Data Mapping
+* **C8Y client** - implements part of the Cumulocity IoT REST API to integrate data
+* **REST endpoints** - custom endpoints which are used by the MQTT Frontend or can be used to add mappings programmatically
+* **Mapper frontend** - A plugin for Cumulocity IoT to provide an UI for MQTT Configuration & Data Mapping
-> **Please Note:** When using MQTT or any other Message Broker beside MQTT Connect you need an instance of this broker available to use the Dynamic Mapper.
+> **Please Note:** When using MQTT or any other Message Broker beside MQTT Service you need to provide this broker available yourself to use the Dynamic Mapper.
The mapper processes messages in both directions:
1. `INBOUND`: from Message Broker to C8Y
2. `OUTBOUND`: from C8Y to Message Broker
-The Dynamic Mapper is a **multi tenant microservice** which means you can deploy it once in your enterprise tenant and subscribe additional tenants using the same hardware resources.
+The Dynamic Mapper can be deployed as a **multi tenant microservice** which means you can deploy it once in your enterprise tenant and subscribe additional tenants using the same hardware resources.
It is also implemented to support **multiple broker connections** at the same time. So you can combine multiple message brokers sharing the same mappings.
This implies of course that all of them use the same topic structure and payload otherwise the mappings will fail.
@@ -79,23 +82,19 @@ This implies of course that all of them use the same topic structure and payload
As we already have a very good C8Y API coverage for mapping not all complex cases might be supported. Currently, the
following Mappings are supported:
-* Inventory
-* Events
-* Measurements
-* Alarms
-* Operations (Outbound to devices)
-
-Beside that complex JSON objects & arrays are supported but not fully tested.
+* inventory
+* events
+* measurements
+* alarms
+* operations (outbound to devices)
-Due to two different libraries to evaluate JSONata in:
+A mapping is defined of mapping properties and substitutions. The substitutions are mapping rules copying date from the incoming payload to the payload in the target system. These substitutions are defined using the standard JSONata as JSONata expressions. These JSONata expressions are evaluated in two different libraries:
1. `dynamic-mapping-ui`: (nodejs) [npmjs JSONata](https://www.npmjs.com/package/jsonata) and
2. `dynamic-mapping-service` (java): [JSONata4Java](https://github.com/IBM/JSONata4Java)
+Please be aware that slight in differences in the evaluation of these expressions exist.
Differences in more advanced expressions can occur. Please test your expressions before you use advanced elements.
-The Paho java client uses memory persistence to persist its state (used to store outbound and inbound messages while they are in flight). When the microservice restarts this information is lost.
-The microservice can not use the default `MqttDefaultFilePersistence` of the paho client. See [Issue](https://github.com/eclipse/paho.mqtt.java/issues/507)
-
### Contribution
> **Pull Requests adding connectors, mappings for other data formats or additional functionally are welcomed!**
@@ -112,10 +111,10 @@ Make sure to use a user with admin privileges in your Tenant.
You need to install two components to your Cumulocity IoT Tenant:
-1. Microservice - (Java)
-2. WebApp Plugin - (Angular/Cumulocity WebSDK)
+1. microservice - (Java)
+2. web app plugin - (angular/Cumulocity WebSDK)
-Both are provided as binaries in [Releases](https://github.com/SoftwareAG/cumulocity-generic-mqtt-agent/releases). Take
+Both are provided as binaries in [releases](https://github.com/SoftwareAG/cumulocity-generic-mqtt-agent/releases). Download
the binaries from the latest release and upload them to your Cumulocity IoT Tenant.
#### Microservice
@@ -125,9 +124,9 @@ In your Enterprise Tenant or Tenant navigate to "Administration" App, go to "Eco
Select the `dynamic-mapping-service.zip`.
Make sure that you subscribe the microservice to your tenant when prompted
-#### Web App Plugin
+#### Web app plugin
-#### Community Store
+#### Community store
The Web App Plugin is part of the community plugins and should be available directly in your Tenant under
"Administration" -> "Ecosystem" -> "Extensions". Just click on "dynamic-mapping" and click on "install plugin".
@@ -173,7 +172,7 @@ The Frontend is build as [Cumulocity plugin](https://cumulocity.com/guides/web/t
## User Guide
### Permissions
-The solution differentiates two different roles:
+The solution differentiates between two different roles:
1. `ROLE_MAPPING_ADMIN`: can use/access all tabs, including **Configuration**, **Processor Extension**. In addition, the relevant endpoints in `MappingRestController`:
1.1. `POST /configuration/connection`
@@ -195,33 +194,38 @@ The available tabs for `ROLE_MAPPING_CREATE` are as follows:
### Configuration connector to broker
-The configurations are persisted in the tenant options of a Cumulocity Tenant and can be manged by the following UI.\
+The configurations are persisted as tenant options in the Cumulocity Tenant and can be manged using the following UI.\
The table of configured connectors to different brokers can be:
* deleted
* enabled / disabled
-* updated
+* updated / copied
-
+
+
+Furthermore, new connectors can be added. The UI is shown on the following screenshot. In the modal dialog you have to first select the type of connector. Currently we support the following connectors:
+* MQTT: supports connections to MQTT version 3.1.1 over websocket and tcp
+* MQTT Service: this connector is a special case of the MQTT connector, to connect to the Cumulocity MQTT Service
+* Kafka: is an initial implementation for connecting to Kafka brokers. It is expected that the implementation of the connector has to be adapted to the specific needs of your project. This applies to configuration for security, transactions, key and payload serialization ( currently StringSerializer)...
+
+The configuration properties are dynamically adapted to the configuration parameter for the chosen connector type:
-
+
-Furthermore, new connectors can be added. The UI is shown on the following screenshot. In the modal dialog you have to select first the type of connector: MQTT, MQTT Connect, Kafka, ... Then the input is dynamically adapted to the configuration paramaeter for the chosen connector type:
+The settings for the Kafka connector can be seen on the following screenshot:
-When you add or change a connection configuration very often it happens that incorrect parameter are given. In this case the connection to the MQTT broker cannot be established and the reason is not known. To identify the incorrect parameter you can follows the error messages in the connections logs:
+
+When you add or change a connection configuration it happens very often that the parameter are incorrect and the connection fails. In this case the connection to the MQTT broker cannot be established and the reason is not known. To identify the incorrect parameter you can follows the error messages in the connections logs on the same UI:
@@ -235,8 +239,10 @@ Once the connection to a broker is configured and successfully enabled you can s
1. Creating new mappings: Press button `Add mapping`
2. Updating existing mapping: Press the pencil in the row of the relevant mapping
3. Deleting existing mapping: Press the "-" icon in the row of the relevant mapping to delete an existing mappings
+4. Importing new mappings
+5. Exporting defined mappings
-After every change the mappings is automatically updated in the mapping cache of the microservice.
+To change a mapping it has to be deactivated.After changes are made the mapping needs to be activated again. The updated version of the mapping is deployed automatically and applied immediately when new messages are sent to the configure mapping topic.
#### Define mappings from source to target format (Cumulocity REST format)
@@ -266,7 +272,7 @@ Further example for JSONata expressions are:
Creation of the new mapping starts by pressing `Add Mapping`. On the next modal UI you can choose the mapping type depending on the structure of your payload. Currently there is support for:
1. `JSON`: if your payload is in JSON format
-1. `FLAT_FILE`: if your payload is in a csv format
+1. `FLAT_FILE`: if your payload is in a CSV format
1. `GENERIC_BINARY`: if your payload is in HEX format
1. `PROTOBUF_STATIC`: if your payload is a serialized protobuf message
1. `PROCESSOR_EXTENSION`: if you want to process the message yourself, by registering a processor extension
@@ -304,13 +310,17 @@ splits the payload and return the second field: ```100```.
And for the binary payload is encoded as hex string:
```
{
- "message": "5a75207370c3a47420303821",
+ "message": "0x575",
}
```
Using appropriate JSONata expression you can parse the payload:
```
-$parseInteger($string("0x"&$substring(message,0,2)),"0")&" C"
+$number(message) & " C"
```
+
+> **Please Note:** Currently this works only with a pached version of the [JSONata library](https://github.com/IBM/JSONata4Java) due to the missing support for hexadecimal number in the current in the original version. The original implementation of the `$number()` function only works for decimal numbers. An [issue](https://github.com/IBM/JSONata4Java/issues/305) is pending for resolution.
+The JSONata function `$parseInteger()` is not supported by [JSONata library](https://github.com/IBM/JSONata4Java) and can't be used.
+
___
1. Define the properties of the topic and API to be used
@@ -349,17 +359,23 @@ For an outbound mapping to be applied two conditions have to be fulfilled:
##### Subscription Topic
This is the topic which is actually subscribed on in the broker. It can contain wildcards, either single level "+" or multilevel "#".
-This occurs must be supported by the configured message broker.
+This must be supported by the configured message broker.
>**_NOTE:_** Multi-level wildcards can only appear at the end of topic. The topic "/device/#/west" is not valid.
Examples of valid topics are: "device/#", "device/data/#", "device/12345/data" etc.
-##### Template Topic
+##### Mapping Topic
The template topic is the key of the persisted mapping. The main difference to the subscription topic is that
a template topic can have a path behind the wildcard for the reason as we can receive multiple topics on a wildcard which might be mapped differently.\
Examples are: "device/+/data, "device/express/+", "device/+"\
-In order to use sample data instead of the wildcard you can add a Template Topic Sample, which must have the same structure, i.e. same level in the topic and when explicit name are used at a topic level in the Template Topic they must exactly be the same in the Template Topic Sample.
-The levels of the Template Topic are split and added to the payload:
+In order to use sample data instead of the wildcard you can add a Mapping Topic Sample, which must have the same structure, i.e. same level in the topic and when explicit name are used at a topic level in the Mapping Topic they must exactly be the same in the Mapping Topic Sample.
+
+
+
+
+
+
+The levels of the Mapping Topic are split and added to the payload:
```
"_TOPIC_LEVEL_": [
"device",
@@ -431,7 +447,7 @@ To define a new substitution the following steps have to be performed:
* ```multi-device-multi-value```
* ```single-device-multi-value```\
Otherwise an extracted array is treated as a single value, see [Different type of substitutions](#different-type-of-substitutions).
- 1. Select option ```Resolve to externalId``` if you want to resolve system Cumulocity Id to externalId using externalIdType. This can onlybe used for OUTBOUND mappings.
+ 1. Select option ```Resolve to externalId``` if you want to resolve system Cumulocity Id to externalId using externalIdType. This can only be used for OUTBOUND mappings.
1. Select a ```Reapir Strategy``` that determines how the mapping is applied:
* ```DEFAULT```: Map the extracted values to the attribute addressed on right side
* ```USE_FIRST_VALUE_OF_ARRAY```: When the left side of the mapping returns an array, only use the 1. item in the array and map this to the right side
@@ -649,7 +665,7 @@ The mapping microservice provides endpoints to control the lifecycle and manage
## Tests & Sample Data
### Load Test
-In the resource section you find a test profil [jmeter_test_01.jmx](./resources/script/performance/jmeter_test_01.jmx) using the performance tool ```jmeter``` and an extension for mqtt: [emqx/mqtt-jmete](https://github.com/emqx/mqtt-jmeter).
+In the resource section you find a test profil [jmeter_test_01.jmx](./resources/script/performance/jmeter_test_01.jmx) using the performance tool ```jmeter``` and an extension for MQTT: [emqx/mqtt-jmeter](https://github.com/emqx/mqtt-jmeter).
This was used to run simple loadtest.
## Setup Sample mappings
diff --git a/dynamic-mapping-extension/pom.xml b/dynamic-mapping-extension/pom.xml
index 76c0eaa2..fe4e12ad 100644
--- a/dynamic-mapping-extension/pom.xml
+++ b/dynamic-mapping-extension/pom.xml
@@ -71,9 +71,14 @@
lombok
compile
+
- org.eclipse.paho
- org.eclipse.paho.client.mqttv3
+ com.hivemq
+ hivemq-mqtt-client
test
diff --git a/dynamic-mapping-extension/src/main/java/dynamic/mapping/processor/extension/external/ProcessorExtensionInboundCustomMeasurement.java b/dynamic-mapping-extension/src/main/java/dynamic/mapping/processor/extension/external/ProcessorExtensionInboundCustomMeasurement.java
index 058f44ad..87beffc5 100644
--- a/dynamic-mapping-extension/src/main/java/dynamic/mapping/processor/extension/external/ProcessorExtensionInboundCustomMeasurement.java
+++ b/dynamic-mapping-extension/src/main/java/dynamic/mapping/processor/extension/external/ProcessorExtensionInboundCustomMeasurement.java
@@ -32,7 +32,6 @@
import dynamic.mapping.processor.model.RepairStrategy;
import lombok.extern.slf4j.Slf4j;
import org.joda.time.DateTime;
-import org.springframework.stereotype.Component;
import javax.ws.rs.ProcessingException;
import java.io.IOException;
diff --git a/dynamic-mapping-extension/src/test/java/dynamic/mapping/ProtobufPahoClient.java b/dynamic-mapping-extension/src/test/java/dynamic/mapping/ProtobufPahoClient.java
index 7037e692..501e14b3 100644
--- a/dynamic-mapping-extension/src/test/java/dynamic/mapping/ProtobufPahoClient.java
+++ b/dynamic-mapping-extension/src/test/java/dynamic/mapping/ProtobufPahoClient.java
@@ -21,68 +21,81 @@
package dynamic.mapping;
-import org.eclipse.paho.client.mqttv3.MqttClient;
-import org.eclipse.paho.client.mqttv3.MqttConnectOptions;
-import org.eclipse.paho.client.mqttv3.MqttException;
-import org.eclipse.paho.client.mqttv3.MqttMessage;
-import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence;
+import java.util.Date;
+
+import com.hivemq.client.mqtt.datatypes.MqttQos;
+import com.hivemq.client.mqtt.mqtt3.Mqtt3AsyncClient;
+import com.hivemq.client.mqtt.mqtt3.Mqtt3BlockingClient;
+import com.hivemq.client.mqtt.mqtt3.Mqtt3Client;
+import com.hivemq.client.mqtt.mqtt3.message.auth.Mqtt3SimpleAuth;
+import com.hivemq.client.mqtt.mqtt3.message.connect.connack.Mqtt3ConnAck;
+import com.hivemq.client.mqtt.mqtt3.message.connect.connack.Mqtt3ConnAckReturnCode;
import dynamic.mapping.processor.extension.external.CustomEventOuter;
import dynamic.mapping.processor.extension.external.CustomEventOuter.CustomEvent;
-
public class ProtobufPahoClient {
-
- static MemoryPersistence persistence = new MemoryPersistence();
+ Mqtt3BlockingClient testClient;
+ static String broker_host = System.getenv("broker_host");
+ static Integer broker_port = Integer.valueOf(System.getenv("broker_port"));
+ static String client_id = System.getenv("client_id");
+ static String broker_username = System.getenv("broker_username");
+ static String broker_password = System.getenv("broker_password");
+
+ public ProtobufPahoClient(Mqtt3BlockingClient sampleClient) {
+ testClient = sampleClient;
+ }
public static void main(String[] args) {
- ProtobufPahoClient client = new ProtobufPahoClient();
+ Mqtt3SimpleAuth simpleAuth = Mqtt3SimpleAuth.builder().username(broker_username)
+ .password(broker_password.getBytes()).build();
+ Mqtt3BlockingClient sampleClient = Mqtt3Client.builder()
+ .serverHost(broker_host)
+ .serverPort(broker_port)
+ .identifier(client_id)
+ .simpleAuth(simpleAuth)
+ .sslWithDefaultConfig()
+ .buildBlocking();
+ ProtobufPahoClient client = new ProtobufPahoClient(sampleClient);
client.testSendEvent();
}
private void testSendEvent() {
- int qos = 0;
- String broker = System.getenv("broker");
- String client_id = System.getenv("client_id");
- String broker_username = System.getenv("broker_username");
- String broker_password = System.getenv("broker_password");
- String topic2 = "protobuf/event";
-
- try {
- MqttClient sampleClient = new MqttClient(broker, client_id, persistence);
- MqttConnectOptions connOpts = new MqttConnectOptions();
- connOpts.setUserName(broker_username);
- connOpts.setPassword(broker_password.toCharArray());
- connOpts.setCleanSession(true);
-
- System.out.println("Connecting to broker: " + broker);
-
- sampleClient.connect(connOpts);
-
- System.out.println("Publishing message: :::");
+ String topic = "protobuf/event";
+
+ System.out.println("Connecting to broker: ssl://" + broker_host + ":" + broker_port);
+
+ // testClient.connect();
+ Mqtt3ConnAck ack = testClient.connectWith()
+ .cleanSession(true)
+ .keepAlive(60)
+ .send();
+ if (!ack.getReturnCode().equals(Mqtt3ConnAckReturnCode.SUCCESS)) {
+ // throw new ConnectorException("Tenant " + tenant + " - Error connecting to
+ // broker:"
+ // + mqttClient.getConfig().getServerHost() + ". Errorcode: "
+ // + ack.getReturnCode().name());
+ System.out.println("Error connecting to broker:"
+ + broker_host + ". Errorcode: "
+ + ack.getReturnCode().name());
+ }
- CustomEventOuter.CustomEvent proto = CustomEvent.newBuilder()
- .setExternalIdType("c8y_Serial")
- .setExternalId("berlin_01")
- .setTxt("Dummy Text")
- .setEventType("c8y_ProtobufEventType")
- .setTimestamp(System.currentTimeMillis())
- .build();
+ System.out.println("Publishing message on topic" + topic);
- MqttMessage message = new MqttMessage(proto.toByteArray());
- message.setQos(qos);
- sampleClient.publish(topic2, message);
+ CustomEventOuter.CustomEvent proto = CustomEvent.newBuilder()
+ .setExternalIdType("c8y_Serial")
+ .setExternalId("berlin_01")
+ .setTxt("Stop at petrol station: " + (new Date().toString()))
+ .setEventType("c8y_ProtobufEventType")
+ .setTimestamp(System.currentTimeMillis())
+ .build();
- System.out.println("Message published");
- sampleClient.disconnect();
- System.out.println("Disconnected");
- //System.exit(0);
+ Mqtt3AsyncClient sampleClientAsync = testClient.toAsync();
+ sampleClientAsync.publishWith().topic(topic).qos(MqttQos.AT_LEAST_ONCE).payload(proto.toByteArray()).send();
- } catch (MqttException me) {
- System.out.println("Exception:" + me.getMessage());
- me.printStackTrace();
- }
+ System.out.println("Message published");
+ testClient.disconnect();
+ System.out.println("Disconnected");
}
-
}
diff --git a/dynamic-mapping-extension/src/test/java/dynamic/mapping/processor/ProcessorExtensionInboundTest.java b/dynamic-mapping-extension/src/test/java/dynamic/mapping/processor/ProcessorExtensionInboundTest.java
index 4c106edd..0ae2c725 100644
--- a/dynamic-mapping-extension/src/test/java/dynamic/mapping/processor/ProcessorExtensionInboundTest.java
+++ b/dynamic-mapping-extension/src/test/java/dynamic/mapping/processor/ProcessorExtensionInboundTest.java
@@ -50,7 +50,7 @@ void testDeserializeCustomEvent() {
.build();
ProcessorExtensionInboundCustomEvent extension = new ProcessorExtensionInboundCustomEvent();
- ProcessingContext context = new ProcessingContext();
+ ProcessingContext context = new ProcessingContext<>();
context.setPayload(proto.toByteArray());
Mapping m1 = new Mapping();
m1.setTargetAPI(API.EVENT);
diff --git a/dynamic-mapping-interface/pom.xml b/dynamic-mapping-interface/pom.xml
index 2f849496..85b94115 100644
--- a/dynamic-mapping-interface/pom.xml
+++ b/dynamic-mapping-interface/pom.xml
@@ -45,9 +45,13 @@
protobuf-java
- org.eclipse.paho
- org.eclipse.paho.client.mqttv3
+ com.fasterxml.jackson.datatype
+ jackson-datatype-joda
+
+ com.hivemq
+ hivemq-mqtt-client
+
com.ibm.jsonata4java
JSONata4Java
@@ -78,7 +82,15 @@
slf4j-api
provided
-
+
+ org.apache.kafka
+ kafka-clients
+
+
+ com.github.loki4j
+ loki-logback-appender
+
+
../dynamic-mapping-service/src/main/java
diff --git a/dynamic-mapping-service/pom.xml b/dynamic-mapping-service/pom.xml
index 470efd61..16733777 100644
--- a/dynamic-mapping-service/pom.xml
+++ b/dynamic-mapping-service/pom.xml
@@ -37,12 +37,25 @@
dynamic-mapping-service
+
+
+
+ com.ibm.jsonata4java
+ JSONata4Java
+ 2.4.9
+
+
+
+
-
- com.fasterxml.jackson.datatype
- jackson-datatype-joda
- provided
-
+
+ io.netty
+ netty-all
+
+
+ com.fasterxml.jackson.datatype
+ jackson-datatype-joda
+
org.json
json
@@ -65,12 +78,8 @@
protobuf-java
- org.eclipse.paho
- org.eclipse.paho.client.mqttv3
-
-
- com.ibm.jsonata4java
- JSONata4Java
+ com.hivemq
+ hivemq-mqtt-client
org.apache.commons
@@ -89,9 +98,37 @@
junit-jupiter
test
+
+ org.apache.kafka
+ kafka-clients
+
+
+ org.springframework
+ spring-aspects
+
+
+ io.opentelemetry.instrumentation
+ opentelemetry-spring-boot-starter
+
+
+ com.ibm.jsonata4java
+ JSONata4JavaFIX
+ 2.4.8-fix
+
+
+
+
+ true
+ src/main/resources
+
+
+
org.springframework.boot
diff --git a/dynamic-mapping-service/src/main/configuration/cumulocity.json b/dynamic-mapping-service/src/main/configuration/cumulocity.json
index ad643785..d5de83d9 100644
--- a/dynamic-mapping-service/src/main/configuration/cumulocity.json
+++ b/dynamic-mapping-service/src/main/configuration/cumulocity.json
@@ -25,7 +25,8 @@
"ROLE_NOTIFICATION_2_ADMIN",
"ROLE_USER_MANAGEMENT_READ",
"ROLE_USER_MANAGEMENT_CREATE",
- "ROLE_USER_MANAGEMENT_ADMIN"
+ "ROLE_USER_MANAGEMENT_ADMIN",
+ "ROLE_MQTT_SERVICE_ADMIN"
],
"roles":[
"ROLE_MAPPING_ADMIN",
diff --git a/dynamic-mapping-service/src/main/configuration/dynamic-mapping-service-logging.xml b/dynamic-mapping-service/src/main/configuration/dynamic-mapping-service-logging.xml
new file mode 100644
index 00000000..45b3fafa
--- /dev/null
+++ b/dynamic-mapping-service/src/main/configuration/dynamic-mapping-service-logging.xml
@@ -0,0 +1,16 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/App.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/App.java
index 8a3f31ad..62815f9e 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/App.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/App.java
@@ -28,11 +28,14 @@
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
-import dynamic.mapping.model.TreeNode;
-import dynamic.mapping.model.TreeNodeSerializer;
+import dynamic.mapping.model.MappingTreeNode;
+import dynamic.mapping.model.MappingTreeNodeSerializer;
+import io.micrometer.core.instrument.MeterRegistry;
import org.joda.time.DateTime;
+import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.SpringApplication;
+import org.springframework.boot.actuate.autoconfigure.metrics.MeterRegistryCustomizer;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Bean;
import org.springframework.core.task.TaskExecutor;
@@ -74,12 +77,18 @@
import com.fasterxml.jackson.datatype.joda.JodaModule;
import lombok.SneakyThrows;
+
@MicroserviceApplication
@EnableContextSupport
@SpringBootApplication
@EnableAsync
@EnableScheduling
public class App {
+ @Bean
+ MeterRegistryCustomizer configurer(
+ @Value("${application.name}") String applicationName) {
+ return (registry) -> registry.config().commonTags("application", applicationName);
+ }
@Bean
public TaskExecutor taskExecutor() {
@@ -100,7 +109,7 @@ public ObjectMapper objectMapper() {
ObjectMapper objectMapper = baseObjectMapper();
objectMapper.registerModule(cumulocityModule());
SimpleModule module = new SimpleModule();
- module.addSerializer(TreeNode.class, new TreeNodeSerializer());
+ module.addSerializer(MappingTreeNode.class, new MappingTreeNodeSerializer());
objectMapper.registerModule(module);
return objectMapper;
}
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/configuration/ConnectorConfiguration.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/configuration/ConnectorConfiguration.java
index 36359a13..33f907ee 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/configuration/ConnectorConfiguration.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/configuration/ConnectorConfiguration.java
@@ -3,6 +3,10 @@
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonSetter;
import com.fasterxml.jackson.annotation.Nulls;
+
+import dynamic.mapping.connector.core.ConnectorProperty;
+import dynamic.mapping.connector.core.ConnectorSpecification;
+import dynamic.mapping.connector.core.client.ConnectorType;
import lombok.Data;
import lombok.ToString;
@@ -26,7 +30,7 @@ public ConnectorConfiguration() {
@NotNull
@JsonSetter(nulls = Nulls.SKIP)
@JsonProperty("connectorType")
- public String connectorType;
+ public ConnectorType connectorType;
@NotNull
@JsonProperty("enabled")
@@ -73,11 +77,19 @@ public Object clone() {
return null;
}
return result;
- // this way we don't get a deep clone
- // try {
- // return super.clone();
- // } catch (CloneNotSupportedException e) {
- // return null;
- // }
+ }
+
+ /**
+ * Copy the properties that are readonly from the specification to the configuration
+ * @param spec the connectorSpecification to use as a template and copy predefined from to the connectorConfiguration
+ */
+ public void copyPredefinedValues(ConnectorSpecification spec) {
+
+ spec.getProperties().entrySet().forEach(prop -> {
+ ConnectorProperty p = prop.getValue();
+ if (p.readonly) {
+ properties.put(prop.getKey(), p.defaultValue);
+ }
+ });
}
}
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/configuration/ConnectorConfigurationComponent.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/configuration/ConnectorConfigurationComponent.java
index 0f50f4c1..8fcd27fe 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/configuration/ConnectorConfigurationComponent.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/configuration/ConnectorConfigurationComponent.java
@@ -146,7 +146,7 @@ public void deleteConnectorConfigurations(String tenant) {
List configs = getConnectorConfigurations(tenant);
for (ConnectorConfiguration config : configs) {
OptionPK optionPK = new OptionPK(OPTION_CATEGORY_CONFIGURATION,
- getConnectorOptionKey(config.getConnectorType()));
+ getConnectorOptionKey(config.getIdent()));
tenantOptionApi.delete(optionPK);
}
}
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/configuration/ServiceConfiguration.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/configuration/ServiceConfiguration.java
index 7124a527..70af3870 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/configuration/ServiceConfiguration.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/configuration/ServiceConfiguration.java
@@ -39,7 +39,7 @@ public ServiceConfiguration() {
this.logSubstitution = false;
this.logConnectorErrorInBackend = false;
this.sendConnectorLifecycle = false;
- this.sendMappingStatus = false;
+ this.sendMappingStatus = true;
this.sendSubscriptionEvents = false;
this.sendNotificationLifecycle = false;
this.externalExtensionEnabled = true;
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/configuration/ServiceConfigurationComponent.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/configuration/ServiceConfigurationComponent.java
index b7eede14..c311eba2 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/configuration/ServiceConfigurationComponent.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/configuration/ServiceConfigurationComponent.java
@@ -84,7 +84,7 @@ public ServiceConfiguration getServiceConfiguration(String tenant) {
ServiceConfiguration.class);
}
log.debug("Tenant {} - Returning service configuration found: {}:", tenant, rt.logPayload);
- log.info("Tenant {} - Found connection configuration: {}", tenant, rt);
+ log.debug("Tenant {} - Found connection configuration: {}", tenant, rt);
} catch (SDKException exception) {
log.warn("Tenant {} - No configuration found, returning empty element!", tenant);
rt = initialize(tenant);
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/ConnectorProperty.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/ConnectorProperty.java
index d72f4334..c5f4b411 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/ConnectorProperty.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/ConnectorProperty.java
@@ -6,6 +6,8 @@
import lombok.Data;
import lombok.ToString;
+import java.util.Map;
+
import javax.validation.constraints.NotNull;
@Data
@@ -25,8 +27,23 @@ public class ConnectorProperty implements Cloneable {
@JsonSetter(nulls = Nulls.SKIP)
public ConnectorPropertyType type;
- public Object clone()
- {
+ @NotNull
+ @JsonSetter(nulls = Nulls.SKIP)
+ public Boolean readonly;
+
+ @NotNull
+ @JsonSetter(nulls = Nulls.SKIP)
+ public Boolean hidden;
+
+ @NotNull
+ @JsonSetter(nulls = Nulls.SKIP)
+ public Object defaultValue;
+
+ @NotNull
+ @JsonSetter(nulls = Nulls.SKIP)
+ public Map options;
+
+ public Object clone() {
try {
return super.clone();
} catch (CloneNotSupportedException e) {
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/ConnectorPropertyType.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/ConnectorPropertyType.java
index 1f224e02..ef1a72dd 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/ConnectorPropertyType.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/ConnectorPropertyType.java
@@ -1,8 +1,11 @@
package dynamic.mapping.connector.core;
public enum ConnectorPropertyType {
+ ID_STRING_PROPERTY,
STRING_PROPERTY,
SENSITIVE_STRING_PROPERTY,
NUMERIC_PROPERTY,
- BOOLEAN_PROPERTY
+ BOOLEAN_PROPERTY,
+ OPTION_PROPERTY,
+ STRING_LARGE_PROPERTY,
}
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/ConnectorSpecification.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/ConnectorSpecification.java
index 2a1d219b..3388c9d8 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/ConnectorSpecification.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/ConnectorSpecification.java
@@ -2,6 +2,8 @@
import com.fasterxml.jackson.annotation.JsonSetter;
import com.fasterxml.jackson.annotation.Nulls;
+
+import dynamic.mapping.connector.core.client.ConnectorType;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.ToString;
@@ -16,16 +18,20 @@ public class ConnectorSpecification implements Cloneable {
@NotNull
@JsonSetter(nulls = Nulls.SKIP)
- public String connectorType;
+ public String description;
@NotNull
@JsonSetter(nulls = Nulls.SKIP)
- public boolean supportsWildcardInTopic;
+ public ConnectorType connectorType;
@NotNull
@JsonSetter(nulls = Nulls.SKIP)
public Map properties;
+ @NotNull
+ @JsonSetter(nulls = Nulls.SKIP)
+ public boolean supportsMessageContext;
+
public boolean isPropertySensitive(String property) {
return ConnectorPropertyType.SENSITIVE_STRING_PROPERTY == properties.get(property).type;
}
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/callback/ConnectorMessage.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/callback/ConnectorMessage.java
index 96dc7aea..99550682 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/callback/ConnectorMessage.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/callback/ConnectorMessage.java
@@ -12,15 +12,19 @@
public class ConnectorMessage {
private byte[] payload;
+ private byte[] key;
+
private String[] headers;
@NotNull
private String tenant;
private String topic;
-
+
@NotNull
private String connectorIdent;
private boolean sendPayload;
+
+ private boolean supportsMessageContext;
}
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/callback/GenericMessageCallback.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/callback/GenericMessageCallback.java
index c428fbd6..b54fc3ea 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/callback/GenericMessageCallback.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/callback/GenericMessageCallback.java
@@ -3,7 +3,7 @@
public interface GenericMessageCallback {
void onClose(String closeMessage, Throwable closeException);
- void onMessage(ConnectorMessage message) throws Exception;
+ void onMessage(ConnectorMessage message);
void onError( Throwable errorException);
}
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/client/AConnectorClient.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/client/AConnectorClient.java
index 0a7b02b5..688adef9 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/client/AConnectorClient.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/client/AConnectorClient.java
@@ -31,6 +31,7 @@
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
@@ -40,10 +41,11 @@
import dynamic.mapping.connector.core.ConnectorSpecification;
import dynamic.mapping.model.Mapping;
import dynamic.mapping.model.MappingServiceRepresentation;
+import dynamic.mapping.model.QOS;
import dynamic.mapping.processor.inbound.AsynchronousDispatcherInbound;
+import org.apache.commons.lang3.mutable.MutableBoolean;
import org.apache.commons.lang3.mutable.MutableInt;
-import org.eclipse.paho.client.mqttv3.MqttException;
import org.joda.time.DateTime;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -68,10 +70,24 @@
@Slf4j
public abstract class AConnectorClient {
+ protected static final int WAIT_PERIOD_MS = 10000;
+
protected String connectorIdent;
protected String connectorName;
+ protected String additionalSubscriptionIdTest;
+
+ protected MutableBoolean connectionState = new MutableBoolean(false);
+
+ @Getter
+ @Setter
+ public ConnectorSpecification specification;
+
+ @Getter
+ @Setter
+ public ConnectorType connectorType;
+
@Getter
@Setter
protected String tenant;
@@ -108,11 +124,21 @@ public abstract class AConnectorClient {
private Future> initializeTask;
+ // keeps track how many active mappings use this topic as subscriptionTopic:
// structure < subscriptionTopic, numberMappings >
public Map activeSubscriptions = new HashMap<>();
+ // keeps track if a specific mapping is deployed in this connector:
+ // a) is it active,
+ // b) does it comply with the capabilities of the connector, i.e. supports
+ // wildcards
+ // structure < ident, mapping >
+ @Getter
+ private Map mappingsDeployed = new ConcurrentHashMap<>();
private Instant start = Instant.now();
+ private ConnectorStatus previousConnectorStatus = ConnectorStatus.UNKNOWN;
+
@Getter
@Setter
public ConnectorConfiguration connectorConfiguration;
@@ -125,9 +151,13 @@ public abstract class AConnectorClient {
@Setter
public ConnectorStatusEvent connectorStatus = ConnectorStatusEvent.unknown();
+ @Getter
+ @Setter
+ public Boolean supportsMessageContext;
+
public void submitInitialize() {
// test if init task is still running, then we don't need to start another task
- log.info("Tenant {} - Called initialize(): {}", tenant, initializeTask == null || initializeTask.isDone());
+ log.debug("Tenant {} - Called initialize(): {}", tenant, initializeTask == null || initializeTask.isDone());
if ((initializeTask == null || initializeTask.isDone())) {
initializeTask = cachedThreadPool.submit(() -> initialize());
}
@@ -135,110 +165,123 @@ public void submitInitialize() {
public abstract boolean initialize();
- public abstract ConnectorSpecification getSpecification();
+ public abstract Boolean supportsWildcardsInTopic();
public void loadConfiguration() {
- connectorConfiguration = connectorConfigurationComponent.getConnectorConfiguration(this.getConnectorIdent(), tenant);
+ connectorConfiguration = connectorConfigurationComponent.getConnectorConfiguration(this.getConnectorIdent(),
+ tenant);
+ this.connectorConfiguration.copyPredefinedValues(getSpecification());
// get the latest serviceConfiguration from the Cumulocity backend in case
// someone changed it in the meantime
// update the in the registry
serviceConfiguration = serviceConfigurationComponent.getServiceConfiguration(tenant);
configurationRegistry.getServiceConfigurations().put(tenant, serviceConfiguration);
- connectorStatus.updateStatus(ConnectorStatus.CONFIGURED, true);
- sendConnectorLifecycle();
+ // updateConnectorStatusAndSend(ConnectorStatus.CONFIGURED, true, true);
}
public void submitConnect() {
+ loadConfiguration();
// test if connect task is still running, then we don't need to start another
// task
- log.info("Tenant {} - Called connect(): connectTask.isDone() {}", tenant,
+ log.debug("Tenant {} - Called connect(): connectTask.isDone() {}", tenant,
connectTask == null || connectTask.isDone());
if (connectTask == null || connectTask.isDone()) {
connectTask = cachedThreadPool.submit(() -> connect());
}
- connectorStatus.updateStatus(ConnectorStatus.CONNECTING, true);
- sendConnectorLifecycle();
}
public void submitDisconnect() {
+ loadConfiguration();
// test if connect task is still running, then we don't need to start another
// task
- log.info("Tenant {} - Called submitDisconnect(): connectTask.isDone() {}", tenant,
+ log.debug("Tenant {} - Called submitDisconnect(): connectTask.isDone() {}", tenant,
connectTask == null || connectTask.isDone());
if (connectTask == null || connectTask.isDone()) {
connectTask = cachedThreadPool.submit(() -> disconnect());
}
- connectorStatus.updateStatus(ConnectorStatus.DISCONNECTING, true);
- sendConnectorLifecycle();
}
public void submitHousekeeping() {
- log.info("Tenant {} - Called submitHousekeeping()", tenant);
+ log.debug("Tenant {} - Called submitHousekeeping()", tenant);
housekeepingExecutor.scheduleAtFixedRate(() -> runHousekeeping(), 0, 30,
TimeUnit.SECONDS);
}
- /***
+ /**
* Connect to the broker
- ***/
+ **/
public abstract void connect();
- /***
+ /**
+ * This method if specifically for Kafka, since it does not have the concept of
+ * a client. Kafka rather supports consumer on topic level. They can fail to
+ * connect
+ **/
+ public abstract void monitorSubscriptions();
+
+ /**
* Should return true when connector is enabled and provided properties are
* valid
- ***/
+ **/
public boolean shouldConnect() {
return isConfigValid(connectorConfiguration) && connectorConfiguration.isEnabled();
}
- /***
+ /**
* Returns true if the connector is currently connected
- ***/
+ **/
public abstract boolean isConnected();
- /***
+ /**
* Disconnect the broker
- ***/
+ **/
public abstract void disconnect();
- /***
+ /**
* Close the connection to broker and release all resources
- ***/
+ **/
public abstract void close();
- /***
+ /**
* Returning the unique ID identifying the connector instance
- ***/
+ **/
public abstract String getConnectorIdent();
- /***
+ /**
* Returning the name of the connector instance
- ***/
+ **/
public abstract String getConnectorName();
- /***
+ /**
* Subscribe to a topic on the Broker
- ***/
- public abstract void subscribe(String topic, Integer qos) throws MqttException;
+ **/
+ public abstract void subscribe(String topic, QOS qos) throws ConnectorException;
- /***
+ /**
* Unsubscribe a topic on the Broker
- ***/
+ **/
public abstract void unsubscribe(String topic) throws Exception;
- /***
+ /**
* Checks if the provided configuration is valid
- ***/
+ **/
public abstract boolean isConfigValid(ConnectorConfiguration configuration);
- /***
+ /**
* This method should publish Cumulocity received Messages to the Connector
* using the provided ProcessContext
* Relevant for Outbound Communication
- ***/
+ **/
public abstract void publishMEAO(ProcessingContext> context);
+ /**
+ * This method is triggered every 30 seconds. It performs the following tasks:
+ * 1. synchronizes snooped payloads with the mapping in the inventory
+ * 2. send an connector lifecycle update
+ * 3. monitor and removes failed subscriptions. This is required for the Kafka
+ * connector
+ **/
public void runHousekeeping() {
try {
Instant now = Instant.now();
@@ -254,18 +297,16 @@ public void runHousekeeping() {
}
mappingComponent.cleanDirtyMappings(tenant);
mappingComponent.sendMappingStatus(tenant);
- // disable since the connector status is submitted as Events with the following
- // method sendConnectorLifecycle()
- // mappingComponent.sendConnectorLifecycle(tenant,
- // getConnectorIdent(),getConnectorStatus(),
- // getConnectorName());
// check if connector is in DISCONNECTED state and then move it to CONFIGURED
// state.
if (ConnectorStatus.DISCONNECTED.equals(connectorStatus.status) && isConfigValid(connectorConfiguration)) {
- connectorStatus.updateStatus(ConnectorStatus.CONFIGURED, true);
+ updateConnectorStatusAndSend(ConnectorStatus.CONFIGURED, true, true);
+ } else {
+ sendConnectorLifecycle();
}
- sendConnectorLifecycle();
+ // remove failed subscriptions
+ monitorSubscriptions();
} catch (Exception ex) {
log.error("Tenant {} - Error during house keeping execution: ", tenant, ex);
}
@@ -280,6 +321,7 @@ public List> test(String topic, boolean sendPayload, Map activeMappingOptional = mappingComponent.getCacheMappingInbound().get(tenant).values()
- .stream()
- .filter(m -> m.id.equals(mapping.id))
- .findFirst();
-
- if (activeMappingOptional.isPresent()) {
- create = false;
- activeMapping = activeMappingOptional.get();
- subscriptionTopicChanged = !mapping.subscriptionTopic.equals(activeMapping.subscriptionTopic);
- }
+ public boolean subscriptionTopicChanged(Mapping mapping) {
+ Boolean subscriptionTopicChanged = false;
+ Mapping activeMapping = null;
+ Optional activeMappingOptional = mappingComponent.getCacheMappingInbound().get(tenant).values()
+ .stream()
+ .filter(m -> m.id.equals(mapping.id))
+ .findFirst();
+
+ if (activeMappingOptional.isPresent()) {
+ activeMapping = activeMappingOptional.get();
+ subscriptionTopicChanged = !mapping.subscriptionTopic.equals(activeMapping.subscriptionTopic);
+ }
+ return subscriptionTopicChanged;
+ }
- if (!getActiveSubscriptions().containsKey(mapping.subscriptionTopic)) {
- getActiveSubscriptions().put(mapping.subscriptionTopic, new MutableInt(0));
- }
- MutableInt updatedMappingSubs = getActiveSubscriptions()
- .get(mapping.subscriptionTopic);
+ public boolean activationChanged(Mapping mapping) {
+ Boolean activationChanged = false;
+ Mapping activeMapping = null;
+ Optional activeMappingOptional = mappingComponent.getCacheMappingInbound().get(tenant).values()
+ .stream()
+ .filter(m -> m.id.equals(mapping.id))
+ .findFirst();
+
+ if (activeMappingOptional.isPresent()) {
+ activeMapping = activeMappingOptional.get();
+ activationChanged = mapping.active != activeMapping.active;
+ }
+ return activationChanged;
+ }
- // consider unsubscribing from previous subscription topic if it has changed
- if (create) {
- updatedMappingSubs.add(1);
- ;
- log.debug("Tenant {} - Subscribing to topic: {}, qos: {}", tenant, mapping.subscriptionTopic,
- mapping.qos.ordinal());
- try {
- subscribe(mapping.subscriptionTopic, mapping.qos.ordinal());
- } catch (MqttException exp) {
- log.error("Tenant {} - Exception when subscribing to topic: {}: ", tenant,
- mapping.subscriptionTopic, exp);
+ /**
+ * This method is called when a mapping is created or an existing mapping is
+ * updated.
+ * It maintains a list of the active subscriptions for this connector.
+ * When a mapping id deleted or deactivated, then it is verified how many other
+ * mapping use the same subscriptionTopic. If there are no other mapping using
+ * the same subscriptionTopic the subscriptionTopic is unsubscribed.
+ * Only inactive mappings can be updated except activation/deactivation.
+ **/
+ public void updateActiveSubscription(Mapping mapping, Boolean create, Boolean activationChanged) {
+ if (isConnected()) {
+ Boolean containsWildcards = mapping.subscriptionTopic.matches(".*[#\\+].*");
+ boolean validDeployment = (supportsWildcardsInTopic() || !containsWildcards);
+ if (validDeployment) {
+ if (!getActiveSubscriptions().containsKey(mapping.subscriptionTopic)) {
+ getActiveSubscriptions().put(mapping.subscriptionTopic, new MutableInt(0));
}
- } else if (subscriptionTopicChanged && activeMapping != null) {
- MutableInt activeMappingSubs = getActiveSubscriptions()
- .get(activeMapping.subscriptionTopic);
- activeMappingSubs.subtract(1);
- if (activeMappingSubs.intValue() <= 0) {
- try {
- unsubscribe(mapping.subscriptionTopic);
- } catch (Exception exp) {
- log.error("Tenant {} - Exception when unsubscribing from topic: {}: ", tenant,
- mapping.subscriptionTopic, exp);
- }
+ if (mapping.active) {
+ getMappingsDeployed().put(mapping.ident, mapping);
+ } else {
+ getMappingsDeployed().remove(mapping.ident);
}
- updatedMappingSubs.add(1);
- if (!getActiveSubscriptions().containsKey(mapping.subscriptionTopic)) {
- log.debug("Tenant {} - Subscribing to topic: {}, qos: {}", tenant, mapping.subscriptionTopic,
- mapping.qos.ordinal());
- try {
- subscribe(mapping.subscriptionTopic, mapping.qos.ordinal());
- } catch (MqttException exp) {
- log.error("Tenant {} - Exception when subscribing to topic: {}: ", tenant,
- mapping.subscriptionTopic, exp);
+ MutableInt updatedMappingSubs = getActiveSubscriptions()
+ .get(mapping.subscriptionTopic);
+
+ // consider unsubscribing from previous subscription topic if it has changed
+ if (create) {
+ if (mapping.active) {
+ updatedMappingSubs.add(1);
+ log.info("Tenant {} - Subscribing to topic: {}, qos: {}", tenant, mapping.subscriptionTopic,
+ mapping.qos);
+ try {
+ subscribe(mapping.subscriptionTopic, mapping.qos);
+ } catch (ConnectorException exp) {
+ log.error("Tenant {} - Exception when subscribing to topic: {}: ", tenant,
+ mapping.subscriptionTopic, exp);
+ }
+ } else {
+ log.error("Tenant {} - Cannot update of active mapping: {}, it is not subscribed to topics ",
+ tenant,
+ mapping.name);
+ }
+ } else {
+ if (mapping.active) {
+ // mapping is activated, we have to subscribe
+ if (updatedMappingSubs.intValue() == 0) {
+ log.info("Tenant {} - Subscribing to topic: {}, qos: {}", tenant,
+ mapping.subscriptionTopic,
+ mapping.qos.ordinal());
+ try {
+ subscribe(mapping.subscriptionTopic, mapping.qos);
+ } catch (ConnectorException exp) {
+ log.error("Tenant {} - Exception when subscribing to topic: {}: ", tenant,
+ mapping.subscriptionTopic, exp);
+ }
+ }
+ updatedMappingSubs.add(1);
+ } else if (activationChanged) {
+ // only unsubscribe if the mapping was deactivated in this call. Otherwise the
+ // mapping was updated which does not result in any changes of the subscription
+ updatedMappingSubs.subtract(1);
+ if (updatedMappingSubs.intValue() <= 0) {
+ try {
+ log.info("Tenant {} - Unsubscribing from topic: {}, qos: {}", tenant, mapping.subscriptionTopic,
+ mapping.qos.ordinal());
+ unsubscribe(mapping.subscriptionTopic);
+ getActiveSubscriptions().remove(mapping.subscriptionTopic);
+ } catch (Exception exp) {
+ log.error("Tenant {} - Exception when unsubscribing from topic: {}: ", tenant,
+ mapping.subscriptionTopic, exp);
+ }
+ }
}
}
}
}
}
+ /**
+ * This method is maintains the list of mappings that are active for this
+ * connector.
+ * If a connector does not support wildcards in this topic subscriptions, i.e.
+ * Kafka, the mapping can't be activated for this connector
+ **/
public void updateActiveSubscriptions(List updatedMappings, boolean reset) {
+
+ mappingsDeployed = new ConcurrentHashMap<>();
if (reset) {
activeSubscriptions = new HashMap();
}
+
if (isConnected()) {
Map updatedSubscriptionCache = new HashMap();
updatedMappings.forEach(mapping -> {
- if (!updatedSubscriptionCache.containsKey(mapping.subscriptionTopic)) {
- updatedSubscriptionCache.put(mapping.subscriptionTopic, new MutableInt(0));
+ Boolean containsWildcards = mapping.subscriptionTopic.matches(".*[#\\+].*");
+ boolean validDeployment = (supportsWildcardsInTopic() || !containsWildcards);
+ if (validDeployment && mapping.isActive()) {
+ if (!updatedSubscriptionCache.containsKey(mapping.subscriptionTopic)) {
+ updatedSubscriptionCache.put(mapping.subscriptionTopic, new MutableInt(0));
+ }
+ MutableInt activeSubs = updatedSubscriptionCache.get(mapping.subscriptionTopic);
+ activeSubs.add(1);
+ mappingsDeployed.put(mapping.ident, mapping);
}
- MutableInt activeSubs = updatedSubscriptionCache.get(mapping.subscriptionTopic);
- activeSubs.add(1);
});
// unsubscribe topics not used
- getActiveSubscriptions().keySet().forEach((topic) -> {
- if (!updatedSubscriptionCache.containsKey(topic)) {
- log.debug("Tenant {} - Unsubscribe from topic: {}", tenant, topic);
+ getActiveSubscriptions().keySet().forEach((subscriptionTopic) -> {
+ if (!updatedSubscriptionCache.containsKey(subscriptionTopic)) {
+ log.debug("Tenant {} - Unsubscribe from topic: {}", tenant, subscriptionTopic);
try {
- unsubscribe(topic);
+ unsubscribe(subscriptionTopic);
} catch (Exception exp) {
- log.error("Tenant {} - Exception when unsubscribing from topic: {}: ", topic, exp);
+ log.error("Tenant {} - Exception when unsubscribing from topic: {}: ", subscriptionTopic, exp);
throw new RuntimeException(exp);
}
}
@@ -402,19 +505,20 @@ public void updateActiveSubscriptions(List updatedMappings, boolean res
// subscribe to new topics
updatedSubscriptionCache.keySet().forEach((topic) -> {
if (!getActiveSubscriptions().containsKey(topic)) {
- int qos = updatedMappings.stream().filter(m -> m.subscriptionTopic.equals(topic))
+ int qosOrdial = updatedMappings.stream().filter(m -> m.subscriptionTopic.equals(topic))
.map(m -> m.qos.ordinal()).reduce(Integer::max).orElse(0);
- log.debug("Tenant {} - Subscribing to topic: {}, qos: {}", tenant, topic, qos);
+ QOS qos = QOS.values()[qosOrdial];
try {
subscribe(topic, qos);
- } catch (MqttException exp) {
+ log.info("Tenant {} - Successfully subscribed to topic: {}, qos: {}", tenant, topic, qos);
+ } catch (ConnectorException exp) {
log.error("Tenant {} - Exception when subscribing to topic: {}: ", tenant, topic, exp);
throw new RuntimeException(exp);
}
}
});
activeSubscriptions = updatedSubscriptionCache;
- log.info("Tenant {} - Updating subscriptions to topics was successful, activeSubscriptions on topic {}",
+ log.info("Tenant {} - Updating subscriptions to topics was successful, active Subscriptions: {}",
tenant, getActiveSubscriptions().size());
}
}
@@ -432,7 +536,9 @@ public void stopHousekeepingAndClose() {
public void sendConnectorLifecycle() {
// stop sending lifecycle event if connector is disabled
- if (serviceConfiguration.sendConnectorLifecycle && connectorConfiguration.enabled) {
+ if (serviceConfiguration.sendConnectorLifecycle
+ && !(connectorStatus.getStatus().equals(previousConnectorStatus))) {
+ previousConnectorStatus = connectorStatus.getStatus();
DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
Date now = new Date();
String date = dateFormat.format(now);
@@ -464,6 +570,36 @@ public void sendSubscriptionEvents(String topic, String action) {
}
}
+ public void connectionLost(String closeMessage, Throwable closeException) {
+ String tenant = getTenant();
+ String connectorIdent = getConnectorIdent();
+ if (closeException != null) {
+ log.error("Tenant {} - Connection lost to broker {}: {}", tenant, connectorIdent,
+ closeException.getMessage());
+ closeException.printStackTrace();
+ }
+ if (closeMessage != null)
+ log.info("Tenant {} - Connection lost to broker: {}", tenant, closeMessage);
+ reconnect();
+ }
+
+ public void updateConnectorStatusAndSend(ConnectorStatus status, boolean clearMessage, boolean send) {
+ connectorStatus.updateStatus(status, clearMessage);
+ if (send) {
+ sendConnectorLifecycle();
+ }
+ }
+
+ protected void updateConnectorStatusToFailed(Exception e) {
+ String msg = " --- " + e.getClass().getName() + ": "
+ + e.getMessage();
+ if (!(e.getCause() == null)) {
+ msg = msg + " --- Caused by " + e.getCause().getClass().getName() + ": " + e.getCause().getMessage();
+ }
+ connectorStatus.setMessage(msg);
+ updateConnectorStatusAndSend(ConnectorStatus.FAILED, false, true);
+ }
+
@Data
@AllArgsConstructor
public static class Certificate {
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/client/ConnectorException.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/client/ConnectorException.java
new file mode 100644
index 00000000..cc490ebc
--- /dev/null
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/client/ConnectorException.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2022 Software AG, Darmstadt, Germany and/or Software AG USA Inc., Reston, VA, USA,
+ * and/or its subsidiaries and/or its affiliates and/or their licensors.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * @authors Christof Strack, Stefan Witschel
+ */
+
+package dynamic.mapping.connector.core.client;
+
+public class ConnectorException extends Exception {
+ public ConnectorException(String string) {
+ super(string);
+ }
+}
\ No newline at end of file
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/client/ConnectorType.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/client/ConnectorType.java
new file mode 100644
index 00000000..7ba7804a
--- /dev/null
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/client/ConnectorType.java
@@ -0,0 +1,7 @@
+package dynamic.mapping.connector.core.client;
+
+public enum ConnectorType {
+ MQTT,
+ MQTT_SERVICE,
+ KAFKA,
+}
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/registry/ConnectorRegistry.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/registry/ConnectorRegistry.java
index 7ce2c6cd..bff61a9e 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/registry/ConnectorRegistry.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/core/registry/ConnectorRegistry.java
@@ -3,6 +3,7 @@
import dynamic.mapping.connector.core.ConnectorSpecification;
import lombok.extern.slf4j.Slf4j;
import dynamic.mapping.connector.core.client.AConnectorClient;
+import dynamic.mapping.connector.core.client.ConnectorType;
import org.springframework.stereotype.Component;
@@ -16,62 +17,63 @@
public class ConnectorRegistry {
// Structure: Tenant,
- protected HashMap> connectorTenantMap = new HashMap<>();
- // Structure: ConnectorId,
- protected Map connectorSpecificationMap = new HashMap<>();
+ protected Map> connectorTenantMap = new HashMap<>();
+ // Structure: ConnectorType,
+ protected Map connectorSpecificationMap = new HashMap<>();
- public void registerConnector(String connectorType, ConnectorSpecification specification) {
+ public void registerConnector(ConnectorType connectorType, ConnectorSpecification specification) {
connectorSpecificationMap.put(connectorType, specification);
}
- public ConnectorSpecification getConnectorSpecification(String connectorType) {
+ public ConnectorSpecification getConnectorSpecification(ConnectorType connectorType) {
return connectorSpecificationMap.get(connectorType);
}
- public Map getConnectorSpecifications() {
+ public Map getConnectorSpecifications() {
return connectorSpecificationMap;
}
public void registerClient(String tenant, AConnectorClient client) throws ConnectorRegistryException {
if (tenant == null)
- throw new ConnectorRegistryException("tenant is missing!");
+ throw new ConnectorRegistryException("Tenant is missing!");
if (client.getConnectorIdent() == null)
throw new ConnectorRegistryException("Connector ident is missing!");
if (connectorTenantMap.get(tenant) == null) {
- HashMap connectorMap = new HashMap<>();
+ Map connectorMap = new HashMap<>();
connectorMap.put(client.getConnectorIdent(), client);
connectorTenantMap.put(tenant, connectorMap);
} else {
- HashMap connectorMap = connectorTenantMap.get(tenant);
+ Map connectorMap = connectorTenantMap.get(tenant);
if (connectorMap.get(client.getConnectorIdent()) == null) {
- log.info("Tenant {} - Adding new client with id {}...", tenant, client.getConnectorIdent());
+ log.debug("Tenant {} - Adding new client with id {}...", tenant, client.getConnectorIdent());
connectorMap.put(client.getConnectorIdent(), client);
connectorTenantMap.put(tenant, connectorMap);
} else {
- log.info("Tenant {} - Client {} is already registered!", tenant, client.getConnectorIdent());
+ log.debug("Tenant {} - Client {} is already registered!", tenant, client.getConnectorIdent());
}
}
}
- public HashMap getClientsForTenant(String tenant) throws ConnectorRegistryException {
+ public Map getClientsForTenant(String tenant) throws ConnectorRegistryException {
if (tenant == null)
- throw new ConnectorRegistryException("tenant is missing!");
+ throw new ConnectorRegistryException("Tenant is missing!");
if (connectorTenantMap.get(tenant) != null) {
return connectorTenantMap.get(tenant);
} else {
- log.info("Tenant {} - No Client is registered!", tenant);
- return null;
+ Map result = new HashMap<>();
+ connectorTenantMap.put(tenant, result);
+ return result;
}
}
public AConnectorClient getClientForTenant(String tenant, String ident) throws ConnectorRegistryException {
if (tenant == null)
- throw new ConnectorRegistryException("tenant is missing!");
+ throw new ConnectorRegistryException("Tenant is missing!");
if (ident == null)
throw new ConnectorRegistryException("Connector ident is missing!");
if (connectorTenantMap.get(tenant) != null) {
- HashMap connectorMap = connectorTenantMap.get(tenant);
+ Map connectorMap = connectorTenantMap.get(tenant);
if (connectorMap.get(ident) != null)
return connectorMap.get(ident);
else {
@@ -86,9 +88,9 @@ public AConnectorClient getClientForTenant(String tenant, String ident) throws C
public void unregisterAllClientsForTenant(String tenant) throws ConnectorRegistryException {
if (tenant == null)
- throw new ConnectorRegistryException("tenant is missing!");
+ throw new ConnectorRegistryException("Tenant is missing!");
if (connectorTenantMap.get(tenant) != null) {
- HashMap connectorMap = connectorTenantMap.get(tenant);
+ Map connectorMap = connectorTenantMap.get(tenant);
Iterator> iterator = connectorMap.entrySet().iterator();
while (iterator.hasNext()) {
Entry entryNext = iterator.next();
@@ -104,12 +106,12 @@ public void unregisterAllClientsForTenant(String tenant) throws ConnectorRegistr
public void unregisterClient(String tenant, String ident) throws ConnectorRegistryException {
if (tenant == null)
- throw new ConnectorRegistryException("tenant is missing!");
+ throw new ConnectorRegistryException("Tenant is missing!");
if (ident == null)
throw new ConnectorRegistryException("Connector ident is missing!");
if (connectorTenantMap.get(tenant) != null) {
- HashMap connectorMap = connectorTenantMap.get(tenant);
+ Map connectorMap = connectorTenantMap.get(tenant);
if (connectorMap.get(ident) != null) {
AConnectorClient client = connectorMap.get(ident);
// to avoid memory leaks
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/kafka/KafkaClient.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/kafka/KafkaClient.java
new file mode 100644
index 00000000..e648630c
--- /dev/null
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/kafka/KafkaClient.java
@@ -0,0 +1,374 @@
+/*
+ * Copyright (c) 2022 Software AG, Darmstadt, Germany and/or Software AG USA Inc., Reston, VA, USA,
+ * and/or its subsidiaries and/or its affiliates and/or their licensors.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * @authors Christof Strack, Stefan Witschel
+ */
+
+package dynamic.mapping.connector.kafka;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.clients.producer.ProducerRecord;
+// import org.apache.kafka.common.serialization.StringDeserializer;
+// import org.apache.kafka.common.serialization.StringSerializer;
+import org.springframework.core.io.support.PropertiesLoaderUtils;
+
+import dynamic.mapping.connector.core.ConnectorProperty;
+import dynamic.mapping.connector.core.ConnectorPropertyType;
+import dynamic.mapping.connector.core.ConnectorSpecification;
+import dynamic.mapping.connector.core.client.AConnectorClient;
+import dynamic.mapping.connector.core.client.ConnectorException;
+import dynamic.mapping.connector.core.client.ConnectorType;
+import dynamic.mapping.core.ConfigurationRegistry;
+import dynamic.mapping.core.ConnectorStatus;
+import dynamic.mapping.model.Mapping;
+import dynamic.mapping.model.QOS;
+import dynamic.mapping.processor.inbound.AsynchronousDispatcherInbound;
+import dynamic.mapping.processor.model.C8YRequest;
+import dynamic.mapping.processor.model.ProcessingContext;
+import lombok.extern.slf4j.Slf4j;
+import dynamic.mapping.configuration.ConnectorConfiguration;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.StringWriter;
+
+import org.springframework.core.io.Resource;
+import org.springframework.core.io.ClassPathResource;
+
+@Slf4j
+// Use pattern to start/stop polling thread from Stackoverflow
+// https://stackoverflow.com/questions/66103052/how-do-i-stop-a-previous-thread-that-is-listening-to-kafka-topic
+
+public class KafkaClient extends AConnectorClient {
+ public KafkaClient() throws FileNotFoundException, IOException {
+ Map configProps = new HashMap<>();
+ configProps.put("bootstrapServers",
+ new ConnectorProperty(true, 0, ConnectorPropertyType.STRING_PROPERTY, false, false, null, null));
+ configProps.put("username",
+ new ConnectorProperty(false, 1, ConnectorPropertyType.STRING_PROPERTY, false, false, null, null));
+ configProps.put("password",
+ new ConnectorProperty(false, 2, ConnectorPropertyType.SENSITIVE_STRING_PROPERTY, false, false, null,
+ null));
+ configProps.put("groupId",
+ new ConnectorProperty(false, 3, ConnectorPropertyType.STRING_PROPERTY, false, false, null, null));
+
+ Resource resourceProducer = new ClassPathResource(KAFKA_PRODUCER_PROPERTIES);
+ defaultPropertiesProducer = PropertiesLoaderUtils.loadProperties(resourceProducer);
+ StringWriter writerProducer = new StringWriter();
+ defaultPropertiesProducer.store(writerProducer,
+ "properties can only be edited in the property file: kafka-producer.properties");
+ configProps.put("propertiesProducer",
+ new ConnectorProperty(false, 4, ConnectorPropertyType.STRING_LARGE_PROPERTY, true, false,
+ removeDateCommentLine(writerProducer.getBuffer().toString()), null));
+
+ Resource resourceConsumer = new ClassPathResource(KAFKA_CONSUMER_PROPERTIES);
+ defaultPropertiesConsumer = PropertiesLoaderUtils.loadProperties(resourceConsumer);
+ StringWriter writerConsumer = new StringWriter();
+ defaultPropertiesConsumer.store(writerConsumer,
+ "properties can only be edited in the property file: kafka-consumer.properties");
+ configProps.put("propertiesConsumer",
+ new ConnectorProperty(false, 5, ConnectorPropertyType.STRING_LARGE_PROPERTY, true, false,
+ removeDateCommentLine(writerConsumer.getBuffer().toString()), null));
+
+ String description = "Generic connector to receive and send messages to a external Kafka broker. Inbound mappings allow to extract values from the payload and the key and map these to the Cumulocity payload. The relevant setting in a mapping is 'supportsMessageContext'.\n In outbound mappings the any string that is mapped to '_CONTEXT_DATA_.key' is used as the outbound Kafka record.";
+ connectorType = ConnectorType.KAFKA;
+ supportsMessageContext = true;
+ specification = new ConnectorSpecification(description, connectorType, configProps, true);
+ }
+
+ private static String removeDateCommentLine(String pt) {
+ String result = pt;
+ String regex = "(?m)^[ ]*#.*$(\r?\n)?";
+ Pattern pattern = Pattern.compile(regex);
+ Matcher matcher = pattern.matcher(pt);
+ // Find the second occurrence of the pattern
+ int count = 0;
+ while (matcher.find()) {
+ count++;
+ if (count == 2) {
+ break;
+ }
+ }
+ // Remove the second line starting with "#"
+ if (count == 2) {
+ result = pt.substring(0, matcher.start()) + pt.substring(matcher.end());
+ }
+ return result;
+ }
+
+ public KafkaClient(ConfigurationRegistry configurationRegistry,
+ ConnectorConfiguration connectorConfiguration,
+ AsynchronousDispatcherInbound dispatcher, String additionalSubscriptionIdTest, String tenant)
+ throws FileNotFoundException, IOException {
+ this();
+ this.configurationRegistry = configurationRegistry;
+ this.mappingComponent = configurationRegistry.getMappingComponent();
+ this.serviceConfigurationComponent = configurationRegistry.getServiceConfigurationComponent();
+ this.connectorConfigurationComponent = configurationRegistry.getConnectorConfigurationComponent();
+ this.connectorConfiguration = connectorConfiguration;
+ // ensure the client knows its identity even if configuration is set to null
+ this.connectorIdent = connectorConfiguration.ident;
+ this.connectorName = connectorConfiguration.name;
+ this.c8yAgent = configurationRegistry.getC8yAgent();
+ this.cachedThreadPool = configurationRegistry.getCachedThreadPool();
+ this.objectMapper = configurationRegistry.getObjectMapper();
+ this.additionalSubscriptionIdTest = additionalSubscriptionIdTest;
+ this.mappingServiceRepresentation = configurationRegistry.getMappingServiceRepresentations().get(tenant);
+ this.serviceConfiguration = configurationRegistry.getServiceConfigurations().get(tenant);
+ this.dispatcher = dispatcher;
+ this.tenant = tenant;
+ this.connectionState.setFalse();
+
+ // defaultPropertiesProducer = new Properties();
+ // String jaasTemplate =
+ // "org.apache.kafka.common.security.scram.ScramLoginModule required
+ // username=\"%s\" password=\"%s\";";
+ // String jaasCfg = String.format(jaasTemplate, username, password);
+ // String serializer = StringSerializer.class.getName();
+ // defaultPropertiesConsumer.put("bootstrap.servers",
+ // "glider.srvs.cloudkafka.com:9094");
+ // defaultPropertiesProducer.put("key.serializer", serializer);
+ // defaultPropertiesProducer.put("value.serializer", serializer);
+ // defaultPropertiesProducer.put("security.protocol", "SASL_SSL");
+ // defaultPropertiesProducer.put("sasl.mechanism", "SCRAM-SHA-256");
+ // defaultPropertiesProducer.put("sasl.jaas.config", jaasCfg);
+ // defaultPropertiesProducer.put("linger.ms", 1);
+ // defaultPropertiesProducer.put("enable.idempotence", false);
+
+ // String deserializer = StringDeserializer.class.getName();
+ // defaultPropertiesConsumer = new Properties();
+ // defaultPropertiesConsumer.put("bootstrap.servers","glider.srvs.cloudkafka.com:9094");
+ // defaultPropertiesConsumer.put("group.id", username + "-consumer");
+ // defaultPropertiesConsumer.put("enable.auto.commit", "true");
+ // defaultPropertiesConsumer.put("auto.commit.interval.ms", "1000");
+ // defaultPropertiesConsumer.put("auto.offset.reset", "earliest");
+ // defaultPropertiesConsumer.put("session.timeout.ms", "30000");
+ // defaultPropertiesConsumer.put("key.deserializer", deserializer);
+ // defaultPropertiesConsumer.put("value.deserializer", deserializer);
+ // defaultPropertiesConsumer.put("key.serializer", serializer);
+ // defaultPropertiesConsumer.put("value.serializer", serializer);
+ // defaultPropertiesConsumer.put("security.protocol", "SASL_SSL");
+ // defaultPropertiesConsumer.put("sasl.mechanism", "SCRAM-SHA-256");
+ // defaultPropertiesConsumer.put("sasl.jaas.config", jaasCfg);
+ // defaultPropertiesConsumer.put("linger.ms", 1);
+ updateConnectorStatusAndSend(ConnectorStatus.UNKNOWN, true, true);
+ }
+
+ private String bootstrapServers;
+ private String password;
+ private String username;
+ private String groupId;
+
+ private HashMap consumerList = new HashMap();
+
+ private Properties defaultPropertiesConsumer;
+ private Properties defaultPropertiesProducer;
+
+ private KafkaProducer kafkaProducer;
+
+ private String KAFKA_CONSUMER_PROPERTIES = "/kafka-consumer.properties";
+ private String KAFKA_PRODUCER_PROPERTIES = "/kafka-producer.properties";
+
+ @Override
+ public boolean initialize() {
+ loadConfiguration();
+ username = (String) connectorConfiguration.getProperties().get("username");
+ password = (String) connectorConfiguration.getProperties().get("password");
+ bootstrapServers = (String) connectorConfiguration.getProperties().get("bootstrapServers");
+ return true;
+ }
+
+ @Override
+ public Boolean supportsWildcardsInTopic() {
+ return false;
+ }
+
+ @Override
+ public void connect() {
+ updateConnectorStatusAndSend(ConnectorStatus.CONNECTING, true, true);
+ log.info("Tenant {} - Trying to connect to {} - phase I: (isConnected:shouldConnect) ({}:{})",
+ tenant, getConnectorName(), isConnected(),
+ shouldConnect());
+ // stay in the loop until successful
+ boolean successful = false;
+ while (!successful) {
+ loadConfiguration();
+ username = (String) connectorConfiguration.getProperties().get("username");
+ password = (String) connectorConfiguration.getProperties().get("password");
+ groupId = (String) connectorConfiguration.getProperties().get("groupId");
+ bootstrapServers = (String) connectorConfiguration.getProperties().get("bootstrapServers");
+ String jaasTemplate = "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"%s\" password=\"%s\";";
+ String jaasCfg = String.format(jaasTemplate, username, password);
+ defaultPropertiesProducer.put("sasl.jaas.config", jaasCfg);
+ defaultPropertiesProducer.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
+ defaultPropertiesProducer.put("group.id", groupId);
+ log.info("Tenant {} - Trying to connect {} - phase II: (shouldConnect):{} {}", tenant,
+ getConnectorName(),
+ shouldConnect(), bootstrapServers);
+ log.info("Tenant {} - Successfully connected to broker {}", tenant,
+ bootstrapServers);
+ try {
+ // test if the mqtt connection is configured and enabled
+ if (shouldConnect()) {
+ mappingComponent.rebuildMappingOutboundCache(tenant);
+ // in order to keep MappingInboundCache and ActiveSubscriptionMappingInbound in
+ // sync, the ActiveSubscriptionMappingInbound is build on the
+ // previously used updatedMappings
+ kafkaProducer = new KafkaProducer<>(defaultPropertiesProducer);
+ connectionState.setTrue();
+ updateConnectorStatusAndSend(ConnectorStatus.CONNECTED, true, true);
+ List updatedMappings = mappingComponent.rebuildMappingInboundCache(tenant);
+ updateActiveSubscriptions(updatedMappings, true);
+ }
+ successful = true;
+ } catch (Exception e) {
+ log.error("Tenant {} - Error on reconnect, retrying ... {}: ", tenant, e.getMessage(), e);
+ updateConnectorStatusToFailed(e);
+ sendConnectorLifecycle();
+ if (serviceConfiguration.logConnectorErrorInBackend) {
+ log.error("Tenant {} - Stacktrace: ", tenant, e);
+ }
+ successful = false;
+ }
+ }
+ }
+
+ @Override
+ public boolean isConnected() {
+ return connectionState.getValue();
+ }
+
+ @Override
+ public void disconnect() {
+ if (isConnected()) {
+ updateConnectorStatusAndSend(ConnectorStatus.DISCONNECTING, true, true);
+ log.info("Tenant {} - Disconnecting connector {} from broker: {}", tenant, getConnectorName(),
+ bootstrapServers);
+ activeSubscriptions.entrySet().forEach(entry -> {
+ // only unsubscribe if still active subscriptions exist
+ String topic = entry.getKey();
+ MutableInt activeSubs = entry.getValue();
+ if (activeSubs.intValue() > 0) {
+ try {
+ unsubscribe(topic);
+ } catch (Exception error) {
+ log.error("Tenant {} - Error unsubscribing topic {} from broker: {}, error {}", tenant, topic,
+ getConnectorName(),
+ error);
+ }
+ }
+ });
+
+ connectionState.setFalse();
+ updateConnectorStatusAndSend(ConnectorStatus.DISCONNECTED, true, true);
+ List updatedMappings = mappingComponent.rebuildMappingInboundCache(tenant);
+ updateActiveSubscriptions(updatedMappings, true);
+ kafkaProducer.close();
+ log.info("Tenant {} - Disconnected from from broker: {}", tenant, getConnectorName(),
+ bootstrapServers);
+ }
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public String getConnectorIdent() {
+ return connectorIdent;
+ }
+
+ @Override
+ public String getConnectorName() {
+ return connectorName;
+ }
+
+ @Override
+ public void subscribe(String topic, QOS qos) throws ConnectorException {
+ TopicConsumer kafkaConsumer = new TopicConsumer(
+ new TopicConfig(tenant, bootstrapServers, topic, username, password, groupId,
+ defaultPropertiesConsumer));
+ consumerList.put(topic, kafkaConsumer);
+ TopicConsumerCallback topicConsumerCallback = new TopicConsumerCallback(dispatcher, tenant, getConnectorIdent(),
+ topic, true);
+ kafkaConsumer.start(topicConsumerCallback);
+ }
+
+ @Override
+ public void monitorSubscriptions() {
+
+ // for (Iterator> me =
+ // getMappingsDeployed().entrySet().iterator(); me.hasNext();) {
+ Iterator it = getMappingsDeployed().keySet().iterator();
+ while (it.hasNext()) {
+ String mapIdent = it.next();
+ Mapping map = getMappingsDeployed().get(mapIdent);
+ // test if topicConsumer was started successfully
+ if (consumerList.containsKey(map.subscriptionTopic)) {
+ TopicConsumer kafkaConsumer = consumerList.get(map.subscriptionTopic);
+ if (kafkaConsumer.shouldStop()) {
+ try {
+ // kafkaConsumer.close();
+ unsubscribe(mapIdent);
+ getMappingsDeployed().remove(map.ident);
+ log.warn(
+ "Tenant {} - Failed to subscribe to subscriptionTopic {} for mapping {} in connector {}!",
+ tenant, map.subscriptionTopic, map, getConnectorName());
+ } catch (Exception e) {
+ // ignore interrupt
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public void unsubscribe(String topic) throws Exception {
+ TopicConsumer kafkaConsumer = consumerList.remove(topic);
+ if (kafkaConsumer != null)
+ kafkaConsumer.close();
+ }
+
+ @Override
+ public boolean isConfigValid(ConnectorConfiguration configuration) {
+ return true;
+ }
+
+ @Override
+ public void publishMEAO(ProcessingContext> context) {
+ C8YRequest currentRequest = context.getCurrentRequest();
+ String payload = currentRequest.getRequest();
+ String key = currentRequest.getSource();
+ if (context.isSupportsMessageContext() && context.getKey() != null) {
+ key = new String(context.getKey());
+ }
+ kafkaProducer.send(new ProducerRecord(context.getMapping().publishTopic, key, payload));
+
+ log.info("Tenant {} - Published outbound message: {} for mapping: {} on topic: {}, {}", tenant, payload,
+ context.getMapping().name, context.getResolvedPublishTopic(), connectorName);
+ }
+}
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/kafka/Topic.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/kafka/Topic.java
new file mode 100644
index 00000000..f5175f7c
--- /dev/null
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/kafka/Topic.java
@@ -0,0 +1,106 @@
+package dynamic.mapping.connector.kafka;
+
+import org.apache.commons.lang3.SerializationUtils;
+import org.apache.kafka.clients.consumer.Consumer;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.Properties;
+
+@Slf4j
+public class Topic implements AutoCloseable {
+ private final TopicConfig topicConfig;
+
+ private final Consumer consumer;
+
+ public Topic(final TopicConfig topicConfig) {
+ this.topicConfig = topicConfig;
+
+ final Properties props = SerializationUtils.clone(topicConfig.getDefaultPropertiesConsumer());
+
+ props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, topicConfig.getBootstrapServers());
+ props.put("group.id", topicConfig.getGroupId());
+ // this is a common topic consumer, so we just pull byte arrays and pass them
+ // to a listener, we don't do any decoding in here
+ props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
+ props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
+ String jaasTemplate = "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"%s\" password=\"%s\";";
+ String jaasCfg = String.format(jaasTemplate, topicConfig.getUsername(), topicConfig.getPassword());
+ props.put("sasl.jaas.config", jaasCfg);
+
+ consumer = new KafkaConsumer<>(props);
+ try {
+ consumer.partitionsFor(topicConfig.getTopic()); // just to check connectivity immediately
+ } catch (final Exception e) {
+ try {
+ consumer.close();
+ } catch (final Exception ignore) {
+ }
+ throw e;
+ }
+ }
+
+ /**
+ * We can exit from this method only by an exception. Most important cases:
+ * 1. org.apache.kafka.common.errors.WakeupException - if we call close() method
+ * 2. org.apache.kafka.common.errors.InterruptException - if the current thread
+ * has
+ * been interrupted
+ *
+ * @see KafkaConsumer#poll(Duration)
+ *
+ * @param listener
+ */
+ public void consumeUntilError(final TopicEventListener listener) {
+ consumer.subscribe(Arrays.asList(topicConfig.getTopic()));
+
+ while (true) {
+ final ConsumerRecords records = consumer.poll(Duration.ofSeconds(10));
+ for (ConsumerRecord record : records) {
+ try {
+ Object key = record.key();
+ Object event = record.value();
+ byte[] keyByte;
+ byte[] eventByte;
+ if (key instanceof String) {
+ keyByte = ((String) key).getBytes();
+ } else {
+ keyByte = record.key();
+ }
+ if (event instanceof String) {
+ eventByte = ((String) event).getBytes();
+ } else {
+ eventByte = record.key();
+ }
+ listener.onEvent(keyByte, eventByte);
+ } catch (final InterruptedException e) { // can be thrown by a blocking operation inside onEvent()
+ throw new org.apache.kafka.common.errors.InterruptException(e);
+ } catch (final Exception error) {
+ // just log ("Unexpected error while listener.onEvent() notification", e)
+ // don't corrupt the consuming loop because of
+ // an error in a listener
+ log.error("Tenant {} - Failed to process message on topic {} with error: ", topicConfig.getTenant(),
+ topicConfig.getTopic(),
+ error);
+ }
+ }
+ }
+ }
+
+ @Override
+ public void close() {
+ try {
+ consumer.wakeup();
+ } finally {
+ consumer.close();
+ }
+ }
+}
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/kafka/TopicConfig.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/kafka/TopicConfig.java
new file mode 100644
index 00000000..b0510dcc
--- /dev/null
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/kafka/TopicConfig.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2022 Software AG, Darmstadt, Germany and/or Software AG USA Inc., Reston, VA, USA,
+ * and/or its subsidiaries and/or its affiliates and/or their licensors.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * @authors Christof Strack, Stefan Witschel
+ */
+
+package dynamic.mapping.connector.kafka;
+
+import java.util.Properties;
+
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+public class TopicConfig {
+ private String tenant;
+ private String bootstrapServers;
+ private String topic;
+ private String username;
+ private String password;
+ private String groupId;
+ private Properties defaultPropertiesConsumer;
+
+ public TopicConfig(String tenant, String bootstrapServers, String topic, String username, String password,
+ String groupId,
+ Properties defaultPropertiesConsumer) {
+ this.tenant = tenant;
+ this.bootstrapServers = bootstrapServers;
+ this.topic = topic;
+ this.username = username;
+ this.password = password;
+ this.groupId = groupId;
+ this.defaultPropertiesConsumer = defaultPropertiesConsumer;
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() +
+ " bootstrapServers='" + bootstrapServers + '\'' +
+ ", topic='" + topic + '\'';
+ }
+}
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/kafka/TopicConsumer.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/kafka/TopicConsumer.java
new file mode 100644
index 00000000..9513a08d
--- /dev/null
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/kafka/TopicConsumer.java
@@ -0,0 +1,172 @@
+/*
+ * Copyright (c) 2022 Software AG, Darmstadt, Germany and/or Software AG USA Inc., Reston, VA, USA,
+ * and/or its subsidiaries and/or its affiliates and/or their licensors.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * @authors Christof Strack, Stefan Witschel
+ */
+
+package dynamic.mapping.connector.kafka;
+
+import org.apache.kafka.common.errors.TopicAuthorizationException;
+
+public class TopicConsumer {
+ private final TopicConfig topicConfig;
+
+ private ConsumingThread consumingThread; // guarded by this
+ private boolean closed; // guarded by this
+
+ public TopicConsumer(final TopicConfig topicConfig) {
+ this.topicConfig = topicConfig;
+ }
+
+ public synchronized void start(final TopicConsumerListener listener) {
+ if (closed) {
+ throw new IllegalStateException("Closed");
+ }
+
+ if (consumingThread != null) {
+ throw new IllegalStateException("Already started");
+ }
+
+ final ConsumingThread ct = new ConsumingThread(listener);
+ ct.start();
+ consumingThread = ct;
+ }
+
+ public void stop() throws InterruptedException {
+ final ConsumingThread ct;
+ synchronized (this) {
+ ct = consumingThread;
+
+ if (ct == null) {
+ return;
+ }
+
+ consumingThread = null;
+ }
+
+ ct.close();
+
+ if (Thread.currentThread() != ct) {
+ ct.join();
+ }
+ }
+
+ public boolean shouldStop() {
+ if (consumingThread == null)
+ return true;
+ return consumingThread.shouldStop;
+ }
+
+ public void close() throws InterruptedException {
+ synchronized (this) {
+ if (closed) {
+ return;
+ }
+ closed = true;
+ }
+
+ stop();
+ }
+
+ private class ConsumingThread extends Thread {
+ private final TopicConsumerListener listener;
+ private volatile boolean closed;
+ boolean shouldStop = false;
+
+ ConsumingThread(final TopicConsumerListener listener) {
+ super("Consumer#" + topicConfig.getBootstrapServers() + "/" + topicConfig.getTopic());
+ this.listener = listener;
+ }
+
+ @Override
+ public void run() {
+ Exception error = null;
+ boolean continueToListen = true;
+
+ while (continueToListen) {
+ Topic tc = null;
+ try {
+ tc = new Topic(topicConfig);
+
+ try {
+ listener.onStarted();
+ } catch (final Exception e) {
+ // log ("Unexpected error while onStarted() notification", e);
+ }
+
+ // we consume the events from the topic until
+ // this thread is interrupted by close()
+ tc.consumeUntilError(listener);
+ } catch (final Exception e) {
+ if (closed) {
+ break;
+ }
+ error = e;
+ if (error instanceof TopicAuthorizationException) {
+ continueToListen = false;
+ shouldStop = true;
+ }
+ } finally {
+ if (tc != null) {
+ try {
+ tc.close();
+ } catch (final Exception ignore) {
+ }
+ }
+ }
+
+ try {
+ listener.onStoppedByErrorAndReconnecting(error);
+ } catch (final Exception e) {
+ // log ("Unexpected error while onStoppedByErrorAndReconnecting() notification",
+ // e)
+ }
+
+ try {
+ Thread.sleep(5000); // TODO: make the timeout configurable and use backoff with jitter
+ } catch (final InterruptedException e) {
+ break; // interrupted by close()
+ // we don't restore the flag interrupted, since we still need
+ // to do some additional work like
+ // to notify listener.onStopped()
+ }
+ }
+
+ try {
+ listener.onStopped();
+ } catch (final Exception e) {
+ // log ("Unexpected error while onStoppedByErrorAndReconnecting() notification",
+ // e);
+ }
+ }
+
+ void close() {
+ if (closed) { // no atomicity/membars required
+ return; // since can be called only by one single thread
+ }
+ closed = true;
+
+ // We stop the consuming with org.apache.kafka.common.errors.InterruptException
+ // In here it isn't convenient to call Topic.close() directly to initiate
+ // org.apache.kafka.common.errors.WakeupException, since we recreate
+ // the instance of Topic and it takes additional efforts to share the
+ // changeable reference to a Topic to close it from other thread.
+ interrupt();
+ }
+ }
+}
\ No newline at end of file
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/kafka/TopicConsumerCallback.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/kafka/TopicConsumerCallback.java
new file mode 100644
index 00000000..efd00147
--- /dev/null
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/kafka/TopicConsumerCallback.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2022 Software AG, Darmstadt, Germany and/or Software AG USA Inc., Reston, VA, USA,
+ * and/or its subsidiaries and/or its affiliates and/or their licensors.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * @authors Christof Strack, Stefan Witschel
+ */
+
+package dynamic.mapping.connector.kafka;
+
+import dynamic.mapping.connector.core.callback.ConnectorMessage;
+import dynamic.mapping.connector.core.callback.GenericMessageCallback;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class TopicConsumerCallback implements TopicConsumerListener {
+ GenericMessageCallback genericMessageCallback;
+ String tenant;
+ String topic;
+ String connectorIdent;
+ boolean supportsMessageContext;
+
+ TopicConsumerCallback(GenericMessageCallback callback, String tenant, String connectorIdent, String topic,
+ boolean supportsMessageContext) {
+ this.genericMessageCallback = callback;
+ this.tenant = tenant;
+ this.topic = topic;
+ this.connectorIdent = connectorIdent;
+ this.supportsMessageContext = supportsMessageContext;
+ }
+
+ @Override
+ public void onEvent(byte[] key, byte[] event) throws Exception {
+ ConnectorMessage connectorMessage = new ConnectorMessage();
+ connectorMessage.setPayload(event);
+ connectorMessage.setKey(key);
+ connectorMessage.setTenant(tenant);
+ connectorMessage.setSendPayload(true);
+ connectorMessage.setTopic(topic);
+ connectorMessage.setConnectorIdent(connectorIdent);
+ connectorMessage.setSupportsMessageContext(supportsMessageContext);
+ genericMessageCallback.onMessage(connectorMessage);
+ }
+
+ @Override
+ public void onStarted() {
+ log.info("Tenant {} - Called method Called method 'onStarted'", tenant);
+ }
+
+ @Override
+ public void onStoppedByErrorAndReconnecting(Exception error) {
+ log.error("Tenant {} - Called method 'onStoppedByErrorAndReconnecting'",tenant, error);
+
+ }
+
+ @Override
+ public void onStopped() {
+ log.info("Tenant {} - Called method 'onStopped'", tenant);
+ }
+
+}
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/kafka/TopicConsumerListener.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/kafka/TopicConsumerListener.java
new file mode 100644
index 00000000..447c5e07
--- /dev/null
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/kafka/TopicConsumerListener.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2022 Software AG, Darmstadt, Germany and/or Software AG USA Inc., Reston, VA, USA,
+ * and/or its subsidiaries and/or its affiliates and/or their licensors.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * @authors Christof Strack, Stefan Witschel
+ */
+package dynamic.mapping.connector.kafka;
+
+public interface TopicConsumerListener extends TopicEventListener {
+ void onStarted();
+
+ void onStoppedByErrorAndReconnecting(Exception error);
+
+ void onStopped();
+}
\ No newline at end of file
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/kafka/TopicEventListener.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/kafka/TopicEventListener.java
new file mode 100644
index 00000000..a6f32361
--- /dev/null
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/kafka/TopicEventListener.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2022 Software AG, Darmstadt, Germany and/or Software AG USA Inc., Reston, VA, USA,
+ * and/or its subsidiaries and/or its affiliates and/or their licensors.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * @authors Christof Strack, Stefan Witschel
+ */
+
+package dynamic.mapping.connector.kafka;
+
+public interface TopicEventListener {
+ void onEvent(byte[] key, byte[] event) throws Exception;
+}
\ No newline at end of file
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/mqtt/MQTTCallback.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/mqtt/MQTTCallback.java
index 29dd7d82..754f1fe8 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/mqtt/MQTTCallback.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/mqtt/MQTTCallback.java
@@ -1,40 +1,45 @@
package dynamic.mapping.connector.mqtt;
-import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken;
-import org.eclipse.paho.client.mqttv3.MqttCallback;
-import org.eclipse.paho.client.mqttv3.MqttMessage;
+import java.nio.ByteBuffer;
+import java.util.function.Consumer;
+
+import com.hivemq.client.mqtt.datatypes.MqttTopic;
+import com.hivemq.client.mqtt.mqtt3.message.publish.Mqtt3Publish;
import dynamic.mapping.connector.core.callback.ConnectorMessage;
import dynamic.mapping.connector.core.callback.GenericMessageCallback;
-public class MQTTCallback implements MqttCallback {
+
+public class MQTTCallback implements Consumer {
GenericMessageCallback genericMessageCallback;
+ static String TOPIC_LEVEL_SEPARATOR = String.valueOf(MqttTopic.TOPIC_LEVEL_SEPARATOR);
String tenant;
String connectorIdent;
+ boolean supportsMessageContext;
- MQTTCallback(GenericMessageCallback callback, String tenant, String connectorIdent) {
+ MQTTCallback(GenericMessageCallback callback, String tenant, String connectorIdent,
+ boolean supportsMessageContext) {
this.genericMessageCallback = callback;
this.tenant = tenant;
this.connectorIdent = connectorIdent;
+ this.supportsMessageContext = supportsMessageContext;
}
@Override
- public void connectionLost(Throwable throwable) {
- genericMessageCallback.onClose(null, throwable);
- }
-
- @Override
- public void messageArrived(String topic, MqttMessage mqttMessage) throws Exception {
+ public void accept(Mqtt3Publish mqttMessage) {
ConnectorMessage connectorMessage = new ConnectorMessage();
- connectorMessage.setPayload(mqttMessage.getPayload());
+ if (mqttMessage.getPayload().isPresent()) {
+ ByteBuffer byteBuffer = mqttMessage.getPayload().get();
+ byte[] byteArray = new byte[byteBuffer.remaining()];
+ byteBuffer.get(byteArray);
+ connectorMessage.setPayload(byteArray);
+ }
connectorMessage.setTenant(tenant);
- connectorMessage.setSendPayload(true);
+ connectorMessage.setSendPayload(true);
+ String topic = String.join(TOPIC_LEVEL_SEPARATOR, mqttMessage.getTopic().getLevels());
connectorMessage.setTopic(topic);
connectorMessage.setConnectorIdent(connectorIdent);
+ connectorMessage.setSupportsMessageContext(supportsMessageContext);
genericMessageCallback.onMessage(connectorMessage);
}
- @Override
- public void deliveryComplete(IMqttDeliveryToken iMqttDeliveryToken) {
-
- }
}
\ No newline at end of file
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/mqtt/MQTTClient.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/mqtt/MQTTClient.java
index 735bccd9..69d5cbb5 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/mqtt/MQTTClient.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/mqtt/MQTTClient.java
@@ -31,30 +31,40 @@
import java.security.cert.CertificateException;
import java.security.cert.CertificateFactory;
import java.security.cert.X509Certificate;
+import java.util.AbstractMap;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLSocketFactory;
-import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;
-
+import com.hivemq.client.mqtt.mqtt3.message.unsubscribe.Mqtt3Unsubscribe;
import dynamic.mapping.connector.core.ConnectorPropertyType;
import dynamic.mapping.connector.core.ConnectorSpecification;
import dynamic.mapping.connector.core.client.AConnectorClient;
+import dynamic.mapping.connector.core.client.ConnectorException;
+import dynamic.mapping.connector.core.client.ConnectorType;
import dynamic.mapping.model.Mapping;
+import dynamic.mapping.model.QOS;
import dynamic.mapping.processor.inbound.AsynchronousDispatcherInbound;
import dynamic.mapping.processor.model.C8YRequest;
import dynamic.mapping.processor.model.ProcessingContext;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.mutable.MutableInt;
-import org.eclipse.paho.client.mqttv3.MqttClient;
-import org.eclipse.paho.client.mqttv3.MqttConnectOptions;
-import org.eclipse.paho.client.mqttv3.MqttException;
-import org.eclipse.paho.client.mqttv3.MqttMessage;
-import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence;
+import com.hivemq.client.mqtt.MqttClientSslConfig;
+import com.hivemq.client.mqtt.MqttClientSslConfigBuilder;
+import com.hivemq.client.mqtt.MqttGlobalPublishFilter;
+import com.hivemq.client.mqtt.datatypes.MqttQos;
+import com.hivemq.client.mqtt.mqtt3.Mqtt3AsyncClient;
+import com.hivemq.client.mqtt.mqtt3.Mqtt3BlockingClient;
+import com.hivemq.client.mqtt.mqtt3.Mqtt3Client;
+import com.hivemq.client.mqtt.mqtt3.Mqtt3ClientBuilder;
+import com.hivemq.client.mqtt.mqtt3.message.auth.Mqtt3SimpleAuth;
+import com.hivemq.client.mqtt.mqtt3.message.auth.Mqtt3SimpleAuthBuilder.Complete;
+import com.hivemq.client.mqtt.mqtt3.message.connect.connack.Mqtt3ConnAck;
+import com.hivemq.client.mqtt.mqtt3.message.connect.connack.Mqtt3ConnAckReturnCode;
+import com.hivemq.client.mqtt.mqtt3.message.publish.Mqtt3Publish;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import dynamic.mapping.configuration.ConnectorConfiguration;
@@ -63,12 +73,46 @@
import dynamic.mapping.core.ConnectorStatus;
@Slf4j
-// This is instantiated manually not using Spring Boot anymore.
public class MQTTClient extends AConnectorClient {
+ public MQTTClient() {
+ Map configProps = new HashMap<>();
+ configProps.put("protocol",
+ new ConnectorProperty(true, 0, ConnectorPropertyType.OPTION_PROPERTY, false, false, "mqtt://",
+ Map.ofEntries(
+ new AbstractMap.SimpleEntry("mqtt://", "mqtt://"),
+ new AbstractMap.SimpleEntry("mqtts://", "mqtts://"),
+ new AbstractMap.SimpleEntry("ws://", "ws://"),
+ new AbstractMap.SimpleEntry("wss://", "wss://"))));
+ configProps.put("mqttHost",
+ new ConnectorProperty(true, 1, ConnectorPropertyType.STRING_PROPERTY, false, false, null, null));
+ configProps.put("mqttPort",
+ new ConnectorProperty(true, 2, ConnectorPropertyType.NUMERIC_PROPERTY, false, false, null, null));
+ configProps.put("user",
+ new ConnectorProperty(false, 3, ConnectorPropertyType.STRING_PROPERTY, false, false, null, null));
+ configProps.put("password",
+ new ConnectorProperty(false, 4, ConnectorPropertyType.SENSITIVE_STRING_PROPERTY, false, false, null,
+ null));
+ configProps.put("clientId",
+ new ConnectorProperty(true, 5, ConnectorPropertyType.STRING_PROPERTY, false, false, null, null));
+ configProps.put("useSelfSignedCertificate",
+ new ConnectorProperty(false, 6, ConnectorPropertyType.BOOLEAN_PROPERTY, false, false, false, null));
+ configProps.put("fingerprintSelfSignedCertificate",
+ new ConnectorProperty(false, 7, ConnectorPropertyType.STRING_PROPERTY, false, false, null, null));
+ configProps.put("nameCertificate",
+ new ConnectorProperty(false, 8, ConnectorPropertyType.STRING_PROPERTY, false, false, null, null));
+ configProps.put("supportsWildcardInTopic",
+ new ConnectorProperty(false, 9, ConnectorPropertyType.BOOLEAN_PROPERTY, false, false, true, null));
+ configProps.put("serverPath",
+ new ConnectorProperty(false, 10, ConnectorPropertyType.STRING_PROPERTY, false, false, null, null));
+ String description = "Generic connector for connecting to external MQTT broker over tcp or websocket.";
+ connectorType = ConnectorType.MQTT;
+ specification = new ConnectorSpecification(description, connectorType, configProps, false);
+ }
public MQTTClient(ConfigurationRegistry configurationRegistry,
ConnectorConfiguration connectorConfiguration,
AsynchronousDispatcherInbound dispatcher, String additionalSubscriptionIdTest, String tenant) {
+ this();
this.configurationRegistry = configurationRegistry;
this.mappingComponent = configurationRegistry.getMappingComponent();
this.serviceConfigurationComponent = configurationRegistry.getServiceConfigurationComponent();
@@ -77,6 +121,7 @@ public MQTTClient(ConfigurationRegistry configurationRegistry,
// ensure the client knows its identity even if configuration is set to null
this.connectorIdent = connectorConfiguration.ident;
this.connectorName = connectorConfiguration.name;
+ // this.connectorType = connectorConfiguration.connectorType;
this.c8yAgent = configurationRegistry.getC8yAgent();
this.cachedThreadPool = configurationRegistry.getCachedThreadPool();
this.objectMapper = configurationRegistry.getObjectMapper();
@@ -85,51 +130,30 @@ public MQTTClient(ConfigurationRegistry configurationRegistry,
this.serviceConfiguration = configurationRegistry.getServiceConfigurations().get(tenant);
this.dispatcher = dispatcher;
this.tenant = tenant;
+ this.supportedQOS = Arrays.asList(QOS.AT_LEAST_ONCE, QOS.AT_MOST_ONCE, QOS.EXACTLY_ONCE);
}
- private static final int WAIT_PERIOD_MS = 10000;
+ protected AConnectorClient.Certificate cert;
- @Getter
- private static final String connectorType = "MQTT";
+ protected MqttClientSslConfig sslConfig;
- @Getter
- public static ConnectorSpecification spec;
- static {
- Map configProps = new HashMap<>();
- configProps.put("mqttHost", new ConnectorProperty(true, 0, ConnectorPropertyType.STRING_PROPERTY));
- configProps.put("mqttPort", new ConnectorProperty(true, 1, ConnectorPropertyType.NUMERIC_PROPERTY));
- configProps.put("user", new ConnectorProperty(false, 2, ConnectorPropertyType.STRING_PROPERTY));
- configProps.put("password",
- new ConnectorProperty(false, 3, ConnectorPropertyType.SENSITIVE_STRING_PROPERTY));
- configProps.put("clientId", new ConnectorProperty(true, 4, ConnectorPropertyType.STRING_PROPERTY));
- configProps.put("useTLS", new ConnectorProperty(false, 5, ConnectorPropertyType.BOOLEAN_PROPERTY));
- configProps.put("useSelfSignedCertificate",
- new ConnectorProperty(false, 6, ConnectorPropertyType.BOOLEAN_PROPERTY));
- configProps.put("fingerprintSelfSignedCertificate",
- new ConnectorProperty(false, 7, ConnectorPropertyType.STRING_PROPERTY));
- configProps.put("nameCertificate", new ConnectorProperty(false, 8, ConnectorPropertyType.STRING_PROPERTY));
- spec = new ConnectorSpecification(connectorType, true, configProps);
- }
-
- private String additionalSubscriptionIdTest;
-
- private AConnectorClient.Certificate cert;
+ protected MQTTCallback mqttCallback = null;
- private SSLSocketFactory sslSocketFactory;
+ protected Mqtt3BlockingClient mqttClient;
- private MQTTCallback mqttCallback = null;
-
- private MqttClient mqttClient;
+ @Getter
+ protected List supportedQOS;
public boolean initialize() {
loadConfiguration();
Boolean useSelfSignedCertificate = (Boolean) connectorConfiguration.getProperties()
.getOrDefault("useSelfSignedCertificate", false);
- log.info("Tenant {} - Testing connector for useSelfSignedCertificate: {} ", tenant, useSelfSignedCertificate);
+ log.debug("Tenant {} - Testing connector for useSelfSignedCertificate: {} ", tenant, useSelfSignedCertificate);
if (useSelfSignedCertificate) {
try {
String nameCertificate = (String) connectorConfiguration.getProperties().get("nameCertificate");
- String fingerprint = (String) connectorConfiguration.getProperties().get("fingerprintSelfSignedCertificate");
+ String fingerprint = (String) connectorConfiguration.getProperties()
+ .get("fingerprintSelfSignedCertificate");
if (nameCertificate == null || fingerprint == null) {
throw new Exception(
"Required properties nameCertificate, fingerprint are not set. Please update the connector configuration!");
@@ -150,11 +174,16 @@ public boolean initialize() {
TrustManagerFactory tmf = TrustManagerFactory
.getInstance(TrustManagerFactory.getDefaultAlgorithm());
tmf.init(trustStore);
- TrustManager[] trustManagers = tmf.getTrustManagers();
-
- SSLContext sslContext = SSLContext.getInstance("TLSv1.2");
- sslContext.init(null, trustManagers, null);
- sslSocketFactory = sslContext.getSocketFactory();
+ // TrustManager[] trustManagers = tmf.getTrustManagers();
+ // SSLContext sslContext = SSLContext.getInstance("TLSv1.2");
+ // sslContext.init(null, trustManagers, null);
+ // sslSocketFactory = sslContext.getSocketFactory();
+ MqttClientSslConfigBuilder sslConfigBuilder = MqttClientSslConfig.builder();
+ // use sample from
+ // https://github.com/micronaut-projects/micronaut-mqtt/blob/ac2720937871b8907ad429f7ea5b8b4664a0776e/mqtt-hivemq/src/main/java/io/micronaut/mqtt/hivemq/v3/client/Mqtt3ClientFactory.java#L118
+ // and https://hivemq.github.io/hivemq-mqtt-client/docs/client-configuration/
+ List expectedProtocols = Arrays.asList("TLSv1.2");
+ sslConfig = sslConfigBuilder.trustManagerFactory(tmf).protocols(expectedProtocols).build();
} catch (NoSuchAlgorithmException | CertificateException | IOException | KeyStoreException
| KeyManagementException e) {
log.error("Tenant {} - Connector {} - Exception when configuring socketFactory for TLS: ", tenant,
@@ -171,30 +200,115 @@ public boolean initialize() {
}
}
log.info("Tenant {} - Connector {} - Initialization of connector {} was successful!", tenant,
+ getConnectorType(),
getConnectorName());
return true;
}
- @Override
- public ConnectorSpecification getSpecification() {
- return MQTTClient.spec;
- }
-
@Override
public void connect() {
- log.info("Tenant {} - Establishing the MQTT connection now - phase I: (isConnected:shouldConnect) ({}:{})",
- tenant, isConnected(),
+ updateConnectorStatusAndSend(ConnectorStatus.CONNECTING, true, true);
+ log.info("Tenant {} - Trying to connect to {} - phase I: (isConnected:shouldConnect) ({}:{})",
+ tenant, getConnectorName(), isConnected(),
shouldConnect());
if (isConnected())
disconnect();
+
+ String protocol = (String) connectorConfiguration.getProperties().getOrDefault("protocol", false);
+ boolean useSelfSignedCertificate = (Boolean) connectorConfiguration.getProperties()
+ .getOrDefault("useSelfSignedCertificate", false);
+
+ String mqttHost = (String) connectorConfiguration.getProperties().get("mqttHost");
+ String clientId = (String) connectorConfiguration.getProperties().get("clientId");
+ int mqttPort = (Integer) connectorConfiguration.getProperties().get("mqttPort");
+ String user = (String) connectorConfiguration.getProperties().get("user");
+ String password = (String) connectorConfiguration.getProperties().get("password");
+ boolean useWSS = (Boolean) connectorConfiguration.getProperties().getOrDefault("useWSS", false);
+
+ Mqtt3ClientBuilder partialBuilder;
+ if (useWSS) {
+ partialBuilder = Mqtt3Client.builder().serverHost(mqttHost).webSocketWithDefaultConfig()
+ .serverPort(mqttPort)
+ .identifier(clientId + additionalSubscriptionIdTest);
+ } else {
+ partialBuilder = Mqtt3Client.builder().serverHost(mqttHost).serverPort(mqttPort)
+ .identifier(clientId + additionalSubscriptionIdTest);
+ }
+
+ // is username & password used
+ if (!StringUtils.isEmpty(user)) {
+ Complete simpleAuthComplete = Mqtt3SimpleAuth.builder().username(user);
+ if (!StringUtils.isEmpty(password)) {
+ simpleAuthComplete = simpleAuthComplete.password(password.getBytes());
+ }
+ partialBuilder = partialBuilder
+ .simpleAuth(simpleAuthComplete.build());
+ }
+
+ // tls configuration
+ if (useSelfSignedCertificate) {
+ partialBuilder = partialBuilder.sslConfig(sslConfig);
+ log.debug("Tenant {} - Using certificate: {}", tenant, cert.getCertInPemFormat());
+ } else if ("mqtts://".equals(protocol) || "wss://".equals(protocol)) {
+ partialBuilder = partialBuilder.sslWithDefaultConfig();
+ }
+
+ // websocket configuration
+ if ("ws://".equals(protocol) || "wss://".equals(protocol)) {
+ String serverPath = (String) connectorConfiguration.getProperties().get("serverPath");
+ partialBuilder = partialBuilder.webSocketConfig()
+ .serverPath(serverPath)
+ .applyWebSocketConfig();
+ log.debug("Tenant {} - Using websocket: {}", tenant, serverPath);
+ }
+
+ // finally build mqttClient
+ mqttClient = partialBuilder
+ .addDisconnectedListener(context -> {
+ // test if we closed the connection deliberately, otherwise we have to try to
+ // reconnect
+ connectionState.setFalse();
+ if (connectorConfiguration.enabled)
+ connectionLost(
+ "Disconnected from: " + context.getSource().toString(), context.getCause());
+ })
+ .addConnectedListener(connext -> {
+ connectionState.setTrue();
+ })
+ .buildBlocking();
+
+ String configuredProtocol = "mqtt";
+ String configuredServerPath = "";
+ if (mqttClient.getConfig().getWebSocketConfig().isPresent()) {
+ if (mqttClient.getConfig().getSslConfig().isPresent()) {
+ configuredProtocol = "wss";
+ } else {
+ configuredProtocol = "ws";
+ }
+ configuredServerPath = "/" + mqttClient.getConfig().getWebSocketConfig().get().getServerPath();
+ } else {
+ if (mqttClient.getConfig().getSslConfig().isPresent()) {
+ configuredProtocol = "mqtts";
+ } else {
+ configuredProtocol = "mqtt";
+ }
+ }
+ String configuredUrl = String.format("%s://%s:%s%s", configuredProtocol, mqttClient.getConfig().getServerHost(),
+ mqttClient.getConfig().getServerPort(), configuredServerPath);
+ // Registering Callback
+ Mqtt3AsyncClient mqtt3AsyncClient = mqttClient.toAsync();
+ mqttCallback = new MQTTCallback(dispatcher, tenant, getConnectorIdent(), false);
+ mqtt3AsyncClient.publishes(MqttGlobalPublishFilter.ALL, mqttCallback);
+
// stay in the loop until successful
boolean successful = false;
while (!successful) {
loadConfiguration();
var firstRun = true;
while (!isConnected() && shouldConnect()) {
- log.info("Tenant {} - Establishing the MQTT connection now - phase II: {}", tenant,
- shouldConnect());
+ log.info("Tenant {} - Trying to connect {} - phase II: (shouldConnect):{} {}", tenant,
+ getConnectorName(),
+ shouldConnect(), configuredUrl);
if (!firstRun) {
try {
Thread.sleep(WAIT_PERIOD_MS);
@@ -204,54 +318,30 @@ tenant, isConnected(),
}
}
try {
- boolean useTLS = (Boolean) connectorConfiguration.getProperties().getOrDefault("useTLS", false);
- boolean useSelfSignedCertificate = (Boolean) connectorConfiguration.getProperties()
- .getOrDefault("useSelfSignedCertificate", false);
- String prefix = useTLS ? "ssl://" : "tcp://";
- String mqttHost = (String) connectorConfiguration.getProperties().get("mqttHost");
- String clientId = (String) connectorConfiguration.getProperties().get("clientId");
- int mqttPort = (Integer) connectorConfiguration.getProperties().get("mqttPort");
- String user = (String) connectorConfiguration.getProperties().get("user");
- String password = (String) connectorConfiguration.getProperties().get("password");
- String broker = prefix + mqttHost + ":"
- + mqttPort;
- // mqttClient = new MqttClient(broker, MqttClient.generateClientId(), new
- // MemoryPersistence());
-
- // before we create a new mqttClient, test if there already exists on and try to
- // close it
- if (mqttClient != null) {
- mqttClient.close(true);
+ Mqtt3ConnAck ack = mqttClient.connectWith()
+ .cleanSession(true)
+ .keepAlive(60)
+ .send();
+ if (!ack.getReturnCode().equals(Mqtt3ConnAckReturnCode.SUCCESS)) {
+
+ throw new ConnectorException(
+ String.format("Tenant %s - Error connecting to broker: %s. Errorcode: %s", tenant,
+ mqttClient.getConfig().getServerHost(), ack.getReturnCode().name()));
}
- mqttClient = new MqttClient(broker,
- clientId + additionalSubscriptionIdTest,
- new MemoryPersistence());
- mqttCallback = new MQTTCallback(dispatcher, tenant, MQTTClient.getConnectorType());
- mqttClient.setCallback(mqttCallback);
- MqttConnectOptions connOpts = new MqttConnectOptions();
- connOpts.setCleanSession(true);
- connOpts.setAutomaticReconnect(false);
- // log.info("Tenant {} - DANGEROUS-LOG password: {}", tenant, password);
- if (!StringUtils.isEmpty(user))
- connOpts.setUserName(user);
- if ( !StringUtils.isEmpty(password))
- connOpts.setPassword(password.toCharArray());
- if (useSelfSignedCertificate) {
- log.debug("Tenant {} - Using certificate: {}", tenant, cert.getCertInPemFormat());
- connOpts.setSocketFactory(sslSocketFactory);
- }
- mqttClient.connect(connOpts);
+
+ connectionState.setTrue();
log.info("Tenant {} - Successfully connected to broker {}", tenant,
- mqttClient.getServerURI());
- connectorStatus.updateStatus(ConnectorStatus.CONNECTED, true);
- sendConnectorLifecycle();
- } catch (MqttException e) {
- log.error("Tenant {} - Error on reconnect: {}", tenant, e.getMessage());
+ mqttClient.getConfig().getServerHost());
+ updateConnectorStatusAndSend(ConnectorStatus.CONNECTED, true, true);
+ List updatedMappings = mappingComponent.rebuildMappingInboundCache(tenant);
+ updateActiveSubscriptions(updatedMappings, true);
+
+ } catch (Exception e) {
+ log.error("Tenant {} - Failed to connect to broker {}, {}, {}, {}", tenant,
+ mqttClient.getConfig().getServerHost(), e.getMessage(), connectionState.booleanValue(),
+ mqttClient.getState().isConnected());
updateConnectorStatusToFailed(e);
sendConnectorLifecycle();
- if (serviceConfiguration.logConnectorErrorInBackend) {
- log.error("Tenant {} - Stacktrace:", tenant, e);
- }
}
firstRun = false;
}
@@ -261,19 +351,17 @@ tenant, isConnected(),
if (shouldConnect()) {
try {
// is not working for broker.emqx.io
- subscribe("$SYS/#", 0);
- } catch (Exception e) {
+ subscribe("$SYS/#", QOS.AT_LEAST_ONCE);
+ } catch (ConnectorException e) {
log.warn(
- "Error on subscribing to topic $SYS/#, this might not be supported by the mqtt broker {} {}",
+ "Tenant {} - Error on subscribing to topic $SYS/#, this might not be supported by the mqtt broker {} {}",
e.getMessage(), e);
}
mappingComponent.rebuildMappingOutboundCache(tenant);
// in order to keep MappingInboundCache and ActiveSubscriptionMappingInbound in
// sync, the ActiveSubscriptionMappingInbound is build on the
- // reviously used updatedMappings
- List updatedMappings = mappingComponent.rebuildMappingInboundCache(tenant);
- updateActiveSubscriptions(updatedMappings, true);
+ // previously used updatedMappings
}
successful = true;
} catch (Exception e) {
@@ -288,25 +376,8 @@ tenant, isConnected(),
}
}
- private void updateConnectorStatusToFailed(Exception e) {
- String msg = " --- " + e.getClass().getName() + ": "
- + e.getMessage();
- if (!(e.getCause() == null)) {
- msg = msg + " --- Caused by " + e.getCause().getClass().getName() + ": " + e.getCause().getMessage();
- }
- connectorStatus.setMessage(msg);
- connectorStatus.updateStatus(ConnectorStatus.FAILED, false);
- }
-
@Override
public void close() {
- if (mqttClient != null) {
- try {
- mqttClient.close();
- } catch (MqttException e) {
- log.error("Tenant {} - Error on closing mqttClient {}: ", tenant, e.getMessage(), e);
- }
- }
}
@Override
@@ -321,8 +392,8 @@ public boolean isConfigValid(ConnectorConfiguration configuration) {
return false;
}
// check if all required properties are set
- for (String property : MQTTClient.getSpec().getProperties().keySet()) {
- if (MQTTClient.getSpec().getProperties().get(property).required
+ for (String property : getSpecification().getProperties().keySet()) {
+ if (getSpecification().getProperties().get(property).required
&& configuration.getProperties().get(property) == null) {
return false;
}
@@ -332,45 +403,44 @@ public boolean isConfigValid(ConnectorConfiguration configuration) {
@Override
public boolean isConnected() {
- // log.info("Tenant {} - TESTING isConnected I:,s {}, {}", tenant, mqttClient, getConnectorIdent(),
- // getConnectorName());
- // if (mqttClient != null)
- // log.info("Tenant {} - TESTING isConnected II: {}", tenant, mqttClient.isConnected());
- // else
- // log.info("Tenant {} - TESTING isConnected II: {}, mqttClient is null", tenant);
- return mqttClient != null ? mqttClient.isConnected() : false;
+ return connectionState.booleanValue();
+ // return mqttClient != null ? mqttClient.getState().isConnected() : false;
}
@Override
public void disconnect() {
- log.info("Tenant {} - Diconnecting from MQTT broker: {}", tenant,
- (mqttClient == null ? null : mqttClient.getServerURI()));
- try {
- if (isConnected()) {
- log.debug("Tenant {} - Disconnected from MQTT broker I: {}", tenant, mqttClient.getServerURI());
- activeSubscriptions.entrySet().forEach(entry -> {
- // only unsubscribe if still active subscriptions exist
- String topic = entry.getKey();
- MutableInt activeSubs = entry.getValue();
- if (activeSubs.intValue() > 0) {
- try {
- mqttClient.unsubscribe(topic);
- } catch (MqttException e) {
- log.error("Tenant {} - Exception when unsubscribing from topic: {}: ", tenant, topic, e);
- }
+ if (isConnected()) {
+ updateConnectorStatusAndSend(ConnectorStatus.DISCONNECTING, true, true);
+ log.info("Tenant {} - Disconnecting from broker: {}", tenant,
+ (mqttClient == null ? (String) connectorConfiguration.getProperties().get("mqttHost")
+ : mqttClient.getConfig().getServerHost()));
+ log.debug("Tenant {} - Disconnected from broker I: {}", tenant,
+ mqttClient.getConfig().getServerHost());
+ activeSubscriptions.entrySet().forEach(entry -> {
+ // only unsubscribe if still active subscriptions exist
+ String topic = entry.getKey();
+ MutableInt activeSubs = entry.getValue();
+ if (activeSubs.intValue() > 0 && mqttClient.getState().isConnected()) {
+ mqttClient.unsubscribe(Mqtt3Unsubscribe.builder().topicFilter(topic).build());
+ }
+ });
- }
- });
- mqttClient.unsubscribe("$SYS");
- mqttClient.disconnect();
- connectorStatus.updateStatus(ConnectorStatus.DISCONNECTED, true);
- sendConnectorLifecycle();
- log.info("Tenant {} - Disconnected from MQTT broker II: {}", tenant, mqttClient.getServerURI());
+ if (mqttClient.getState().isConnected()) {
+ mqttClient.unsubscribe(Mqtt3Unsubscribe.builder().topicFilter("$SYS").build());
}
- } catch (MqttException e) {
- log.error("Tenant {} - Error on disconnecting MQTT Client: ", tenant, e);
- updateConnectorStatusToFailed(e);
- sendConnectorLifecycle();
+
+ try {
+ if (mqttClient != null && mqttClient.getState().isConnected())
+ mqttClient.disconnect();
+ } catch (Exception e) {
+ log.error("Tenant {} - Error disconnecting from MQTT broker:", tenant,
+ e);
+ }
+ updateConnectorStatusAndSend(ConnectorStatus.DISCONNECTED, true, true);
+ List updatedMappings = mappingComponent.rebuildMappingInboundCache(tenant);
+ updateActiveSubscriptions(updatedMappings, true);
+ log.info("Tenant {} - Disconnected from MQTT broker II: {}", tenant,
+ mqttClient.getConfig().getServerHost());
}
}
@@ -380,38 +450,68 @@ public String getConnectorIdent() {
}
@Override
- public void subscribe(String topic, Integer qos) throws MqttException {
+ public void subscribe(String topic, QOS qos) throws ConnectorException {
log.debug("Tenant {} - Subscribing on topic: {}", tenant, topic);
+ QOS usedQOS = qos;
sendSubscriptionEvents(topic, "Subscribing");
- if (qos != null)
- mqttClient.subscribe(topic, qos);
- else
- mqttClient.subscribe(topic);
- log.debug("Tenant {} - Successfully subscribed on topic: {}", tenant, topic);
+ if (usedQOS.equals(null))
+ usedQOS = QOS.AT_LEAST_ONCE;
+ else if (!supportedQOS.contains(qos)) {
+ // determine maximum supported QOS
+ usedQOS = QOS.AT_LEAST_ONCE;
+ for (int i = 1; i < qos.ordinal(); i++) {
+ if (supportedQOS.contains(QOS.values()[i])) {
+ usedQOS = QOS.values()[i];
+ }
+ }
+ if (usedQOS.ordinal() < qos.ordinal()) {
+ log.warn("Tenant {} - QOS {} is not supported. Using instead: {}", tenant, qos, usedQOS);
+ }
+ }
+
+ // We don't need to add a handler on subscribe using hive client
+ Mqtt3AsyncClient asyncMqttClient = mqttClient.toAsync();
+ asyncMqttClient.subscribeWith().topicFilter(topic).qos(MqttQos.fromCode(usedQOS.ordinal())).send()
+ .thenRun(() -> {
+ log.debug("Tenant {} - Successfully subscribed on topic: {}", tenant, topic);
+ }).exceptionally(throwable -> {
+ log.error("Tenant {} - Failed to subscribe on topic {} with error: ", tenant, topic,
+ throwable.getMessage());
+ return null;
+ });
}
public void unsubscribe(String topic) throws Exception {
log.debug("Tenant {} - Unsubscribing from topic: {}", tenant, topic);
sendSubscriptionEvents(topic, "Unsubscribing");
- mqttClient.unsubscribe(topic);
+ mqttClient.unsubscribe(Mqtt3Unsubscribe.builder().topicFilter(topic).build());
}
public void publishMEAO(ProcessingContext> context) {
- MqttMessage mqttMessage = new MqttMessage();
C8YRequest currentRequest = context.getCurrentRequest();
String payload = currentRequest.getRequest();
- mqttMessage.setPayload(payload.getBytes());
- try {
- mqttClient.publish(context.getResolvedPublishTopic(), mqttMessage);
- } catch (MqttException e) {
- throw new RuntimeException(e);
- }
- log.info("Tenant {} - Published outbound message: {} for mapping: {} on topic: {}", tenant, payload,
- context.getMapping().name, context.getResolvedPublishTopic());
+ MqttQos mqttQos = MqttQos.fromCode(context.getQos().ordinal());
+ Mqtt3Publish mqttMessage = Mqtt3Publish.builder().topic(context.getResolvedPublishTopic()).qos(mqttQos)
+ .payload(payload.getBytes()).build();
+ mqttClient.publish(mqttMessage);
+
+ log.info("Tenant {} - Published outbound message: {} for mapping: {} on topic: {}, {}", tenant, payload,
+ context.getMapping().name, context.getResolvedPublishTopic(), connectorName);
}
@Override
public String getConnectorName() {
return connectorName;
}
+
+ @Override
+ public Boolean supportsWildcardsInTopic() {
+ return Boolean.parseBoolean(connectorConfiguration.getProperties().get("supportsWildcardInTopic").toString());
+ }
+
+ @Override
+ public void monitorSubscriptions() {
+ // nothing to do
+ }
+
}
\ No newline at end of file
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/mqtt/MQTTServiceClient.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/mqtt/MQTTServiceClient.java
new file mode 100644
index 00000000..72092164
--- /dev/null
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/connector/mqtt/MQTTServiceClient.java
@@ -0,0 +1,118 @@
+/*
+ * Copyright (c) 2022 Software AG, Darmstadt, Germany and/or Software AG USA Inc., Reston, VA, USA,
+ * and/or its subsidiaries and/or its affiliates and/or their licensors.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * @authors Christof Strack, Stefan Witschel
+ */
+
+package dynamic.mapping.connector.mqtt;
+
+import java.util.AbstractMap;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+
+import com.cumulocity.microservice.context.credentials.MicroserviceCredentials;
+
+import dynamic.mapping.connector.core.ConnectorPropertyType;
+import dynamic.mapping.connector.core.ConnectorSpecification;
+import dynamic.mapping.connector.core.client.ConnectorType;
+import dynamic.mapping.processor.inbound.AsynchronousDispatcherInbound;
+import dynamic.mapping.configuration.ConnectorConfiguration;
+import dynamic.mapping.connector.core.ConnectorProperty;
+import dynamic.mapping.core.ConfigurationRegistry;
+import dynamic.mapping.model.QOS;
+
+public class MQTTServiceClient extends MQTTClient {
+ public MQTTServiceClient() {
+ Map configProps = new HashMap<>();
+ configProps.put("protocol",
+ new ConnectorProperty(true, 0, ConnectorPropertyType.OPTION_PROPERTY, true, true, "mqtt://",
+ Map.ofEntries(
+ new AbstractMap.SimpleEntry("mqtt://", "mqtt://"),
+ new AbstractMap.SimpleEntry("mqtts://", "mqtts://"),
+ new AbstractMap.SimpleEntry("ws://", "ws://"),
+ new AbstractMap.SimpleEntry("wss://", "wss://"))));
+ configProps.put("mqttHost",
+ new ConnectorProperty(true, 1, ConnectorPropertyType.STRING_PROPERTY, true, true, "cumulocity",
+ null));
+ configProps.put("mqttPort",
+ new ConnectorProperty(true, 2, ConnectorPropertyType.NUMERIC_PROPERTY, true, true, 2883, null));
+ configProps.put("user",
+ new ConnectorProperty(true, 3, ConnectorPropertyType.STRING_PROPERTY, true, true, null, null));
+ configProps.put("password",
+ new ConnectorProperty(true, 4, ConnectorPropertyType.SENSITIVE_STRING_PROPERTY, true, true, null,
+ null));
+ configProps.put("clientId",
+ new ConnectorProperty(true, 5, ConnectorPropertyType.ID_STRING_PROPERTY, true, true,
+ MQTTServiceClient.nextId(), null));
+ configProps.put("useSelfSignedCertificate",
+ new ConnectorProperty(false, 6, ConnectorPropertyType.BOOLEAN_PROPERTY, true, true, false, null));
+ configProps.put("fingerprintSelfSignedCertificate",
+ new ConnectorProperty(false, 7, ConnectorPropertyType.STRING_PROPERTY, true, true, false, null));
+ configProps.put("nameCertificate",
+ new ConnectorProperty(false, 8, ConnectorPropertyType.STRING_PROPERTY, true, true, false, null));
+ configProps.put("supportsWildcardInTopic",
+ new ConnectorProperty(false, 9, ConnectorPropertyType.BOOLEAN_PROPERTY, true, true, false, null));
+ String description = "Specific connector for connecting to Cumulocity MQTT Service. The MQTT Service does not support wildcards, i.e. '+', '#'. The QOS 'exactly once' is reduced to 'at least once'.";
+ connectorType = ConnectorType.MQTT_SERVICE;
+ specification = new ConnectorSpecification(description, connectorType, configProps, false);
+ }
+
+ private static Random random = new Random();
+
+ private static String nextId() {
+ return "MQTT_SERVICE" + Integer.toString(random.nextInt(Integer.MAX_VALUE - 100000) + 100000, 36);
+ }
+ // return random.nextInt(max - min) + min;
+
+ public MQTTServiceClient(ConfigurationRegistry configurationRegistry,
+ ConnectorConfiguration connectorConfiguration,
+ AsynchronousDispatcherInbound dispatcher, String additionalSubscriptionIdTest, String tenant) {
+ this();
+ this.configurationRegistry = configurationRegistry;
+ this.mappingComponent = configurationRegistry.getMappingComponent();
+ this.serviceConfigurationComponent = configurationRegistry.getServiceConfigurationComponent();
+ this.connectorConfigurationComponent = configurationRegistry.getConnectorConfigurationComponent();
+ this.connectorConfiguration = connectorConfiguration;
+ // ensure the client knows its identity even if configuration is set to null
+ this.connectorIdent = connectorConfiguration.ident;
+ this.connectorName = connectorConfiguration.name;
+ this.c8yAgent = configurationRegistry.getC8yAgent();
+ this.cachedThreadPool = configurationRegistry.getCachedThreadPool();
+ this.objectMapper = configurationRegistry.getObjectMapper();
+ this.additionalSubscriptionIdTest = additionalSubscriptionIdTest;
+ this.mappingServiceRepresentation = configurationRegistry.getMappingServiceRepresentations().get(tenant);
+ this.serviceConfiguration = configurationRegistry.getServiceConfigurations().get(tenant);
+ this.dispatcher = dispatcher;
+ this.tenant = tenant;
+ MicroserviceCredentials msc = configurationRegistry.getMicroserviceCredential(tenant);
+ String user = String.format("%s/%s", tenant, msc.getUsername());
+ getSpecification().getProperties().put("user",
+ new ConnectorProperty(true, 2, ConnectorPropertyType.STRING_PROPERTY, true, true, user, null));
+ getSpecification().getProperties().put("password",
+ new ConnectorProperty(true, 3, ConnectorPropertyType.SENSITIVE_STRING_PROPERTY, true, true,
+ msc.getPassword(), null));
+ this.supportedQOS = Arrays.asList(QOS.AT_LEAST_ONCE, QOS.AT_MOST_ONCE);
+ }
+
+ @Override
+ public Boolean supportsWildcardsInTopic() {
+ return false;
+ }
+}
\ No newline at end of file
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/core/BootstrapService.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/core/BootstrapService.java
index a48c0c04..b1bd91d8 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/core/BootstrapService.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/core/BootstrapService.java
@@ -1,6 +1,6 @@
package dynamic.mapping.core;
-import java.util.HashMap;
+import java.io.IOException;
import java.util.List;
import java.util.TimeZone;
import java.util.concurrent.ExecutorService;
@@ -8,8 +8,10 @@
import dynamic.mapping.configuration.ServiceConfiguration;
import dynamic.mapping.configuration.ServiceConfigurationComponent;
import dynamic.mapping.connector.core.client.AConnectorClient;
+import dynamic.mapping.connector.core.client.ConnectorType;
import dynamic.mapping.connector.core.registry.ConnectorRegistry;
import dynamic.mapping.connector.core.registry.ConnectorRegistryException;
+import dynamic.mapping.connector.kafka.KafkaClient;
import dynamic.mapping.model.MappingServiceRepresentation;
import dynamic.mapping.processor.inbound.AsynchronousDispatcherInbound;
import dynamic.mapping.processor.outbound.AsynchronousDispatcherOutbound;
@@ -29,6 +31,7 @@
import dynamic.mapping.configuration.ConnectorConfiguration;
import dynamic.mapping.configuration.ConnectorConfigurationComponent;
import dynamic.mapping.connector.mqtt.MQTTClient;
+import dynamic.mapping.connector.mqtt.MQTTServiceClient;
@Service
@EnableScheduling
@@ -72,7 +75,6 @@ public void destroy(MicroserviceSubscriptionRemovedEvent event) {
configurationRegistry.getNotificationSubscriber().unsubscribeTenantSubscriber(tenant);
configurationRegistry.getNotificationSubscriber().unsubscribeDeviceSubscriber(tenant);
-
try {
connectorRegistry.unregisterAllClientsForTenant(tenant);
} catch (ConnectorRegistryException e) {
@@ -91,6 +93,7 @@ public void destroy(MicroserviceSubscriptionRemovedEvent event) {
public void initialize(MicroserviceSubscriptionAddedEvent event) {
// Executed for each tenant subscribed
String tenant = event.getCredentials().getTenant();
+ configurationRegistry.getMicroserviceCredentials().put(tenant, event.getCredentials());
log.info("Tenant {} - Microservice subscribed", tenant);
TimeZone.setDefault(TimeZone.getTimeZone("Europe/Berlin"));
ManagedObjectRepresentation mappingServiceMOR = configurationRegistry.getC8yAgent()
@@ -107,11 +110,14 @@ public void initialize(MicroserviceSubscriptionAddedEvent event) {
configurationRegistry.getMappingServiceRepresentations().put(tenant, mappingServiceRepresentation);
mappingComponent.initializeMappingStatus(tenant, false);
mappingComponent.initializeMappingCaches(tenant);
-
- // TODO Add other clients static property definition here
- connectorRegistry.registerConnector(MQTTClient.getConnectorType(), MQTTClient.getSpec());
+ mappingComponent.rebuildMappingOutboundCache(tenant);
+ mappingComponent.rebuildMappingInboundCache(tenant);
try {
+ // TODO Add other clients static property definition here
+ connectorRegistry.registerConnector(ConnectorType.MQTT, new MQTTClient().getSpecification());
+ connectorRegistry.registerConnector(ConnectorType.MQTT_SERVICE, new MQTTServiceClient().getSpecification());
+ connectorRegistry.registerConnector(ConnectorType.KAFKA, new KafkaClient().getSpecification());
if (serviceConfiguration != null) {
List connectorConfigurationList = connectorConfigurationComponent
.getConnectorConfigurations(tenant);
@@ -129,7 +135,7 @@ public void initialize(MicroserviceSubscriptionAddedEvent event) {
log.info("Tenant {} - OutputMapping Config Enabled: {}", tenant, outputMappingEnabled);
if (outputMappingEnabled) {
- //configurationRegistry.getNotificationSubscriber().initTenantClient();
+ // configurationRegistry.getNotificationSubscriber().initTenantClient();
configurationRegistry.getNotificationSubscriber().initDeviceClient();
}
}
@@ -137,20 +143,17 @@ public void initialize(MicroserviceSubscriptionAddedEvent event) {
public AConnectorClient initializeConnectorByConfiguration(ConnectorConfiguration connectorConfiguration,
ServiceConfiguration serviceConfiguration, String tenant) throws ConnectorRegistryException {
AConnectorClient connectorClient = null;
-
- if (MQTTClient.getConnectorType().equals(connectorConfiguration.getConnectorType())) {
- log.info("Tenant {} - Initializing MQTT Connector with ident {}", tenant,
- connectorConfiguration.getIdent());
- MQTTClient mqttClient = new MQTTClient(configurationRegistry, connectorConfiguration,
- null,
+ try {
+ connectorClient = configurationRegistry.createConnectorClient(connectorConfiguration,
additionalSubscriptionIdTest, tenant);
-
- connectorRegistry.registerClient(tenant, mqttClient);
- connectorClient = mqttClient;
+ } catch (IOException e) {
+ log.error("Tenant {} - Error on creating connector {} {}", connectorConfiguration.getConnectorType(), e);
+ throw new ConnectorRegistryException(e.getMessage());
}
-
+ connectorRegistry.registerClient(tenant, connectorClient);
// initialize AsynchronousDispatcherInbound
- AsynchronousDispatcherInbound dispatcherInbound = new AsynchronousDispatcherInbound(configurationRegistry, connectorClient);
+ AsynchronousDispatcherInbound dispatcherInbound = new AsynchronousDispatcherInbound(configurationRegistry,
+ connectorClient);
configurationRegistry.initializePayloadProcessorsInbound(tenant);
connectorClient.setDispatcher(dispatcherInbound);
connectorClient.reconnect();
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/core/C8YAgent.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/core/C8YAgent.java
index 4d6f2657..e53c0f97 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/core/C8YAgent.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/core/C8YAgent.java
@@ -21,51 +21,9 @@
package dynamic.mapping.core;
-import static java.util.Map.entry;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.Enumeration;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import javax.ws.rs.core.MediaType;
-
-import dynamic.mapping.App;
-import dynamic.mapping.configuration.TrustedCertificateCollectionRepresentation;
-import dynamic.mapping.configuration.TrustedCertificateRepresentation;
-import dynamic.mapping.connector.core.client.AConnectorClient;
-import dynamic.mapping.core.facade.IdentityFacade;
-import dynamic.mapping.core.facade.InventoryFacade;
-import dynamic.mapping.model.Extension;
-import dynamic.mapping.model.ExtensionEntry;
-import dynamic.mapping.model.MappingServiceRepresentation;
-import dynamic.mapping.processor.ProcessingException;
-import dynamic.mapping.processor.extension.ExtensibleProcessorInbound;
-import dynamic.mapping.processor.extension.ExtensionsComponent;
-import dynamic.mapping.processor.extension.ProcessorExtensionInbound;
-import dynamic.mapping.processor.model.C8YRequest;
-import dynamic.mapping.processor.model.ProcessingContext;
-import org.apache.commons.io.IOUtils;
-import org.joda.time.DateTime;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.annotation.ImportBeanDefinitionRegistrar;
-import org.springframework.context.annotation.Lazy;
-import org.springframework.stereotype.Component;
-import org.svenson.JSONParser;
-
+import c8y.IsDevice;
+import com.cumulocity.microservice.context.ContextService;
+import com.cumulocity.microservice.context.credentials.MicroserviceCredentials;
import com.cumulocity.microservice.subscription.service.MicroserviceSubscriptionsService;
import com.cumulocity.model.Agent;
import com.cumulocity.model.ID;
@@ -88,16 +46,49 @@
import com.cumulocity.sdk.client.inventory.BinariesApi;
import com.cumulocity.sdk.client.measurement.MeasurementApi;
import com.fasterxml.jackson.core.JsonProcessingException;
-
-import c8y.IsDevice;
+import dynamic.mapping.App;
+import dynamic.mapping.configuration.TrustedCertificateCollectionRepresentation;
+import dynamic.mapping.configuration.TrustedCertificateRepresentation;
+import dynamic.mapping.connector.core.client.AConnectorClient;
+import dynamic.mapping.core.facade.IdentityFacade;
+import dynamic.mapping.core.facade.InventoryFacade;
+import dynamic.mapping.model.API;
+import dynamic.mapping.model.Extension;
+import dynamic.mapping.model.ExtensionEntry;
+import dynamic.mapping.model.MappingServiceRepresentation;
+import dynamic.mapping.processor.ProcessingException;
+import dynamic.mapping.processor.extension.ExtensibleProcessorInbound;
+import dynamic.mapping.processor.extension.ExtensionsComponent;
+import dynamic.mapping.processor.extension.ProcessorExtensionInbound;
+import dynamic.mapping.processor.model.C8YRequest;
+import dynamic.mapping.processor.model.ProcessingContext;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
-import dynamic.mapping.model.API;
+import org.apache.commons.io.IOUtils;
+import org.joda.time.DateTime;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.ImportBeanDefinitionRegistrar;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Component;
+import org.svenson.JSONParser;
+
+import javax.ws.rs.core.MediaType;
+import java.io.*;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.*;
+
+import static java.util.Map.entry;
@Slf4j
@Component
public class C8YAgent implements ImportBeanDefinitionRegistrar {
+ ConnectorStatus previousConnectorStatus = ConnectorStatus.UNKNOWN;
+
@Autowired
private EventApi eventApi;
@@ -125,6 +116,9 @@ public class C8YAgent implements ImportBeanDefinitionRegistrar {
@Autowired
private MicroserviceSubscriptionsService subscriptionsService;
+ @Autowired
+ private ContextService contextService;
+
private ExtensionsComponent extensionsComponent;
@Autowired
@@ -157,8 +151,11 @@ public void setConfigurationRegistry(@Lazy ConfigurationRegistry configurationRe
private static final String PACKAGE_MAPPING_PROCESSOR_EXTENSION_EXTERNAL = "dynamic.mapping.processor.extension.external";
+ @Value("${application.version}")
+ private String version;
+
public ExternalIDRepresentation resolveExternalId2GlobalId(String tenant, ID identity,
- ProcessingContext> context) {
+ ProcessingContext> context) {
if (identity.getType() == null) {
identity.setType("c8y_Serial");
}
@@ -174,7 +171,7 @@ public ExternalIDRepresentation resolveExternalId2GlobalId(String tenant, ID ide
}
public ExternalIDRepresentation resolveGlobalId2ExternalId(String tenant, GId gid, String idType,
- ProcessingContext> context) {
+ ProcessingContext> context) {
if (idType == null) {
idType = "c8y_Serial";
}
@@ -191,59 +188,67 @@ public ExternalIDRepresentation resolveGlobalId2ExternalId(String tenant, GId gi
}
public MeasurementRepresentation createMeasurement(String name, String type, ManagedObjectRepresentation mor,
- DateTime dateTime, HashMap mvMap, String tenant) {
+ DateTime dateTime, HashMap mvMap, String tenant) {
MeasurementRepresentation measurementRepresentation = new MeasurementRepresentation();
subscriptionsService.runForTenant(tenant, () -> {
- try {
- measurementRepresentation.set(mvMap, name);
- measurementRepresentation.setType(type);
- measurementRepresentation.setSource(mor);
- measurementRepresentation.setDateTime(dateTime);
- log.debug("Tenant {} - Creating Measurement {}", tenant, measurementRepresentation);
- MeasurementRepresentation mrn = measurementApi.create(measurementRepresentation);
- measurementRepresentation.setId(mrn.getId());
- } catch (SDKException e) {
- log.error("Tenant {} - Error creating Measurement", tenant, e);
- }
+ MicroserviceCredentials context = removeAppKeyHeaderFromContext(contextService.getContext());
+ contextService.runWithinContext(context, () -> {
+ try {
+ measurementRepresentation.set(mvMap, name);
+ measurementRepresentation.setType(type);
+ measurementRepresentation.setSource(mor);
+ measurementRepresentation.setDateTime(dateTime);
+ log.debug("Tenant {} - Creating Measurement {}", tenant, measurementRepresentation);
+ MeasurementRepresentation mrn = measurementApi.create(measurementRepresentation);
+ measurementRepresentation.setId(mrn.getId());
+ } catch (SDKException e) {
+ log.error("Tenant {} - Error creating Measurement", tenant, e);
+ }
+ });
});
return measurementRepresentation;
}
public AlarmRepresentation createAlarm(String severity, String message, String type, DateTime alarmTime,
- ManagedObjectRepresentation parentMor, String tenant) {
+ ManagedObjectRepresentation parentMor, String tenant) {
AlarmRepresentation alarmRepresentation = subscriptionsService.callForTenant(tenant, () -> {
- AlarmRepresentation ar = new AlarmRepresentation();
- ar.setSeverity(severity);
- ar.setSource(parentMor);
- ar.setText(message);
- ar.setDateTime(alarmTime);
- ar.setStatus("ACTIVE");
- ar.setType(type);
-
- return this.alarmApi.create(ar);
+ MicroserviceCredentials context = removeAppKeyHeaderFromContext(contextService.getContext());
+ return contextService.callWithinContext(context, () -> {
+ AlarmRepresentation ar = new AlarmRepresentation();
+ ar.setSeverity(severity);
+ ar.setSource(parentMor);
+ ar.setText(message);
+ ar.setDateTime(alarmTime);
+ ar.setStatus("ACTIVE");
+ ar.setType(type);
+ return this.alarmApi.create(ar);
+ });
});
return alarmRepresentation;
}
public void createEvent(String message, String type, DateTime eventTime, MappingServiceRepresentation source,
- String tenant, Map properties) {
+ String tenant, Map properties) {
subscriptionsService.runForTenant(tenant, () -> {
- EventRepresentation er = new EventRepresentation();
- ManagedObjectRepresentation mor = new ManagedObjectRepresentation();
- mor.setId(new GId(source.getId()));
- er.setSource(mor);
- er.setText(message);
- er.setDateTime(eventTime);
- er.setType(type);
- if (properties != null) {
- er.setProperty(C8YAgent.CONNECTOR_FRAGMENT, properties);
- }
- this.eventApi.createAsync(er);
+ MicroserviceCredentials context = removeAppKeyHeaderFromContext(contextService.getContext());
+ contextService.runWithinContext(context, () -> {
+ EventRepresentation er = new EventRepresentation();
+ ManagedObjectRepresentation mor = new ManagedObjectRepresentation();
+ mor.setId(new GId(source.getId()));
+ er.setSource(mor);
+ er.setText(message);
+ er.setDateTime(eventTime);
+ er.setType(type);
+ if (properties != null) {
+ er.setProperty(C8YAgent.CONNECTOR_FRAGMENT, properties);
+ }
+ this.eventApi.createAsync(er);
+ });
});
}
public AConnectorClient.Certificate loadCertificateByName(String certificateName, String fingerprint,
- String tenant, String connectorName) {
+ String tenant, String connectorName) {
TrustedCertificateRepresentation result = subscriptionsService.callForTenant(tenant,
() -> {
log.info("Tenant {} - Connector {} - Retrieving certificate {} ", tenant, connectorName,
@@ -305,39 +310,42 @@ public AbstractExtensibleRepresentation createMEAO(ProcessingContext> context)
String payload = currentRequest.getRequest();
API targetAPI = context.getMapping().getTargetAPI();
AbstractExtensibleRepresentation result = subscriptionsService.callForTenant(tenant, () -> {
- AbstractExtensibleRepresentation rt = null;
- try {
- if (targetAPI.equals(API.EVENT)) {
- EventRepresentation eventRepresentation = configurationRegistry.getObjectMapper().readValue(payload,
- EventRepresentation.class);
- rt = eventApi.create(eventRepresentation);
- log.info("Tenant {} - New event posted: {}", tenant, rt);
- } else if (targetAPI.equals(API.ALARM)) {
- AlarmRepresentation alarmRepresentation = configurationRegistry.getObjectMapper().readValue(payload,
- AlarmRepresentation.class);
- rt = alarmApi.create(alarmRepresentation);
- log.info("Tenant {} - New alarm posted: {}", tenant, rt);
- } else if (targetAPI.equals(API.MEASUREMENT)) {
- MeasurementRepresentation measurementRepresentation = jsonParser
- .parse(MeasurementRepresentation.class, payload);
- rt = measurementApi.create(measurementRepresentation);
- log.info("Tenant {} - New measurement posted: {}", tenant, rt);
- } else if (targetAPI.equals(API.OPERATION)) {
- OperationRepresentation operationRepresentation = jsonParser
- .parse(OperationRepresentation.class, payload);
- rt = deviceControlApi.create(operationRepresentation);
- log.info("Tenant {} - New operation posted: {}", tenant, rt);
- } else {
- log.error("Tenant {} - Not existing API!", tenant);
+ MicroserviceCredentials contextCredentials = removeAppKeyHeaderFromContext(contextService.getContext());
+ return contextService.callWithinContext(contextCredentials, () -> {
+ AbstractExtensibleRepresentation rt = null;
+ try {
+ if (targetAPI.equals(API.EVENT)) {
+ EventRepresentation eventRepresentation = configurationRegistry.getObjectMapper().readValue(payload,
+ EventRepresentation.class);
+ rt = eventApi.create(eventRepresentation);
+ log.info("Tenant {} - New event posted: {}", tenant, rt);
+ } else if (targetAPI.equals(API.ALARM)) {
+ AlarmRepresentation alarmRepresentation = configurationRegistry.getObjectMapper().readValue(payload,
+ AlarmRepresentation.class);
+ rt = alarmApi.create(alarmRepresentation);
+ log.info("Tenant {} - New alarm posted: {}", tenant, rt);
+ } else if (targetAPI.equals(API.MEASUREMENT)) {
+ MeasurementRepresentation measurementRepresentation = jsonParser
+ .parse(MeasurementRepresentation.class, payload);
+ rt = measurementApi.create(measurementRepresentation);
+ log.info("Tenant {} - New measurement posted: {}", tenant, rt);
+ } else if (targetAPI.equals(API.OPERATION)) {
+ OperationRepresentation operationRepresentation = jsonParser
+ .parse(OperationRepresentation.class, payload);
+ rt = deviceControlApi.create(operationRepresentation);
+ log.info("Tenant {} - New operation posted: {}", tenant, rt);
+ } else {
+ log.error("Tenant {} - Not existing API!", tenant);
+ }
+ } catch (JsonProcessingException e) {
+ log.error("Tenant {} - Could not map payload: {} {}", tenant, targetAPI, payload);
+ error.append("Could not map payload: " + targetAPI + "/" + payload);
+ } catch (SDKException s) {
+ log.error("Tenant {} - Could not sent payload to c8y: {} {}: ", tenant, targetAPI, payload, s);
+ error.append("Could not sent payload to c8y: " + targetAPI + "/" + payload + "/" + s);
}
- } catch (JsonProcessingException e) {
- log.error("Tenant {} - Could not map payload: {} {}", tenant, targetAPI, payload);
- error.append("Could not map payload: " + targetAPI + "/" + payload);
- } catch (SDKException s) {
- log.error("Tenant {} - Could not sent payload to c8y: {} {}: ", tenant, targetAPI, payload, s);
- error.append("Could not sent payload to c8y: " + targetAPI + "/" + payload + "/" + s);
- }
- return rt;
+ return rt;
+ });
});
if (!error.toString().equals("")) {
throw new ProcessingException(error.toString());
@@ -350,34 +358,46 @@ public ManagedObjectRepresentation upsertDevice(String tenant, ID identity, Proc
StringBuffer error = new StringBuffer("");
C8YRequest currentRequest = context.getCurrentRequest();
ManagedObjectRepresentation device = subscriptionsService.callForTenant(tenant, () -> {
- ManagedObjectRepresentation mor = configurationRegistry.getObjectMapper().readValue(
- currentRequest.getRequest(),
- ManagedObjectRepresentation.class);
- try {
- ExternalIDRepresentation extId = resolveExternalId2GlobalId(tenant, identity, context);
- if (extId == null) {
- // Device does not exist
- // append external id to name
- mor.setName(mor.getName());
- mor.set(new IsDevice());
- // remove id
- mor.setId(null);
-
- mor = inventoryApi.create(mor, context);
- log.info("Tenant {} - New device created: {}", tenant, mor);
- identityApi.create(mor, identity, context);
- } else {
- // Device exists - update needed
- mor.setId(extId.getManagedObject().getId());
- mor = inventoryApi.update(mor, context);
+ MicroserviceCredentials contextCredentials = removeAppKeyHeaderFromContext(contextService.getContext());
+ return contextService.callWithinContext(contextCredentials, () -> {
+ ManagedObjectRepresentation mor = configurationRegistry.getObjectMapper().readValue(
+ currentRequest.getRequest(),
+ ManagedObjectRepresentation.class);
+ try {
+ ExternalIDRepresentation extId = resolveExternalId2GlobalId(tenant, identity, context);
+ if (extId == null) {
+ // Device does not exist
+ // append external id to name
+ mor.setName(mor.getName());
+ /*
+ mor.set(new Agent());
+ HashMap agentFragments = new HashMap<>();
+ agentFragments.put("name", "Dynamic Mapper");
+ agentFragments.put("version", version);
+ agentFragments.put("url", "https://github.com/SoftwareAG/cumulocity-dynamic-mapper");
+ agentFragments.put("maintainer", "Open-Source");
+ mor.set(agentFragments, "c8y_Agent");
+ */
+ mor.set(new IsDevice());
+ // remove id
+ mor.setId(null);
+
+ mor = inventoryApi.create(mor, context);
+ log.info("Tenant {} - New device created: {}", tenant, mor);
+ identityApi.create(mor, identity, context);
+ } else {
+ // Device exists - update needed
+ mor.setId(extId.getManagedObject().getId());
+ mor = inventoryApi.update(mor, context);
- log.info("Tenant {} - Device updated: {}", tenant, mor);
+ log.info("Tenant {} - Device updated: {}", tenant, mor);
+ }
+ } catch (SDKException s) {
+ log.error("Tenant {} - Could not sent payload to c8y: {}: ", tenant, currentRequest.getRequest(), s);
+ error.append("Could not sent payload to c8y: " + currentRequest.getRequest() + " " + s);
}
- } catch (SDKException s) {
- log.error("Tenant {} - Could not sent payload to c8y: {}: ", tenant, currentRequest.getRequest(), s);
- error.append("Could not sent payload to c8y: " + currentRequest.getRequest() + " " + s);
- }
- return mor;
+ return mor;
+ });
});
if (!error.toString().equals("")) {
throw new ProcessingException(error.toString());
@@ -393,7 +413,7 @@ public void loadProcessorExtensions(String tenant) {
Map, ?> props = (Map, ?>) (extension.get(ExtensionsComponent.PROCESSOR_EXTENSION_TYPE));
String extName = props.get("name").toString();
boolean external = (Boolean) props.get("external");
- log.info("Tenant {} - Trying to load extension id: {}, name: {}", tenant, extension.getId().getValue(),
+ log.debug("Tenant {} - Trying to load extension id: {}, name: {}", tenant, extension.getId().getValue(),
extName);
try {
if (external) {
@@ -413,7 +433,7 @@ public void loadProcessorExtensions(String tenant) {
IOUtils.copy(downloadInputStream, outputStream);
// step 3 parse list of extensions
- URL[] urls = { tempFile.toURI().toURL() };
+ URL[] urls = {tempFile.toURI().toURL()};
externalClassLoader = new URLClassLoader(urls, App.class.getClassLoader());
registerExtensionInProcessor(tenant, extension.getId().getValue(), extName, externalClassLoader,
external);
@@ -422,14 +442,15 @@ public void loadProcessorExtensions(String tenant) {
external);
}
} catch (IOException e) {
- log.error("Tenant {} - Exception occurred, When loading extension, starting without extensions: ", tenant,
+ log.error("Tenant {} - Exception occurred, When loading extension, starting without extensions: ",
+ tenant,
e);
}
}
}
private void registerExtensionInProcessor(String tenant, String id, String extensionName, ClassLoader dynamicLoader,
- boolean external)
+ boolean external)
throws IOException {
ExtensibleProcessorInbound extensibleProcessor = configurationRegistry.getExtensibleProcessors().get(tenant);
extensibleProcessor.addExtension(tenant, new Extension(id, extensionName, external));
@@ -450,7 +471,7 @@ private void registerExtensionInProcessor(String tenant, String id, String exten
if (buffered != null)
newExtensions.load(buffered);
- log.info("Tenant {} - Preparing to load extensions:" + newExtensions.toString(), tenant);
+ log.debug("Tenant {} - Preparing to load extensions:" + newExtensions.toString(), tenant);
Enumeration> extensions = newExtensions.propertyNames();
while (extensions.hasMoreElements()) {
@@ -481,7 +502,7 @@ private void registerExtensionInProcessor(String tenant, String id, String exten
.newInstance();
// springUtil.registerBean(key, clazz);
extensionEntry.setExtensionImplementation(extensionImpl);
- log.info("Tenant {} - Successfully registered bean: {} for key: {}", tenant,
+ log.debug("Tenant {} - Successfully registered bean: {} for key: {}", tenant,
newExtensions.getProperty(key),
key);
}
@@ -538,18 +559,21 @@ public ManagedObjectRepresentation getManagedObjectForId(String tenant, String d
}
public void updateOperationStatus(String tenant, OperationRepresentation op, OperationStatus status,
- String failureReason) {
+ String failureReason) {
subscriptionsService.runForTenant(tenant, () -> {
- try {
- op.setStatus(status.toString());
- if (failureReason != null)
- op.setFailureReason(failureReason);
- deviceControlApi.update(op);
- } catch (SDKException exception) {
- log.error("Tenant {} - Operation with id {} could not be updated: {}", tenant,
- op.getDeviceId().getValue(),
- exception.getLocalizedMessage());
- }
+ MicroserviceCredentials contextCredentials = removeAppKeyHeaderFromContext(contextService.getContext());
+ contextService.runWithinContext(contextCredentials, () -> {
+ try {
+ op.setStatus(status.toString());
+ if (failureReason != null)
+ op.setFailureReason(failureReason);
+ deviceControlApi.update(op);
+ } catch (SDKException exception) {
+ log.error("Tenant {} - Operation with id {} could not be updated: {}", tenant,
+ op.getDeviceId().getValue(),
+ exception.getLocalizedMessage());
+ }
+ });
});
}
@@ -565,13 +589,16 @@ public ManagedObjectRepresentation initializeMappingServiceObject(String tenant)
log.info("Tenant {} - Agent with ID {} already exists {}", tenant,
MappingServiceRepresentation.AGENT_ID,
mappingServiceIdRepresentation, amo.getId());
- log.info("Tenant {} - Agent representation {}", tenant,
- MappingServiceRepresentation.AGENT_ID,
- mappingServiceIdRepresentation);
} else {
amo.setName(MappingServiceRepresentation.AGENT_NAME);
amo.setType(MappingServiceRepresentation.AGENT_TYPE);
amo.set(new Agent());
+ HashMap agentFragments = new HashMap<>();
+ agentFragments.put("name", "Dynamic Mapper");
+ agentFragments.put("version", version);
+ agentFragments.put("url", "https://github.com/SoftwareAG/cumulocity-dynamic-mapper");
+ agentFragments.put("maintainer", "Open-Source");
+ amo.set(agentFragments, "c8y_Agent");
amo.set(new IsDevice());
amo.setProperty(C8YAgent.MAPPING_FRAGMENT,
new ArrayList<>());
@@ -589,7 +616,7 @@ public ManagedObjectRepresentation initializeMappingServiceObject(String tenant)
public void createExtensibleProcessor(String tenant) {
ExtensibleProcessorInbound extensibleProcessor = new ExtensibleProcessorInbound(configurationRegistry);
configurationRegistry.getExtensibleProcessors().put(tenant, extensibleProcessor);
- log.info("Tenant {} - create ExtensibleProcessor {}", tenant, extensibleProcessor);
+ log.debug("Tenant {} - Create ExtensibleProcessor {}", tenant, extensibleProcessor);
// check if managedObject for internal mapping extension exists
List internalExtension = extensionsComponent.getInternal();
@@ -605,13 +632,15 @@ public void createExtensibleProcessor(String tenant) {
} else {
ie = internalExtension.get(0);
}
- log.info("Tenant {} - Internal extension: {} registered: {}", tenant,
+ log.debug("Tenant {} - Internal extension: {} registered: {}", tenant,
ExtensionsComponent.PROCESSOR_EXTENSION_INTERNAL_NAME,
ie.getId().getValue(), ie);
}
public void sendNotificationLifecycle(String tenant, ConnectorStatus connectorStatus, String message) {
- if (configurationRegistry.getServiceConfigurations().get(tenant).sendNotificationLifecycle) {
+ if (configurationRegistry.getServiceConfigurations().get(tenant).sendNotificationLifecycle
+ && !(connectorStatus.equals(previousConnectorStatus))) {
+ previousConnectorStatus = connectorStatus;
DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
Date now = new Date();
String date = dateFormat.format(now);
@@ -629,4 +658,13 @@ public void sendNotificationLifecycle(String tenant, ConnectorStatus connectorSt
}
}
+ public MicroserviceCredentials removeAppKeyHeaderFromContext(MicroserviceCredentials context) {
+ final MicroserviceCredentials clonedContext = new MicroserviceCredentials(
+ context.getTenant(),
+ context.getUsername(), context.getPassword(),
+ context.getOAuthAccessToken(), context.getXsrfToken(),
+ context.getTfaToken(), null);
+ return clonedContext;
+ }
+
}
\ No newline at end of file
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/core/ConfigurationRegistry.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/core/ConfigurationRegistry.java
index 75734a09..8e54ba50 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/core/ConfigurationRegistry.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/core/ConfigurationRegistry.java
@@ -1,5 +1,7 @@
package dynamic.mapping.core;
+import java.io.FileNotFoundException;
+import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ExecutorService;
@@ -8,12 +10,18 @@
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
+import com.cumulocity.microservice.context.credentials.MicroserviceCredentials;
import com.fasterxml.jackson.databind.ObjectMapper;
+import dynamic.mapping.configuration.ConnectorConfiguration;
import dynamic.mapping.configuration.ConnectorConfigurationComponent;
import dynamic.mapping.configuration.ServiceConfiguration;
import dynamic.mapping.configuration.ServiceConfigurationComponent;
import dynamic.mapping.connector.core.client.AConnectorClient;
+import dynamic.mapping.connector.core.client.ConnectorType;
+import dynamic.mapping.connector.kafka.KafkaClient;
+import dynamic.mapping.connector.mqtt.MQTTClient;
+import dynamic.mapping.connector.mqtt.MQTTServiceClient;
import dynamic.mapping.model.MappingServiceRepresentation;
import dynamic.mapping.notification.C8YNotificationSubscriber;
import dynamic.mapping.processor.extension.ExtensibleProcessorInbound;
@@ -33,6 +41,9 @@
@Component
public class ConfigurationRegistry {
+ @Getter
+ private Map microserviceCredentials = new HashMap<>();
+
// structure: >
@Getter
private Map mappingServiceRepresentations = new HashMap<>();
@@ -89,7 +100,8 @@ public void setMappingComponent(@Lazy MappingComponent mappingComponent) {
private ConnectorConfigurationComponent connectorConfigurationComponent;
@Autowired
- public void setConnectorConfigurationComponent(@Lazy ConnectorConfigurationComponent connectorConfigurationComponent) {
+ public void setConnectorConfigurationComponent(
+ @Lazy ConnectorConfigurationComponent connectorConfigurationComponent) {
this.connectorConfigurationComponent = connectorConfigurationComponent;
}
@@ -108,7 +120,6 @@ public void setServiceConfigurationComponent(@Lazy ServiceConfigurationComponent
public Map> createPayloadProcessorsInbound(String tenant) {
ExtensibleProcessorInbound extensibleProcessor = getExtensibleProcessors().get(tenant);
- log.info("Tenant {} - payloadProcessorsInbound {}", tenant, extensibleProcessor);
return Map.of(
MappingType.JSON, new JSONProcessorInbound(this),
MappingType.FLAT_FILE, new FlatFileProcessorInbound(this),
@@ -117,6 +128,31 @@ MappingType.PROTOBUF_STATIC, new StaticProtobufProcessor(this),
MappingType.PROCESSOR_EXTENSION, extensibleProcessor);
}
+ public AConnectorClient createConnectorClient(ConnectorConfiguration connectorConfiguration,
+ String additionalSubscriptionIdTest, String tenant) throws FileNotFoundException, IOException {
+ AConnectorClient connectorClient = null;
+ if (ConnectorType.MQTT.equals(connectorConfiguration.getConnectorType())) {
+ connectorClient = new MQTTClient(this, connectorConfiguration,
+ null,
+ additionalSubscriptionIdTest, tenant);
+ log.info("Tenant {} - Initializing MQTT Connector with ident {}", tenant,
+ connectorConfiguration.getIdent());
+ } else if (ConnectorType.MQTT_SERVICE.equals(connectorConfiguration.getConnectorType())) {
+ connectorClient = new MQTTServiceClient(this, connectorConfiguration,
+ null,
+ additionalSubscriptionIdTest, tenant);
+ log.info("Tenant {} - Initializing MQTTService Connector with ident {}", tenant,
+ connectorConfiguration.getIdent());
+ } else if (ConnectorType.KAFKA.equals(connectorConfiguration.getConnectorType())) {
+ connectorClient = new KafkaClient(this, connectorConfiguration,
+ null,
+ additionalSubscriptionIdTest, tenant);
+ log.info("Tenant {} - Initializing Kafka Connector with ident {}", tenant,
+ connectorConfiguration.getIdent());
+ }
+ return connectorClient;
+ }
+
public Map> createPayloadProcessorsOutbound(
AConnectorClient connectorClient) {
return Map.of(
@@ -145,4 +181,9 @@ public void initializePayloadProcessorsOutbound(AConnectorClient connectorClient
createPayloadProcessorsOutbound(connectorClient));
}
}
+
+ public MicroserviceCredentials getMicroserviceCredential(String tenant) {
+ MicroserviceCredentials ms = microserviceCredentials.get(tenant);
+ return ms;
+ }
}
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/core/MappingComponent.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/core/MappingComponent.java
index 77e895fe..5bda99a5 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/core/MappingComponent.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/core/MappingComponent.java
@@ -21,8 +21,6 @@
package dynamic.mapping.core;
-import static java.util.Map.entry;
-
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -49,7 +47,7 @@
import lombok.extern.slf4j.Slf4j;
import dynamic.mapping.model.API;
import dynamic.mapping.model.Direction;
-import dynamic.mapping.model.TreeNode;
+import dynamic.mapping.model.MappingTreeNode;
import dynamic.mapping.model.Mapping;
import dynamic.mapping.model.MappingRepresentation;
import dynamic.mapping.model.MappingServiceRepresentation;
@@ -95,20 +93,20 @@ public class MappingComponent {
// cache of inbound mappings stored in a tree used for resolving
@Getter
- private Map resolverMappingInbound = new HashMap<>();
+ private Map resolverMappingInbound = new HashMap<>();
public void initializeMappingCaches(String tenant) {
cacheMappingInbound.put(tenant, new HashMap<>());
cacheMappingOutbound.put(tenant, new HashMap<>());
resolverMappingOutbound.put(tenant, new HashMap<>());
- resolverMappingInbound.put(tenant, TreeNode.createRootNode(tenant));
+ resolverMappingInbound.put(tenant, MappingTreeNode.createRootNode(tenant));
}
public void initializeMappingStatus(String tenant, boolean reset) {
MappingServiceRepresentation mappingServiceRepresentation = configurationRegistry
.getMappingServiceRepresentations().get(tenant);
if (mappingServiceRepresentation.getMappingStatus() != null && !reset) {
- log.info("Tenant {} - Initializing status: {}, {} ", tenant,
+ log.debug("Tenant {} - Initializing status: {}, {} ", tenant,
mappingServiceRepresentation.getMappingStatus(),
(mappingServiceRepresentation.getMappingStatus() == null
|| mappingServiceRepresentation.getMappingStatus().size() == 0 ? 0
@@ -126,7 +124,7 @@ public void initializeMappingStatus(String tenant, boolean reset) {
MappingStatus.UNSPECIFIED_MAPPING_STATUS);
}
initializedMappingStatus.put(tenant, true);
- resolverMappingInbound.put(tenant, TreeNode.createRootNode(tenant));
+ resolverMappingInbound.put(tenant, MappingTreeNode.createRootNode(tenant));
if (cacheMappingInbound.get(tenant) == null)
cacheMappingInbound.put(tenant, new HashMap<>());
if (cacheMappingOutbound.get(tenant) == null)
@@ -173,32 +171,6 @@ public void sendMappingStatus(String tenant) {
}
}
- public void sendConnectorLifecycle(String tenant, String connectorIdent, ConnectorStatusEvent connectorStatus,
- String connectorName) {
- if (configurationRegistry.getServiceConfigurations().get(tenant).sendConnectorLifecycle) {
- subscriptionsService.runForTenant(tenant, () -> {
- MappingServiceRepresentation mappingServiceRepresentation = configurationRegistry
- .getMappingServiceRepresentations().get(tenant);
- Map> ccs = consolidatedConnectorStatus.getOrDefault(tenant,
- new HashMap>());
- log.debug("Tenant {} - Sending status connector: {}", tenant, ccs);
- Map stMap = Map.ofEntries(
- entry("status", connectorStatus.getStatus().name()),
- entry("message", connectorStatus.message),
- entry("connectorName", connectorName),
- entry("date", connectorStatus.date));
- ccs.put(connectorIdent, stMap);
- consolidatedConnectorStatus.put(tenant, ccs);
- Map service = new HashMap();
- service.put(C8YAgent.CONNECTOR_FRAGMENT, ccs);
- ManagedObjectRepresentation updateMor = new ManagedObjectRepresentation();
- updateMor.setId(GId.asGId(mappingServiceRepresentation.getId()));
- updateMor.setAttrs(service);
- this.inventoryApi.update(updateMor);
- });
- }
- }
-
public MappingStatus getMappingStatus(String tenant, Mapping m) {
// log.info("Tenant {} - get MappingStatus: {}", tenant, m.ident);
Map statusMapping = tenantStatusMapping.get(tenant);
@@ -235,7 +207,7 @@ public Mapping getMapping(String tenant, String id) {
ManagedObjectRepresentation mo = inventoryApi.get(GId.asGId(id));
if (mo != null) {
Mapping mt = toMappingObject(mo).getC8yMQTTMapping();
- log.info("Tenant {} - Found Mapping: {}", tenant, mt.id);
+ log.debug("Tenant {} - Found Mapping: {}", tenant, mt.id);
return mt;
}
return null;
@@ -249,15 +221,15 @@ public Mapping deleteMapping(String tenant, String id) {
ManagedObjectRepresentation mo = inventoryApi.get(GId.asGId(id));
MappingRepresentation m = toMappingObject(mo);
if (m.getC8yMQTTMapping().isActive()) {
- throw new IllegalArgumentException("Tenant " + tenant + " - Mapping " + id
- + " is still active, deactivate mapping before deleting!");
+ throw new IllegalArgumentException(String.format(
+ "Tenant %s - Mapping %s is still active, deactivate mapping before deleting!", tenant, id));
}
// mapping is deactivated and we can delete it
inventoryApi.delete(GId.asGId(id));
deleteMappingStatus(tenant, id);
return m.getC8yMQTTMapping();
});
- log.info("Tenant {} - Deleted Mapping: {}", tenant, id);
+ //log.info("Tenant {} - Deleted Mapping: {}", tenant, id);
return result;
}
@@ -284,8 +256,9 @@ public Mapping updateMapping(String tenant, Mapping mapping, boolean allowUpdate
// when we do housekeeping tasks we need to update active mapping, e.g. add
// snooped messages. This is an exception
if (!allowUpdateWhenActive && mapping.isActive()) {
- throw new IllegalArgumentException("Tenant " + tenant + " - Mapping " + mapping.id
- + " is still active, deactivate mapping before deleting!");
+ throw new IllegalArgumentException(
+ String.format("Tenant %s - Mapping %s is still active, deactivate mapping before updating!",
+ tenant, mapping.id));
}
// mapping is deactivated and we can delete it
List mappings = getMappings(tenant);
@@ -321,7 +294,7 @@ public Mapping createMapping(String tenant, Mapping mapping) {
if (errors.size() != 0) {
String errorList = errors.stream().map(e -> e.toString()).reduce("",
(res, error) -> res + "[ " + error + " ]");
- throw new RuntimeException("Validation errors:" + errorList);
+ throw new RuntimeException(String.format("Validation errors: %s", errorList));
}
Mapping result = subscriptionsService.callForTenant(tenant, () -> {
MappingRepresentation mr = new MappingRepresentation();
@@ -339,7 +312,8 @@ public Mapping createMapping(String tenant, Mapping mapping) {
mor.setId(GId.asGId(mapping.id));
mor.setName(mapping.name);
inventoryApi.update(mor);
- log.info("Tenant {} - Created mapping: {}", mor, tenant);
+ log.info("Tenant {} - Mapping created: {}", tenant, mor.getName());
+ log.debug("Tenant {} - Mapping created: {}", tenant, mor);
return mapping;
});
return result;
@@ -428,8 +402,8 @@ public Mapping deleteFromMappingCache(String tenant, Mapping mapping) {
}
}
- public TreeNode rebuildMappingTree(List mappings, String tenant) {
- TreeNode in = TreeNode.createRootNode(tenant);
+ public MappingTreeNode rebuildMappingTree(List mappings, String tenant) {
+ MappingTreeNode in = MappingTreeNode.createRootNode(tenant);
mappings.forEach(m -> {
try {
in.addMapping(m);
@@ -456,14 +430,14 @@ public List rebuildMappingInboundCache(String tenant) {
return rebuildMappingInboundCache(tenant, updatedMappings);
}
- public void setActivationMapping(String tenant, String id, Boolean active) throws Exception {
+ public Mapping setActivationMapping(String tenant, String mappingId, Boolean active) throws Exception {
// step 1. update activation for mapping
- log.info("Tenant {} - Setting active: {} got mapping: {}", tenant, id, active);
- Mapping mapping = getMapping(tenant, id);
+ log.debug("Tenant {} - Setting active: {} got mapping: {}", tenant, active, mappingId );
+ Mapping mapping = getMapping(tenant, mappingId);
mapping.setActive(active);
if (Direction.INBOUND.equals(mapping.direction)) {
// step 2. retrieve collected snoopedTemplates
- mapping.setSnoopedTemplates(cacheMappingInbound.get(tenant).get(id).getSnoopedTemplates());
+ mapping.setSnoopedTemplates(cacheMappingInbound.get(tenant).get(mappingId).getSnoopedTemplates());
}
// step 3. update mapping in inventory
// don't validate mapping when setting active = false, this allows to remove
@@ -480,6 +454,32 @@ public void setActivationMapping(String tenant, String id, Boolean active) throw
addToCacheMappingInbound(tenant, mapping);
cacheMappingInbound.get(tenant).put(mapping.id, mapping);
}
+ return mapping;
+ }
+
+ public void setDebugMapping(String tenant, String id, Boolean debug) throws Exception {
+ // step 1. update debug for mapping
+ log.info("Tenant {} - Setting debug: {} got mapping: {}", tenant, id, debug);
+ Mapping mapping = getMapping(tenant, id);
+ mapping.setDebug(debug);
+ if (Direction.INBOUND.equals(mapping.direction)) {
+ // step 2. retrieve collected snoopedTemplates
+ mapping.setSnoopedTemplates(cacheMappingInbound.get(tenant).get(id).getSnoopedTemplates());
+ }
+ // step 3. update mapping in inventory
+ // don't validate mapping when setting active = false, this allows to remove
+ // mappings that are not working
+ updateMapping(tenant, mapping, true, true);
+ // step 4. delete mapping from update cache
+ removeDirtyMapping(tenant, mapping);
+ // step 5. update caches
+ if (Direction.OUTBOUND.equals(mapping.direction)) {
+ rebuildMappingOutboundCache(tenant);
+ } else {
+ deleteFromCacheMappingInbound(tenant, mapping);
+ addToCacheMappingInbound(tenant, mapping);
+ cacheMappingInbound.get(tenant).put(mapping.id, mapping);
+ }
}
public void cleanDirtyMappings(String tenant) throws Exception {
@@ -510,7 +510,7 @@ public void addDirtyMapping(String tenant, Mapping mapping) {
}
public List resolveMappingInbound(String tenant, String topic) throws ResolveException {
- List resolvedMappings = getResolverMappingInbound().get(tenant)
+ List resolvedMappings = getResolverMappingInbound().get(tenant)
.resolveTopicPath(Mapping.splitTopicIncludingSeparatorAsList(topic));
return resolvedMappings.stream().filter(tn -> tn.isMappingNode())
.map(mn -> mn.getMapping()).collect(Collectors.toList());
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/core/Operation.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/core/Operation.java
index e28081c3..b8af23bb 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/core/Operation.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/core/Operation.java
@@ -30,4 +30,6 @@ public enum Operation {
RELOAD_MAPPINGS,
RESET_STATUS_MAPPING,
REFRESH_NOTIFICATIONS_SUBSCRIPTIONS,
+ DEBUG_MAPPING,
+
}
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/core/mock/MockIdentity.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/core/mock/MockIdentity.java
index ae8a0b14..df945c68 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/core/mock/MockIdentity.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/core/mock/MockIdentity.java
@@ -24,10 +24,8 @@
import com.cumulocity.model.ID;
import com.cumulocity.model.idtype.GId;
import com.cumulocity.rest.representation.identity.ExternalIDRepresentation;
-import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
-@Slf4j
@Service
public class MockIdentity {
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/model/API.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/model/API.java
index 0feda15e..e6e233f1 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/model/API.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/model/API.java
@@ -62,7 +62,7 @@ private API(String name, String identifier, String notificationFilter) {
static public API fromString(String value) {
API api = ALIAS_MAP.get(value);
if (api == null)
- throw new IllegalArgumentException("Not an alias: " + value);
+ throw new IllegalArgumentException(String.format("Not an alias: %s", value));
return api;
}
}
\ No newline at end of file
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/model/Mapping.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/model/Mapping.java
index b0d54abb..ef2e6d6b 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/model/Mapping.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/model/Mapping.java
@@ -41,141 +41,152 @@
@ToString(exclude = { "source", "target", "snoopedTemplates" })
public class Mapping implements Serializable {
- public static final String TOKEN_TOPIC_LEVEL = "_TOPIC_LEVEL_";
+ public static final String TOKEN_TOPIC_LEVEL = "_TOPIC_LEVEL_";
+ public static final String TOKEN_CONTEXT_DATA = "_CONTEXT_DATA_";
+ public static final String CONTEXT_DATA_KEY_NAME = "key";
+
+ public static final String TIME = "time";
+ public static int SNOOP_TEMPLATES_MAX = 10;
+ public static final String SPLIT_TOPIC_REGEXP = "((?<=/)|(?=/))";
+ public static Mapping UNSPECIFIED_MAPPING;
+
+ static {
+ UNSPECIFIED_MAPPING = new Mapping();
+ UNSPECIFIED_MAPPING.setId(MappingStatus.IDENT_UNSPECIFIED_MAPPING);
+ UNSPECIFIED_MAPPING.setIdent(MappingStatus.IDENT_UNSPECIFIED_MAPPING);
+ }
+
+ @NotNull
+ public String name;
- public static final String TIME = "time";
- public static int SNOOP_TEMPLATES_MAX = 5;
- public static final String SPLIT_TOPIC_REGEXP = "((?<=/)|(?=/))";
- public static Mapping UNSPECIFIED_MAPPING;
+ @NotNull
+ public String id;
- static {
- UNSPECIFIED_MAPPING = new Mapping();
- UNSPECIFIED_MAPPING.setId(MappingStatus.IDENT_UNSPECIFIED_MAPPING);
- UNSPECIFIED_MAPPING.setIdent(MappingStatus.IDENT_UNSPECIFIED_MAPPING);
- }
+ @NotNull
+ public String ident;
- @NotNull
- public String name;
+ @NotNull
+ public String subscriptionTopic;
- @NotNull
- public String id;
+ @NotNull
+ public String publishTopic;
- @NotNull
- public String ident;
+ @NotNull
+ public String publishTopicSample;
- @NotNull
- public String subscriptionTopic;
+ @NotNull
+ public String mappingTopic;
- @NotNull
- public String publishTopic;
+ @NotNull
+ public String mappingTopicSample;
- @NotNull
- public String templateTopic;
+ @NotNull
+ public API targetAPI;
- @NotNull
- public String templateTopicSample;
+ @NotNull
+ public String source;
- @NotNull
- public API targetAPI;
+ @NotNull
+ public String target;
- @NotNull
- public String source;
+ @NotNull
+ public boolean active;
- @NotNull
- public String target;
+ @NotNull
+ public boolean tested;
- @NotNull
- public boolean active;
+ @NotNull
+ public QOS qos;
- @NotNull
- public boolean tested;
+ @NotNull
+ public MappingSubstitution[] substitutions;
- @NotNull
- public QOS qos;
+ @NotNull
+ public boolean mapDeviceIdentifier;
- @NotNull
- public MappingSubstitution[] substitutions;
+ @NotNull
+ public boolean createNonExistingDevice;
- @NotNull
- public boolean mapDeviceIdentifier;
+ @NotNull
+ public boolean updateExistingDevice;
- @NotNull
- public boolean createNonExistingDevice;
+ @NotNull
+ public String externalIdType;
- @NotNull
- public boolean updateExistingDevice;
+ @NotNull
+ public SnoopStatus snoopStatus;
- @NotNull
- public String externalIdType;
+ @NotNull
+ public ArrayList snoopedTemplates;
- @NotNull
- public SnoopStatus snoopStatus;
+ @NotNull
+ public MappingType mappingType;
- @NotNull
- public ArrayList snoopedTemplates;
+ @NotNull
+ @JsonSetter(nulls = Nulls.SKIP)
+ public ExtensionEntry extension;
- @NotNull
- public MappingType mappingType;
+ @NotNull
+ @JsonSetter(nulls = Nulls.SKIP)
+ public Direction direction;
- @NotNull
- @JsonSetter(nulls = Nulls.SKIP)
- public ExtensionEntry extension;
+ @NotNull
+ @JsonSetter(nulls = Nulls.SKIP)
+ public String filterOutbound;
- @NotNull
- @JsonSetter(nulls = Nulls.SKIP)
- public Direction direction;
+ @NotNull
+ @JsonSetter(nulls = Nulls.SKIP)
+ public Boolean autoAckOperation;
- @NotNull
- @JsonSetter(nulls = Nulls.SKIP)
- public String filterOutbound;
+ @NotNull
+ public boolean debug;
- @NotNull
- @JsonSetter(nulls = Nulls.SKIP)
- public Boolean autoAckOperation;
+ @NotNull
+ public boolean supportsMessageContext;
- @NotNull
- public long lastUpdate;
+ @NotNull
+ public long lastUpdate;
+
+ @Override
+ public boolean equals(Object m) {
+ return (m instanceof Mapping) && id == ((Mapping) m).id;
+ }
- @Override
- public boolean equals(Object m) {
- return (m instanceof Mapping) && id == ((Mapping) m).id;
- }
+ public void addSnoopedTemplate(String payloadMessage) {
+ snoopedTemplates.add(payloadMessage);
+ if (snoopedTemplates.size() > SNOOP_TEMPLATES_MAX) {
+ // remove oldest payload
+ snoopedTemplates.remove(0);
+ } else {
+ snoopStatus = SnoopStatus.STARTED;
+ }
+ }
+
+ public void sortSubstitutions() {
+ MappingSubstitution[] sortedSubstitutions = Arrays.stream(substitutions).sorted(
+ (s1, s2) -> -(Boolean.valueOf(s1.definesDeviceIdentifier(targetAPI, direction))
+ .compareTo(Boolean.valueOf(s2.definesDeviceIdentifier(targetAPI, direction)))))
+ .toArray(size -> new MappingSubstitution[size]);
+ substitutions = sortedSubstitutions;
+ }
+
+ public static String[] splitTopicIncludingSeparatorAsArray(String topic) {
+ topic = topic.trim().replaceAll("(\\/{1,}$)|(^\\/{1,})", "/");
+ return topic.split(SPLIT_TOPIC_REGEXP);
+ }
+
+ public static List splitTopicIncludingSeparatorAsList(String topic) {
+ return new ArrayList(
+ Arrays.asList(Mapping.splitTopicIncludingSeparatorAsArray(topic)));
+ }
+
+ public static String[] splitTopicExcludingSeparatorAsArray(String topic) {
+ topic = topic.trim().replaceAll("(\\/{1,}$)|(^\\/{1,})", "");
+ return topic.split("\\/");
+ }
- public void addSnoopedTemplate(String payloadMessage) {
- snoopedTemplates.add(payloadMessage);
- if (snoopedTemplates.size() >= SNOOP_TEMPLATES_MAX) {
- // remove oldest payload
- snoopedTemplates.remove(0);
- } else {
- snoopStatus = SnoopStatus.STARTED;
+ public static List splitTopicExcludingSeparatorAsList(String topic) {
+ return new ArrayList(
+ Arrays.asList(Mapping.splitTopicExcludingSeparatorAsArray(topic)));
}
- }
-
- public void sortSubstitutions() {
- MappingSubstitution[] sortedSubstitutions = Arrays.stream(substitutions).sorted(
- (s1, s2) -> -(Boolean.valueOf(s1.definesDeviceIdentifier(targetAPI, direction))
- .compareTo(Boolean.valueOf(s2.definesDeviceIdentifier(targetAPI, direction)))))
- .toArray(size -> new MappingSubstitution[size]);
- substitutions = sortedSubstitutions;
- }
-
- public static String[] splitTopicIncludingSeparatorAsArray(String topic) {
- topic = topic.trim().replaceAll("(\\/{1,}$)|(^\\/{1,})", "/");
- return topic.split(SPLIT_TOPIC_REGEXP);
- }
-
- public static List splitTopicIncludingSeparatorAsList(String topic) {
- return new ArrayList(
- Arrays.asList(Mapping.splitTopicIncludingSeparatorAsArray(topic)));
- }
-
- public static String[] splitTopicExcludingSeparatorAsArray(String topic) {
- topic = topic.trim().replaceAll("(\\/{1,}$)|(^\\/{1,})", "");
- return topic.split("\\/");
- }
-
- public static List splitTopicExcludingSeparatorAsList(String topic) {
- return new ArrayList(
- Arrays.asList(Mapping.splitTopicExcludingSeparatorAsArray(topic)));
- }
}
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/model/MappingDeployment.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/model/MappingDeployment.java
new file mode 100644
index 00000000..644790b5
--- /dev/null
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/model/MappingDeployment.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2022 Software AG, Darmstadt, Germany and/or Software AG USA Inc., Reston, VA, USA,
+ * and/or its subsidiaries and/or its affiliates and/or their licensors.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * @authors Christof Strack, Stefan Witschel
+ */
+
+package dynamic.mapping.model;
+
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import dynamic.mapping.configuration.ConnectorConfiguration;
+
+import javax.validation.constraints.NotNull;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+
+@Getter
+@Setter
+@NoArgsConstructor
+public class MappingDeployment implements Serializable {
+ public MappingDeployment(String ident) {
+ this.ident = ident;
+ this.deployedToConnectors = new ArrayList<>();
+ }
+
+ @NotNull
+ public String ident;
+ @NotNull
+ public ArrayList deployedToConnectors;
+}
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/model/MappingRepresentation.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/model/MappingRepresentation.java
index 7abd23c4..e6ff1ff9 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/model/MappingRepresentation.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/model/MappingRepresentation.java
@@ -40,244 +40,251 @@
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
import dynamic.mapping.processor.model.MappingType;
+@Slf4j
@Data
@NoArgsConstructor
@AllArgsConstructor
public class MappingRepresentation implements Serializable {
- public static final String MAPPING_TYPE = "d11r_mapping";
- public static final String MAPPING_FRAGMENT = "d11r_mapping";
- public static final String MAPPING_GENERATED_TEST_DEVICE = "d11r_device_generatedType";
- static final String REGEXP_REMOVE_TRAILING_SLASHES = "#\\/$";
- static final String REGEXP_REDUCE_LEADING_TRAILING_SLASHES = "(\\/{2,}$)|(^\\/{2,})";
- static String TOPIC_WILDCARD_MULTI = "#";
- static String TOPIC_WILDCARD_SINGLE = "+";
-
- @JsonProperty("id")
- private String id;
-
- @JsonProperty("type")
- private String type;
-
- @JsonProperty(value = "name")
- private String name;
-
- @JsonProperty(value = "description")
- private String description;
-
- @JsonProperty(value = MAPPING_FRAGMENT)
- private Mapping c8yMQTTMapping;
-
- static public boolean isWildcardTopic(String topic) {
- var result = topic.contains(TOPIC_WILDCARD_MULTI) || topic.contains(TOPIC_WILDCARD_SINGLE);
- return result;
- }
-
- /*
- * only one substitution can be marked with definesIdentifier == true
- */
- static public ArrayList isSubstitutionValid(Mapping mapping) {
- ArrayList result = new ArrayList();
- long count = Arrays.asList(mapping.substitutions).stream()
- .filter(sub -> sub.definesDeviceIdentifier(mapping.targetAPI, mapping.direction)).count();
-
- if (mapping.snoopStatus != SnoopStatus.ENABLED && mapping.snoopStatus != SnoopStatus.STARTED
- && !mapping.mappingType.equals(MappingType.PROCESSOR_EXTENSION)
- && !mapping.mappingType.equals(MappingType.PROTOBUF_STATIC)
- && !mapping.direction.equals(Direction.OUTBOUND)) {
- if (count > 1) {
- result.add(ValidationError.Only_One_Substitution_Defining_Device_Identifier_Can_Be_Used);
- }
- if (count < 1) {
- result.add(ValidationError.One_Substitution_Defining_Device_Identifier_Must_Be_Used);
- }
+ public static final String MAPPING_TYPE = "d11r_mapping";
+ public static final String MAPPING_FRAGMENT = "d11r_mapping";
+ public static final String MAPPING_GENERATED_TEST_DEVICE = "d11r_device_generatedType";
+ static final String REGEXP_REMOVE_TRAILING_SLASHES = "#\\/$";
+ static final String REGEXP_REDUCE_LEADING_TRAILING_SLASHES = "(\\/{2,}$)|(^\\/{2,})";
+ static String TOPIC_WILDCARD_MULTI = "#";
+ static String TOPIC_WILDCARD_SINGLE = "+";
+ @JsonProperty("id")
+ private String id;
+
+ @JsonProperty("type")
+ private String type;
+
+ @JsonProperty(value = "name")
+ private String name;
+
+ @JsonProperty(value = "description")
+ private String description;
+
+ @JsonProperty(value = MAPPING_FRAGMENT)
+ private Mapping c8yMQTTMapping;
+
+ static public boolean isWildcardTopic(String topic) {
+ var result = topic.contains(TOPIC_WILDCARD_MULTI) || topic.contains(TOPIC_WILDCARD_SINGLE);
+ return result;
}
- return result;
- }
-
- static public ArrayList isTemplateTopicValid(String topic) {
- // templateTopic can contain any number of "+" TOPIC_WILDCARD_SINGLE but no "#"
- // TOPIC_WILDCARD_MULTI
- ArrayList result = new ArrayList();
- int count = topic.length() - topic.replace(TOPIC_WILDCARD_MULTI, "").length();
- if (count >= 1) {
- result.add(ValidationError.No_Multi_Level_Wildcard_Allowed_In_TemplateTopic);
- }
- return result;
- }
-
- static public ArrayList isSubscriptionTopicValid(String topic) {
- ArrayList result = new ArrayList();
- int count = topic.length() - topic.replace(TOPIC_WILDCARD_SINGLE, "").length();
- if (count > 1) {
- result.add(ValidationError.Only_One_Single_Level_Wildcard);
+
+ /*
+ * only one substitution can be marked with definesIdentifier == true
+ */
+ static public ArrayList isSubstitutionValid(Mapping mapping) {
+ ArrayList result = new ArrayList();
+ long count = Arrays.asList(mapping.substitutions).stream()
+ .filter(sub -> sub.definesDeviceIdentifier(mapping.targetAPI, mapping.direction)).count();
+
+ if (mapping.snoopStatus != SnoopStatus.ENABLED && mapping.snoopStatus != SnoopStatus.STARTED
+ && !mapping.mappingType.equals(MappingType.PROCESSOR_EXTENSION)
+ && !mapping.mappingType.equals(MappingType.PROTOBUF_STATIC)
+ && !mapping.direction.equals(Direction.OUTBOUND)) {
+ if (count > 1) {
+ result.add(ValidationError.Only_One_Substitution_Defining_Device_Identifier_Can_Be_Used);
+ }
+ if (count < 1) {
+ result.add(ValidationError.One_Substitution_Defining_Device_Identifier_Must_Be_Used);
+ }
+
+ }
+ return result;
}
- count = topic.length() - topic.replace(TOPIC_WILDCARD_MULTI, "").length();
- if (count > 1) {
- result.add(ValidationError.Only_One_Multi_Level_Wildcard);
+
+ static public ArrayList isMappingTopicValid(String topic) {
+ // mappingTopic can contain any number of "+" TOPIC_WILDCARD_SINGLE but no "#"
+ // TOPIC_WILDCARD_MULTI
+ ArrayList result = new ArrayList();
+ int count = topic.length() - topic.replace(TOPIC_WILDCARD_MULTI, "").length();
+ if (count >= 1) {
+ result.add(ValidationError.No_Multi_Level_Wildcard_Allowed_In_MappingTopic);
+ }
+ return result;
}
- if (count >= 1 && topic.indexOf(TOPIC_WILDCARD_MULTI) != topic.length() - 1) {
- result.add(ValidationError.Multi_Level_Wildcard_Only_At_End);
+
+ static public ArrayList isSubscriptionTopicValid(String topic) {
+ ArrayList result = new ArrayList();
+ int count = topic.length() - topic.replace(TOPIC_WILDCARD_SINGLE, "").length();
+ // disable this test: Why is it still needed?
+ // if (count > 1) {
+ // result.add(ValidationError.Only_One_Single_Level_Wildcard);
+ // }
+
+ count = topic.length() - topic.replace(TOPIC_WILDCARD_MULTI, "").length();
+ if (count > 1) {
+ result.add(ValidationError.Only_One_Multi_Level_Wildcard);
+ }
+ if (count >= 1 && topic.indexOf(TOPIC_WILDCARD_MULTI) != topic.length() - 1) {
+ result.add(ValidationError.Multi_Level_Wildcard_Only_At_End);
+ }
+ return result;
}
- return result;
- }
-
- static public List isTemplateTopicSubscriptionTopicValid(Mapping mapping) {
- List result = new ArrayList();
-
- // does the template topic is covered by the subscriptionTopic
- BiFunction topicMatcher = (st,
- tt) -> (Pattern.matches(String.join("[^\\/]+", st.replace("/", "\\/").split("\\+")).replace("#", ".*"), tt));
- boolean error = (!topicMatcher.apply(mapping.subscriptionTopic, mapping.templateTopic));
- if (error) {
- result.add(ValidationError.TemplateTopic_Must_Match_The_SubscriptionTopic);
+
+ static public List isMappingTopicAndSubscriptionTopicValid(Mapping mapping) {
+ List result = new ArrayList();
+
+ // is the template topic covered by the subscriptionTopic
+ BiFunction topicMatcher = (st,
+ tt) -> (Pattern.matches(String.join("[^\\/]+", st.replace("/", "\\/").split("\\+")).replace("#", ".*"),
+ tt));
+ // append trailing null character to avoid that the last "+" is swallowed
+ String st = mapping.subscriptionTopic + "\u0000";
+ String mt = mapping.mappingTopic + "\u0000";
+
+ log.debug("Testing st:" + st + "Testing tt:" + mt );
+ boolean error = (!topicMatcher.apply(st, mt));
+ if (error) {
+ result.add(ValidationError.MappingTopic_Must_Match_The_SubscriptionTopic);
+ }
+ return result;
}
- return result;
- }
-
- static public List isTemplateTopicUnique(List mappings, Mapping mapping) {
- ArrayList result = new ArrayList();
- var templateTopic = mapping.templateTopic;
- mappings.forEach(m -> {
- if ((templateTopic.startsWith(m.templateTopic) || m.templateTopic.startsWith(templateTopic))
- && (mapping.id != m.id)) {
- result.add(ValidationError.TemplateTopic_Must_Not_Be_Substring_Of_Other_TemplateTopic);
- }
- });
- return result;
- }
-
- static public List isFilterOutboundUnique(List mappings, Mapping mapping) {
- ArrayList result = new ArrayList();
- var filterOutbound = mapping.filterOutbound;
- mappings.forEach(m -> {
- if ((filterOutbound.equals(m.filterOutbound))
- && (mapping.id != m.id)) {
- result.add(ValidationError.FilterOutbound_Must_Be_Unique);
- }
- });
- return result;
- }
-
- static public List isMappingValid(List mappings, Mapping mapping) {
- ArrayList result = new ArrayList();
- result.addAll(isSubstitutionValid(mapping));
- result.addAll(isTemplateTopicValid(mapping.templateTopic));
- if (mapping.direction.equals(Direction.INBOUND)) {
- result.addAll(isSubscriptionTopicValid(mapping.subscriptionTopic));
- result.addAll(isTemplateTopicTemplateAndTopicSampleValid(mapping.templateTopic, mapping.templateTopicSample));
- } else {
- // test if we can attach multiple outbound mappings to the same filterOutbound
- // result.addAll(isFilterOutboundUnique(mappings,mapping));
- result.addAll(isPublishTopicTemplateAndTopicSampleValid(mapping.publishTopic, mapping.templateTopicSample));
+
+ static public List isFilterOutboundUnique(List mappings, Mapping mapping) {
+ ArrayList result = new ArrayList();
+ var filterOutbound = mapping.filterOutbound;
+ mappings.forEach(m -> {
+ if ((filterOutbound.equals(m.filterOutbound))
+ && (mapping.id != m.id)) {
+ result.add(ValidationError.FilterOutbound_Must_Be_Unique);
+ }
+ });
+ return result;
}
- result.addAll(areJSONTemplatesValid(mapping));
- // result.addAll(isTemplateTopicUnique(mappings, mapping));
- return result;
- }
-
- private static Collection extends ValidationError> isPublishTopicTemplateAndTopicSampleValid(
- @NotNull String publishTopic, @NotNull String templateTopicSample) {
- ArrayList result = new ArrayList();
- String[] splitPT = Mapping.splitTopicIncludingSeparatorAsArray(publishTopic);
- String[] splitTTS = Mapping.splitTopicIncludingSeparatorAsArray(templateTopicSample);
- if (splitPT.length != splitTTS.length) {
- result.add(ValidationError.PublishTopic_And_TemplateTopicSample_Do_Not_Have_Same_Number_Of_Levels_In_Topic_Name);
- } else {
- for (int i = 0; i < splitPT.length; i++) {
- if (("/").equals(splitPT[i]) && !("/").equals(splitTTS[i])) {
- result.add(ValidationError.PublishTopic_And_TemplateTopicSample_Do_Not_Have_Same_Structure_In_Topic_Name);
- break;
- }
- if (("/").equals(splitTTS[i]) && !("/").equals(splitPT[i])) {
- result.add(ValidationError.PublishTopic_And_TemplateTopicSample_Do_Not_Have_Same_Structure_In_Topic_Name);
- break;
+ static public List isMappingValid(List mappings, Mapping mapping) {
+ ArrayList result = new ArrayList();
+ result.addAll(isSubstitutionValid(mapping));
+ if (mapping.direction.equals(Direction.INBOUND)) {
+ result.addAll(isSubscriptionTopicValid(mapping.subscriptionTopic));
+ result.addAll(isMappingTopicAndSubscriptionTopicValid(mapping));
+ // result.addAll(isMappingTopicTemplateAndTopicSampleValid(mapping.subscriptionTopic,
+ // mapping.mappingTopicSample));
+ } else {
+ // test if we can attach multiple outbound mappings to the same filterOutbound
+ result.addAll(
+ isPublishTopicTemplateAndPublishTopicSampleValid(mapping.publishTopic, mapping.publishTopicSample));
}
- if (!("/").equals(splitPT[i]) && !("+").equals(splitPT[i]) && !("#").equals(splitPT[i])) {
- if (!splitPT[i].equals(splitTTS[i])) {
- result.add(ValidationError.PublishTopic_And_TemplateTopicSample_Do_Not_Have_Same_Structure_In_Topic_Name);
- break;
- }
+
+ result.addAll(areJSONTemplatesValid(mapping));
+ // result.addAll(isMappingTopicUnique(mappings, mapping));
+ return result;
+ }
+
+ private static Collection extends ValidationError> isPublishTopicTemplateAndPublishTopicSampleValid(
+ @NotNull String publishTopic, @NotNull String publishTopicSample) {
+ ArrayList result = new ArrayList();
+ String[] splitPT = Mapping.splitTopicIncludingSeparatorAsArray(publishTopic);
+ String[] splitTTS = Mapping.splitTopicIncludingSeparatorAsArray(publishTopicSample);
+ if (splitPT.length != splitTTS.length) {
+ result.add(
+ ValidationError.PublishTopic_And_PublishTopicSample_Do_Not_Have_Same_Number_Of_Levels_In_Topic_Name);
+ } else {
+ for (int i = 0; i < splitPT.length; i++) {
+ if (("/").equals(splitPT[i]) && !("/").equals(splitTTS[i])) {
+ result.add(
+ ValidationError.PublishTopic_And_PublishTopicSample_Do_Not_Have_Same_Structure_In_Topic_Name);
+ break;
+ }
+ if (("/").equals(splitTTS[i]) && !("/").equals(splitPT[i])) {
+ result.add(
+ ValidationError.PublishTopic_And_PublishTopicSample_Do_Not_Have_Same_Structure_In_Topic_Name);
+ break;
+ }
+ if (!("/").equals(splitPT[i]) && !("+").equals(splitPT[i]) && !("#").equals(splitPT[i])) {
+ if (!splitPT[i].equals(splitTTS[i])) {
+ result.add(
+ ValidationError.PublishTopic_And_PublishTopicSample_Do_Not_Have_Same_Structure_In_Topic_Name);
+ break;
+ }
+ }
+ }
}
- }
+ return result;
}
- return result;
- }
-
- /*
- * test if mapping.templateTopic and mapping.templateTopicSample have the same
- * structure and same number of levels
- */
- public static List isTemplateTopicTemplateAndTopicSampleValid(String templateTopic,
- String templateTopicSample) {
- ArrayList result = new ArrayList();
- String[] splitTT = Mapping.splitTopicIncludingSeparatorAsArray(templateTopic);
- String[] splitTTS = Mapping.splitTopicIncludingSeparatorAsArray(templateTopicSample);
- if (splitTT.length != splitTTS.length) {
- result.add(ValidationError.TemplateTopic_And_TemplateTopicSample_Do_Not_Have_Same_Number_Of_Levels_In_Topic_Name);
- } else {
- for (int i = 0; i < splitTT.length; i++) {
- if (("/").equals(splitTT[i]) && !("/").equals(splitTTS[i])) {
- result.add(ValidationError.TemplateTopic_And_TemplateTopicSample_Do_Not_Have_Same_Structure_In_Topic_Name);
- break;
+
+ /*
+ * test if mapping.mappingTopic and mapping.mappingTopicSample have the same
+ * structure and same number of levels
+ */
+ public static List isMappingTopicAndMappingTopicSampleValid(String mappingTopic,
+ String mappingTopicSample) {
+ ArrayList result = new ArrayList();
+ String[] splitTT = Mapping.splitTopicIncludingSeparatorAsArray(mappingTopic);
+ String[] splitTTS = Mapping.splitTopicIncludingSeparatorAsArray(mappingTopicSample);
+ if (splitTT.length != splitTTS.length) {
+ result.add(
+ ValidationError.MappingTopic_And_MappingTopicSample_Do_Not_Have_Same_Number_Of_Levels_In_Topic_Name);
+ } else {
+ for (int i = 0; i < splitTT.length; i++) {
+ if (("/").equals(splitTT[i]) && !("/").equals(splitTTS[i])) {
+ result.add(
+ ValidationError.MappingTopic_And_MappingTopicSample_Do_Not_Have_Same_Structure_In_Topic_Name);
+ break;
+ }
+ if (("/").equals(splitTTS[i]) && !("/").equals(splitTT[i])) {
+ result.add(
+ ValidationError.MappingTopic_And_MappingTopicSample_Do_Not_Have_Same_Structure_In_Topic_Name);
+ break;
+ }
+ if (!("/").equals(splitTT[i]) && !("+").equals(splitTT[i])) {
+ if (!splitTT[i].equals(splitTTS[i])) {
+ result.add(
+ ValidationError.MappingTopic_And_MappingTopicSample_Do_Not_Have_Same_Structure_In_Topic_Name);
+ break;
+ }
+ }
+ }
}
- if (("/").equals(splitTTS[i]) && !("/").equals(splitTT[i])) {
- result.add(ValidationError.TemplateTopic_And_TemplateTopicSample_Do_Not_Have_Same_Structure_In_Topic_Name);
- break;
+ return result;
+ }
+
+ private static Collection areJSONTemplatesValid(Mapping mapping) {
+ ArrayList result = new ArrayList();
+ try {
+ new JSONTokener(mapping.source).nextValue();
+ } catch (JSONException e) {
+ result.add(ValidationError.Source_Template_Must_Be_Valid_JSON);
}
- if (!("/").equals(splitTT[i]) && !("+").equals(splitTT[i])) {
- if (!splitTT[i].equals(splitTTS[i])) {
- result.add(ValidationError.TemplateTopic_And_TemplateTopicSample_Do_Not_Have_Same_Structure_In_Topic_Name);
- break;
- }
+
+ if (!mapping.mappingType.equals(MappingType.PROCESSOR_EXTENSION)
+ && !mapping.mappingType.equals(MappingType.PROTOBUF_STATIC)) {
+ try {
+ new JSONObject(mapping.target);
+ } catch (JSONException e) {
+ result.add(ValidationError.Target_Template_Must_Be_Valid_JSON);
+ }
}
- }
- }
- return result;
- }
-
- private static Collection areJSONTemplatesValid(Mapping mapping) {
- ArrayList result = new ArrayList();
- try {
- new JSONTokener(mapping.source).nextValue();
- } catch (JSONException e) {
- result.add(ValidationError.Source_Template_Must_Be_Valid_JSON);
+
+ return result;
}
- if (!mapping.mappingType.equals(MappingType.PROCESSOR_EXTENSION)
- && !mapping.mappingType.equals(MappingType.PROTOBUF_STATIC)) {
- try {
- new JSONObject(mapping.target);
- } catch (JSONException e) {
- result.add(ValidationError.Target_Template_Must_Be_Valid_JSON);
- }
+ static public String normalizeTopic(String topic) {
+ if (topic == null)
+ topic = "";
+ // reduce multiple leading or trailing "/" to just one "/"
+ String nt = topic.trim().replaceAll(REGEXP_REDUCE_LEADING_TRAILING_SLASHES, "/");
+ // do not use starting slashes, see as well
+ // https://www.hivemq.com/blog/mqtt-essentials-part-5-mqtt-topics-best-practices/
+ nt = nt.replaceAll(REGEXP_REMOVE_TRAILING_SLASHES, "#");
+ return nt;
}
- return result;
- }
-
- static public String normalizeTopic(String topic) {
- if (topic == null)
- topic = "";
- // reduce multiple leading or trailing "/" to just one "/"
- String nt = topic.trim().replaceAll(REGEXP_REDUCE_LEADING_TRAILING_SLASHES, "/");
- // do not use starting slashes, see as well
- // https://www.hivemq.com/blog/mqtt-essentials-part-5-mqtt-topics-best-practices/
- nt = nt.replaceAll(REGEXP_REMOVE_TRAILING_SLASHES, "#");
- return nt;
- }
-
- static public MappingSubstitution findDeviceIdentifier(Mapping mapping) {
- Object[] mp = Arrays.stream(mapping.substitutions)
- .filter(sub -> sub.definesDeviceIdentifier(mapping.targetAPI, mapping.direction)).toArray();
- if (mp.length > 0) {
- return (MappingSubstitution) mp[0];
- } else {
- return null;
+ static public MappingSubstitution findDeviceIdentifier(Mapping mapping) {
+ Object[] mp = Arrays.stream(mapping.substitutions)
+ .filter(sub -> sub.definesDeviceIdentifier(mapping.targetAPI, mapping.direction)).toArray();
+ if (mp.length > 0) {
+ return (MappingSubstitution) mp[0];
+ } else {
+ return null;
+ }
}
- }
}
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/model/TreeNode.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/model/MappingTreeNode.java
similarity index 78%
rename from dynamic-mapping-service/src/main/java/dynamic/mapping/model/TreeNode.java
rename to dynamic-mapping-service/src/main/java/dynamic/mapping/model/MappingTreeNode.java
index 98ea74ae..54b31566 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/model/TreeNode.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/model/MappingTreeNode.java
@@ -40,9 +40,9 @@
@ToString
@Setter
@Getter
-public class TreeNode {
+public class MappingTreeNode {
- private Map> childNodes;
+ private Map> childNodes;
private Mapping mapping;
@@ -51,7 +51,7 @@ public class TreeNode {
private long depthIndex;
@ToString.Exclude
- private TreeNode parentNode;
+ private MappingTreeNode parentNode;
private String absolutePath;
@@ -59,8 +59,8 @@ public class TreeNode {
private String tenant;
- static public TreeNode createRootNode(String tenant) {
- TreeNode in = new TreeNode();
+ static public MappingTreeNode createRootNode(String tenant) {
+ MappingTreeNode in = new MappingTreeNode();
in.setDepthIndex(0);
in.setLevel("root");
in.setTenant(tenant);
@@ -70,8 +70,8 @@ static public TreeNode createRootNode(String tenant) {
return in;
}
- public static TreeNode createInnerNode(TreeNode parent, String level) {
- TreeNode node = new TreeNode();
+ public static MappingTreeNode createInnerNode(MappingTreeNode parent, String level) {
+ MappingTreeNode node = new MappingTreeNode();
node.setParentNode(parent);
node.setLevel(level);
node.setTenant(parent.getTenant());
@@ -81,8 +81,8 @@ public static TreeNode createInnerNode(TreeNode parent, String level) {
return node;
}
- public static TreeNode createMappingNode(TreeNode parent, String level, Mapping mapping) {
- TreeNode node = new TreeNode();
+ public static MappingTreeNode createMappingNode(MappingTreeNode parent, String level, Mapping mapping) {
+ MappingTreeNode node = new MappingTreeNode();
node.setParentNode(parent);
node.setMapping(mapping);
node.setLevel(level);
@@ -94,39 +94,48 @@ public static TreeNode createMappingNode(TreeNode parent, String level, Mapping
return node;
}
- public TreeNode() {
- this.childNodes = new HashMap>();
+ public MappingTreeNode() {
+ this.childNodes = new HashMap>();
}
- public List resolveTopicPath(List remainingLevels) throws ResolveException {
+ public List resolveTopicPath(List remainingLevels) throws ResolveException {
Set set = childNodes.keySet();
String joinedSet = String.join(",", set);
String joinedPath = String.join("", remainingLevels);
log.debug("Tenant {} - Trying to resolve: '{}' in [{}]", tenant, joinedPath, joinedSet);
- List results = new ArrayList();
+ List results = new ArrayList();
if (remainingLevels.size() >= 1) {
String currentLevel = remainingLevels.get(0);
remainingLevels.remove(0);
if (childNodes.containsKey(currentLevel)) {
- List revolvedNodes = childNodes.get(currentLevel);
- for (TreeNode node : revolvedNodes) {
+ List revolvedNodes = childNodes.get(currentLevel);
+ for (MappingTreeNode node : revolvedNodes) {
results.addAll(node.resolveTopicPath(remainingLevels));
}
}
if (childNodes.containsKey(MappingRepresentation.TOPIC_WILDCARD_SINGLE)) {
- List revolvedNodes = childNodes.get(MappingRepresentation.TOPIC_WILDCARD_SINGLE);
- for (TreeNode node : revolvedNodes) {
+ List revolvedNodes = childNodes.get(MappingRepresentation.TOPIC_WILDCARD_SINGLE);
+ for (MappingTreeNode node : revolvedNodes) {
results.addAll(node.resolveTopicPath(remainingLevels));
}
// test if single level wildcard "+" match exists for this level
}
+ else if (childNodes.containsKey(MappingRepresentation.TOPIC_WILDCARD_MULTI)) {
+ List revolvedNodes = childNodes.get(MappingRepresentation.TOPIC_WILDCARD_MULTI);
+ for (MappingTreeNode node : revolvedNodes) {
+ results.addAll(node.resolveTopicPath(remainingLevels));
+ }
+ // test if single level wildcard "+" match exists for this level
+
+ }
} else if (remainingLevels.size() == 0) {
if (isMappingNode()) {
results.add(this);
} else {
String remaining = String.join("/", remainingLevels);
throw new ResolveException(
- "No mapping registered for this path: " + this.getAbsolutePath() + remaining + "!");
+ String.format("No mapping registered for this path: %s %s!", this.getAbsolutePath(),
+ remaining));
}
}
return results;
@@ -134,14 +143,14 @@ public List resolveTopicPath(List remainingLevels) throws Reso
public void addMapping(Mapping mapping, List levels, int currentLevel)
throws ResolveException {
- List specificChildren = getChildNodes().getOrDefault(levels.get(currentLevel),
- new ArrayList());
+ List specificChildren = getChildNodes().getOrDefault(levels.get(currentLevel),
+ new ArrayList());
String currentPathMonitoring = createPathMonitoring(levels, currentLevel);
if (currentLevel == levels.size() - 1) {
log.debug(
"Tenant {} - Adding mappingNode : currentPathMonitoring: {}, currentNode.absolutePath: {}, mappingId : {}",
tenant, currentPathMonitoring, getAbsolutePath(), mapping.id);
- TreeNode child = TreeNode.createMappingNode(this, levels.get(currentLevel), mapping);
+ MappingTreeNode child = MappingTreeNode.createMappingNode(this, levels.get(currentLevel), mapping);
log.debug("Tenant {} - Adding mappingNode : currentPathMonitoring {}, child: {}", tenant,
currentPathMonitoring,
child.toString());
@@ -151,23 +160,23 @@ public void addMapping(Mapping mapping, List levels, int currentLevel)
log.debug(
"Tenant {} - Adding innerNode : currentPathMonitoring: {}, currentNode.absolutePath: {}",
tenant, currentPathMonitoring, getLevel(), getAbsolutePath());
- TreeNode child;
+ MappingTreeNode child;
if (getChildNodes().containsKey(levels.get(currentLevel))) {
if (specificChildren.size() == 1) {
if (!specificChildren.get(0).isMappingNode()) {
child = specificChildren.get(0);
} else {
- throw new ResolveException(
- "Could not add mapping to tree, since at this node is already blocked by mappingId : "
- + specificChildren.get(0).toString());
+ throw new ResolveException(String.format(
+ "Could not add mapping to tree, since at this node is already blocked by mappingId : %s",
+ specificChildren.get(0).toString()));
}
} else {
- throw new ResolveException(
- "Could not add mapping to tree, multiple mappings are only allowed at the end of the tree. This node already contains: "
- + specificChildren.size() + " nodes");
+ throw new ResolveException(String.format(
+ "Could not add mapping to tree, multiple mappings are only allowed at the end of the tree. This node already contains: %s nodes",
+ specificChildren.size()));
}
} else {
- child = TreeNode.createInnerNode(this, levels.get(currentLevel));
+ child = MappingTreeNode.createInnerNode(this, levels.get(currentLevel));
log.debug("Tenant {} - Adding innerNode: currentPathMonitoring: {}, child: {}, {}", tenant,
currentPathMonitoring,
child.toString());
@@ -176,14 +185,14 @@ public void addMapping(Mapping mapping, List levels, int currentLevel)
}
child.addMapping(mapping, levels, currentLevel + 1);
} else {
- throw new ResolveException("Could not add mapping to tree: " + mapping.toString());
+ throw new ResolveException(String.format("Could not add mapping to tree: %s", mapping.toString()));
}
}
public void addMapping(Mapping mapping) throws ResolveException {
if (mapping != null) {
- var path = mapping.templateTopic;
- // if templateTopic is not set use topic instead
+ var path = mapping.mappingTopic;
+ // if mappingTopic is not set use topic instead
if (path == null || path.equals("")) {
path = mapping.subscriptionTopic;
}
@@ -194,8 +203,8 @@ public void addMapping(Mapping mapping) throws ResolveException {
public void deleteMapping(Mapping mapping) throws ResolveException {
if (mapping != null) {
- var path = mapping.templateTopic;
- // if templateTopic is not set use topic instead
+ var path = mapping.mappingTopic;
+ // if mappingTopic is not set use topic instead
if (path == null || path.equals("")) {
path = mapping.subscriptionTopic;
}
@@ -221,7 +230,7 @@ private boolean deleteMapping(Mapping mapping, List levels, int currentL
tn.getValue().removeIf(tnn -> {
if (tnn.isMappingNode()) {
if (tnn.getMapping().id.equals(mapping.id)) {
- log.info(
+ log.debug(
"Tenant {} - Deleting mappingNode : currentPathMonitoring: {}, branchingLevel: {}, mappingId: {}",
tenant,
currentPathMonitoring, branchingLevel, mapping.id);
@@ -245,7 +254,7 @@ private boolean deleteMapping(Mapping mapping, List levels, int currentL
tenant,
currentPathMonitoring, branchingLevel);
if (getChildNodes().containsKey(levels.get(currentLevel))) {
- List tns = getChildNodes().get(levels.get(currentLevel));
+ List tns = getChildNodes().get(levels.get(currentLevel));
tns.removeIf(tn -> {
boolean bm = false;
if (!tn.isMappingNode() && !foundMapping.booleanValue()) {
@@ -262,7 +271,7 @@ private boolean deleteMapping(Mapping mapping, List levels, int currentL
currentPathMonitoring, branchingLevel, e.getMessage());
}
if (currentLevel < branchingLevel.getValue()) {
- log.info(
+ log.debug(
"Tenant {} - Deleting innerNode stopped: currentPathMonitoring: {}, branchingLevel: {}",
tenant,
currentPathMonitoring, branchingLevel);
@@ -270,7 +279,7 @@ private boolean deleteMapping(Mapping mapping, List levels, int currentL
}
}
if (bm) {
- log.info(
+ log.debug(
"Tenant {} - Deleting innerNode : currentPathMonitoring: {}, branchingLevel: {}",
tenant,
currentPathMonitoring, branchingLevel);
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/model/TreeNodeSerializer.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/model/MappingTreeNodeSerializer.java
similarity index 88%
rename from dynamic-mapping-service/src/main/java/dynamic/mapping/model/TreeNodeSerializer.java
rename to dynamic-mapping-service/src/main/java/dynamic/mapping/model/MappingTreeNodeSerializer.java
index 3cda3697..d592899b 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/model/TreeNodeSerializer.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/model/MappingTreeNodeSerializer.java
@@ -30,19 +30,19 @@
import java.io.IOException;
@Slf4j
-public class TreeNodeSerializer extends StdSerializer {
+public class MappingTreeNodeSerializer extends StdSerializer {
- public TreeNodeSerializer() {
+ public MappingTreeNodeSerializer() {
this(null);
}
- public TreeNodeSerializer(Class t) {
+ public MappingTreeNodeSerializer(Class t) {
super(t);
}
@Override
public void serialize(
- TreeNode value, JsonGenerator jgen, SerializerProvider provider)
+ MappingTreeNode value, JsonGenerator jgen, SerializerProvider provider)
throws IOException, JsonProcessingException {
log.debug("Serializing node {}, {}", value.getLevel(), value.getAbsolutePath());
jgen.writeStartObject();
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/model/ValidationError.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/model/ValidationError.java
index 682bf27d..65ba8a0d 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/model/ValidationError.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/model/ValidationError.java
@@ -27,16 +27,16 @@ public enum ValidationError {
Multi_Level_Wildcard_Only_At_End,
Only_One_Substitution_Defining_Device_Identifier_Can_Be_Used,
One_Substitution_Defining_Device_Identifier_Must_Be_Used,
- TemplateTopic_Must_Match_The_SubscriptionTopic,
- TemplateTopic_Not_Unique,
- TemplateTopic_Must_Not_Be_Substring_Of_Other_TemplateTopic,
+ MappingTopic_Must_Match_The_SubscriptionTopic,
+ MappingTopic_Not_Unique,
+ MappingTopic_Must_Not_Be_Substring_Of_Other_MappingTopic,
Target_Template_Must_Be_Valid_JSON,
Source_Template_Must_Be_Valid_JSON,
- No_Multi_Level_Wildcard_Allowed_In_TemplateTopic,
+ No_Multi_Level_Wildcard_Allowed_In_MappingTopic,
Device_Identifier_Must_Be_Selected,
- TemplateTopic_And_TemplateTopicSample_Do_Not_Have_Same_Number_Of_Levels_In_Topic_Name,
- TemplateTopic_And_TemplateTopicSample_Do_Not_Have_Same_Structure_In_Topic_Name,
- PublishTopic_And_TemplateTopicSample_Do_Not_Have_Same_Number_Of_Levels_In_Topic_Name,
- PublishTopic_And_TemplateTopicSample_Do_Not_Have_Same_Structure_In_Topic_Name,
+ MappingTopic_And_MappingTopicSample_Do_Not_Have_Same_Number_Of_Levels_In_Topic_Name,
+ MappingTopic_And_MappingTopicSample_Do_Not_Have_Same_Structure_In_Topic_Name,
+ PublishTopic_And_PublishTopicSample_Do_Not_Have_Same_Number_Of_Levels_In_Topic_Name,
+ PublishTopic_And_PublishTopicSample_Do_Not_Have_Same_Structure_In_Topic_Name,
FilterOutbound_Must_Be_Unique
}
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/notification/C8YNotificationSubscriber.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/notification/C8YNotificationSubscriber.java
index 4ab208bc..ac24d279 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/notification/C8YNotificationSubscriber.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/notification/C8YNotificationSubscriber.java
@@ -22,7 +22,6 @@
package dynamic.mapping.notification;
import com.cumulocity.microservice.subscription.service.MicroserviceSubscriptionsService;
-import com.cumulocity.model.JSONBase;
import com.cumulocity.model.idtype.GId;
import com.cumulocity.rest.representation.inventory.ManagedObjectRepresentation;
import com.cumulocity.rest.representation.reliable.notification.NotificationSubscriptionFilterRepresentation;
@@ -43,7 +42,6 @@
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import dynamic.mapping.model.API;
-import dynamic.mapping.notification.websocket.Notification;
import org.apache.commons.collections.ArrayStack;
import org.java_websocket.enums.ReadyState;
import org.springframework.beans.factory.annotation.Autowired;
@@ -99,7 +97,6 @@ public void setConfigurationRegistry(@Lazy ConfigurationRegistry configurationRe
private ScheduledExecutorService executorService = null;
private final String DEVICE_SUBSCRIBER = "DynamicMapperDeviceSubscriber";
private final String DEVICE_SUBSCRIPTION = "DynamicMapperDeviceSubscription";
- private final String TENANT_SUBSCRIBER = "DynamicMapperTenantSubscriber";
private final String TENANT_SUBSCRIPTION = "DynamicMapperTenantSubscription";
private Map> deviceClientMap = new HashMap<>();
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/extension/ExtensibleProcessorInbound.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/extension/ExtensibleProcessorInbound.java
index 9ce4d5a9..6f4dc86d 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/extension/ExtensibleProcessorInbound.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/extension/ExtensibleProcessorInbound.java
@@ -64,18 +64,14 @@ public void extractFromSource(ProcessingContext context)
String message = String.format("Tenant %s - Extension %s:%s could not be found!", tenant,
context.getMapping().extension.getName(),
context.getMapping().extension.getEvent());
- log.warn("Tenant {} - Extension {}:{} could not be found!", tenant,
- context.getMapping().extension.getName(),
- context.getMapping().extension.getEvent());
+ log.warn(message);
throw new ProcessingException(message);
}
} catch (Exception ex) {
String message = String.format("Tenant %s - Extension %s:%s could not be found!", tenant,
context.getMapping().extension.getName(),
context.getMapping().extension.getEvent());
- log.warn("Tenant {} - Extension {}:{} could not be found!", tenant,
- context.getMapping().extension.getName(),
- context.getMapping().extension.getEvent());
+ log.warn(message);
throw new ProcessingException(message);
}
extension.extractFromSource(context);
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/inbound/AsynchronousDispatcherInbound.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/inbound/AsynchronousDispatcherInbound.java
index 1c5e2c37..c852aee3 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/inbound/AsynchronousDispatcherInbound.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/inbound/AsynchronousDispatcherInbound.java
@@ -78,10 +78,12 @@ public class AsynchronousDispatcherInbound implements GenericMessageCallback {
private ConfigurationRegistry configurationRegistry;
- public AsynchronousDispatcherInbound(ConfigurationRegistry configurationRegistry, AConnectorClient connectorClient) {
+ public AsynchronousDispatcherInbound(ConfigurationRegistry configurationRegistry,
+ AConnectorClient connectorClient) {
this.connectorClient = connectorClient;
this.cachedThreadPool = configurationRegistry.getCachedThreadPool();
- this.mappingComponent = configurationRegistry.getMappingComponent();;
+ this.mappingComponent = configurationRegistry.getMappingComponent();
+ ;
this.configurationRegistry = configurationRegistry;
}
@@ -131,6 +133,8 @@ public List> call() throws Exception {
context.setMapping(mapping);
context.setSendPayload(sendPayload);
context.setTenant(tenant);
+ context.setSupportsMessageContext(connectorMessage.isSupportsMessageContext() && mapping.supportsMessageContext);
+ context.setKey(connectorMessage.getKey());
context.setServiceConfiguration(serviceConfiguration);
// identify the correct processor based on the mapping type
MappingType mappingType = context.getMappingType();
@@ -139,7 +143,7 @@ public List> call() throws Exception {
if (processor != null) {
try {
processor.deserializePayload(context, connectorMessage);
- if (serviceConfiguration.logPayload) {
+ if (serviceConfiguration.logPayload || mapping.debug) {
log.info("Tenant {} - New message on topic: '{}', wrapped message: {}", tenant,
context.getTopic(),
context.getPayload().toString());
@@ -188,7 +192,7 @@ public List> call() throws Exception {
} catch (Exception e) {
log.warn("Tenant {} - Message could NOT be parsed, ignoring this message: {}", tenant,
e.getMessage());
- log.info("Tenant {} - Message Stacktrace: ", tenant, e);
+ log.debug("Tenant {} - Message Stacktrace: ", tenant, e);
mappingStatus.errors++;
}
} else {
@@ -203,7 +207,7 @@ public List> call() throws Exception {
}
}
- public Future>> processMessage(ConnectorMessage message) throws Exception {
+ public Future>> processMessage(ConnectorMessage message) {
String topic = message.getTopic();
String tenant = message.getTenant();
@@ -216,7 +220,9 @@ public Future>> processMessage(ConnectorMessage messag
try {
resolvedMappings = mappingComponent.resolveMappingInbound(tenant, topic);
} catch (Exception e) {
- log.warn("Tenant {} - Error resolving appropriate map for topic {}. Could NOT be parsed. Ignoring this message!", tenant, topic);
+ log.warn(
+ "Tenant {} - Error resolving appropriate map for topic {}. Could NOT be parsed. Ignoring this message!",
+ tenant, topic);
log.debug(e.getMessage(), e);
mappingStatusUnspecified.errors++;
}
@@ -237,19 +243,10 @@ public Future>> processMessage(ConnectorMessage messag
@Override
public void onClose(String closeMessage, Throwable closeException) {
- String tenant = connectorClient.getTenant();
- String connectorIdent = connectorClient.getConnectorIdent();
- if (closeException != null)
- log.error("Tenant {} - Connection Lost to broker {}: {}", tenant, connectorIdent,
- closeException.getMessage());
- closeException.printStackTrace();
- if (closeMessage != null)
- log.info("Tenant {} - Connection Lost to MQTT broker: {}", tenant, closeMessage);
- connectorClient.reconnect();
}
@Override
- public void onMessage(ConnectorMessage message) throws Exception {
+ public void onMessage(ConnectorMessage message) {
processMessage(message);
}
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/inbound/BasePayloadProcessorInbound.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/inbound/BasePayloadProcessorInbound.java
index 139b84ef..de6be225 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/inbound/BasePayloadProcessorInbound.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/inbound/BasePayloadProcessorInbound.java
@@ -35,7 +35,6 @@
import dynamic.mapping.model.Mapping;
import dynamic.mapping.model.MappingSubstitution;
import lombok.extern.slf4j.Slf4j;
-import dynamic.mapping.configuration.ServiceConfiguration;
import dynamic.mapping.connector.core.callback.ConnectorMessage;
import dynamic.mapping.core.C8YAgent;
import dynamic.mapping.core.ConfigurationRegistry;
@@ -50,7 +49,6 @@
import org.springframework.web.bind.annotation.RequestMethod;
import java.io.IOException;
-import java.text.MessageFormat;
import java.util.*;
import java.util.Map.Entry;
@@ -78,7 +76,7 @@ public ProcessingContext substituteInTargetAndSend(ProcessingContext conte
Mapping mapping = context.getMapping();
String tenant = context.getTenant();
- // if there are to little device idenfified then we replicate the first device
+ // if there are too few devices identified then we replicate the first device
Map> postProcessingCache = context.getPostProcessingCache();
String maxEntry = postProcessingCache.entrySet()
.stream()
@@ -139,6 +137,7 @@ public ProcessingContext substituteInTargetAndSend(ProcessingContext conte
"device_" + mapping.externalIdType + "_" + substituteValue.value.asText());
request.put(MappingRepresentation.MAPPING_GENERATED_TEST_DEVICE, null);
request.put("c8y_IsDevice", null);
+ request.put("com_cumulocity_model_Agent", null);
try {
var requestString = objectMapper.writeValueAsString(request);
var newPredecessor = context.addRequest(
@@ -154,9 +153,9 @@ public ProcessingContext substituteInTargetAndSend(ProcessingContext conte
context.getCurrentRequest().setError(e);
}
} else if (sourceId == null && context.isSendPayload()) {
- throw new RuntimeException(
- "External id " + substituteValue.typedValue().toString() + " for type "
- + mapping.externalIdType + " not found!");
+ throw new RuntimeException(String.format(
+ "External id %s for type %s not found!", substituteValue.typedValue().toString(),
+ mapping.externalIdType));
} else if (sourceId == null) {
substituteValue.value = null;
} else {
@@ -220,15 +219,6 @@ public void substituteValueInObject(MappingType type, MappingSubstitution.Substi
throws JSONException {
boolean subValueMissing = sub.value == null;
boolean subValueNull = (sub.value == null) || (sub.value != null && sub.value.isNull());
- // variant where the default strategy for PROCESSOR_EXTENSION is
- // REMOVE_IF_MISSING
- // if ((sub.repairStrategy.equals(RepairStrategy.REMOVE_IF_MISSING) &&
- // subValueMissing) ||
- // (sub.repairStrategy.equals(RepairStrategy.REMOVE_IF_NULL) && subValueNull) ||
- // ((type.equals(MappingType.PROCESSOR_EXTENSION) ||
- // type.equals(MappingType.PROTOBUF_STATIC))
- // && (subValueMissing || subValueNull)))
-
try {
if ("$".equals(keys)) {
Object replacement = sub.typedValue();
@@ -253,7 +243,7 @@ public void substituteValueInObject(MappingType type, MappingSubstitution.Substi
}
}
} catch (PathNotFoundException e) {
- throw new PathNotFoundException(MessageFormat.format("Path: \"{0}\" not found!", keys));
+ throw new PathNotFoundException(String.format("Path: %s not found!", keys));
}
}
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/inbound/JSONProcessorInbound.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/inbound/JSONProcessorInbound.java
index 520f17f3..2cd16e35 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/inbound/JSONProcessorInbound.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/inbound/JSONProcessorInbound.java
@@ -42,6 +42,7 @@
import org.joda.time.DateTime;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -80,13 +81,22 @@ public void extractFromSource(ProcessingContext context)
splitTopicAsList.forEach(s -> topicLevels.add(s));
if (payloadJsonNode instanceof ObjectNode) {
((ObjectNode) payloadJsonNode).set(Mapping.TOKEN_TOPIC_LEVEL, topicLevels);
+ if (context.isSupportsMessageContext() && context.getKey() != null) {
+ ObjectNode contextData = objectMapper.createObjectNode();
+ String keyString = new String(context.getKey(), StandardCharsets.UTF_8);
+ contextData.put(Mapping.CONTEXT_DATA_KEY_NAME, keyString);
+ ((ObjectNode) payloadJsonNode).set(Mapping.TOKEN_CONTEXT_DATA, contextData);
+ }
} else {
log.warn("Tenant {} - Parsing this message as JSONArray, no elements from the topic level can be used!",
tenant);
}
String payload = payloadJsonNode.toPrettyString();
- log.info("Tenant {} - Patched payload: {}", tenant, payload);
+ if (serviceConfiguration.logPayload || mapping.debug) {
+ log.info("Tenant {} - Patched payload: {} {} {} {}", tenant, payload, serviceConfiguration.logPayload,
+ mapping.debug, serviceConfiguration.logPayload || mapping.debug);
+ }
boolean substitutionTimeExists = false;
for (MappingSubstitution substitution : mapping.substitutions) {
@@ -114,8 +124,7 @@ public void extractFromSource(ProcessingContext context)
substitution.pathTarget,
new ArrayList());
if (extractedSourceContent == null) {
- log.error("Tenant {} - No substitution for: {}, {}", tenant, substitution.pathSource,
- payload);
+ log.warn("Tenant {} - Substitution {} not in message payload. Check your mapping {}", tenant, substitution.pathSource, mapping.getMappingTopic());
postProcessingCacheEntry
.add(new MappingSubstitution.SubstituteValue(extractedSourceContent,
MappingSubstitution.SubstituteValue.TYPE.IGNORE, substitution.repairStrategy));
@@ -184,7 +193,7 @@ public void extractFromSource(ProcessingContext context)
MappingSubstitution.SubstituteValue.TYPE.OBJECT, substitution.repairStrategy));
postProcessingCache.put(substitution.pathTarget, postProcessingCacheEntry);
}
- if (serviceConfiguration.logSubstitution) {
+ if (serviceConfiguration.logSubstitution || mapping.debug) {
log.info("Tenant {} - Evaluated substitution (pathSource:substitute)/({}:{}), (pathTarget)/({})",
tenant,
substitution.pathSource, extractedSourceContent.toString(), substitution.pathTarget);
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/model/ProcessingContext.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/model/ProcessingContext.java
index 74462608..1474bbc4 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/model/ProcessingContext.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/model/ProcessingContext.java
@@ -24,6 +24,7 @@
import dynamic.mapping.configuration.ServiceConfiguration;
import dynamic.mapping.model.Mapping;
import dynamic.mapping.model.MappingSubstitution;
+import dynamic.mapping.model.QOS;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@@ -49,6 +50,8 @@ public class ProcessingContext {
private String topic;
+ private QOS qos;
+
private String resolvedPublishTopic;
private O payload;
@@ -75,6 +78,12 @@ public class ProcessingContext {
private ServiceConfiguration serviceConfiguration;
+ private boolean supportsMessageContext = false;
+
+ private byte[] key;
+
+ private String source;
+
public static final String SOURCE_ID = "source.id";
public boolean hasError() {
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/outbound/AsynchronousDispatcherOutbound.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/outbound/AsynchronousDispatcherOutbound.java
index 43bbb879..a039e70f 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/outbound/AsynchronousDispatcherOutbound.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/outbound/AsynchronousDispatcherOutbound.java
@@ -100,16 +100,17 @@ public class AsynchronousDispatcherOutbound implements NotificationCallback {
// The Outbound Dispatcher is hardly connected to the Connector otherwise it is
// not possible to correlate messages received bei Notification API to the
// correct Connector
- public AsynchronousDispatcherOutbound(ConfigurationRegistry configurationRegistry, AConnectorClient connectorClient) {
+ public AsynchronousDispatcherOutbound(ConfigurationRegistry configurationRegistry,
+ AConnectorClient connectorClient) {
this.objectMapper = configurationRegistry.getObjectMapper();
this.c8yAgent = configurationRegistry.getC8yAgent();
this.mappingComponent = configurationRegistry.getMappingComponent();
this.cachedThreadPool = configurationRegistry.getCachedThreadPool();
this.connectorClient = connectorClient;
// log.info("Tenant {} - HIER I {} {}", connectorClient.getTenant(),
- // configurationRegistry.getPayloadProcessorsOutbound());
+ // configurationRegistry.getPayloadProcessorsOutbound());
// log.info("Tenant {} - HIER II {} {}", connectorClient.getTenant(),
- // configurationRegistry.getPayloadProcessorsOutbound().get(connectorClient.getTenant()));
+ // configurationRegistry.getPayloadProcessorsOutbound().get(connectorClient.getTenant()));
this.payloadProcessorsOutbound = configurationRegistry.getPayloadProcessorsOutbound()
.get(connectorClient.getTenant())
.get(connectorClient.getConnectorIdent());
@@ -127,11 +128,13 @@ public void onOpen(URI serverUri) {
@Override
public void onNotification(Notification notification) {
- // We don't care about UPDATES nor DELETES
- if ("CREATE".equals(notification.getNotificationHeaders().get(1))) {
+ // We don't care about UPDATES nor DELETES and ignore notifications if connector
+ // is not connected
+ if ("CREATE".equals(notification.getNotificationHeaders().get(1)) && connectorClient.isConnected()) {
String tenant = getTenantFromNotificationHeaders(notification.getNotificationHeaders());
- log.info("Tenant {} - Notification received: <{}>", tenant, notification.getMessage());
- log.info("Tenant {} - Notification headers: <{}>", tenant, notification.getNotificationHeaders());
+ log.info("Tenant {} - Notification received: <{}>, <{}>, <{}>, <{}>", tenant, notification.getMessage(),
+ notification.getNotificationHeaders(), connectorClient.connectorConfiguration.name,
+ connectorClient.isConnected());
C8YMessage c8yMessage = new C8YMessage();
c8yMessage.setPayload(notification.getMessage());
c8yMessage.setApi(notification.getApi());
@@ -203,8 +206,10 @@ public List> call() throws Exception {
context.setTopic(mapping.publishTopic);
context.setMappingType(mapping.mappingType);
context.setMapping(mapping);
+ context.setSupportsMessageContext(mapping.supportsMessageContext);;
context.setSendPayload(sendPayload);
context.setTenant(tenant);
+ context.setQos(mapping.getQos());
context.setServiceConfiguration(serviceConfiguration);
// identify the correct processor based on the mapping type
MappingType mappingType = context.getMappingType();
@@ -213,13 +218,14 @@ public List> call() throws Exception {
if (processor != null) {
try {
processor.deserializePayload(context, c8yMessage);
- if (serviceConfiguration.logPayload) {
+ if (serviceConfiguration.logPayload || mapping.debug) {
log.info("Tenant {} - New message on topic: '{}', wrapped message: {}",
tenant,
context.getTopic(),
context.getPayload().toString());
} else {
- log.info("Tenant {} - New message on topic: '{}', sendPayload: {}", tenant, context.getTopic(), sendPayload);
+ log.info("Tenant {} - New message on topic: '{}', sendPayload: {}", tenant,
+ context.getTopic(), sendPayload);
}
mappingStatus.messagesReceived++;
if (mapping.snoopStatus == SnoopStatus.ENABLED
@@ -263,7 +269,7 @@ public List> call() throws Exception {
} catch (Exception e) {
log.warn("Tenant {} - Message could NOT be parsed, ignoring this message: {}", tenant,
e.getMessage());
- log.debug("Tenant {} - Message Stacktrace: ", tenant, e);
+ log.error("Tenant {} - Message Stacktrace: ", tenant, e);
mappingStatus.errors++;
}
} else {
@@ -296,14 +302,15 @@ public Future>> processMessage(C8YMessage c8yMessage)
try {
JsonNode message = objectMapper.readTree(c8yMessage.getPayload());
resolvedMappings = mappingComponent.resolveMappingOutbound(tenant, message, c8yMessage.getApi());
- if(resolvedMappings.size() > 0 && op != null)
+ if (resolvedMappings.size() > 0 && op != null)
c8yAgent.updateOperationStatus(tenant, op, OperationStatus.EXECUTING, null);
} catch (Exception e) {
log.warn("Tenant {} - Error resolving appropriate map. Could NOT be parsed. Ignoring this message!",
tenant);
log.debug(e.getMessage(), tenant);
- //if (op != null)
- // c8yAgent.updateOperationStatus(tenant, op, OperationStatus.FAILED, e.getLocalizedMessage());
+ // if (op != null)
+ // c8yAgent.updateOperationStatus(tenant, op, OperationStatus.FAILED,
+ // e.getLocalizedMessage());
mappingStatusUnspecified.errors++;
}
} else {
@@ -328,13 +335,15 @@ public Future>> processMessage(C8YMessage c8yMessage)
}
} else {
// No Mapping found
- //c8yAgent.updateOperationStatus(tenant, op, OperationStatus.FAILED,
- // "No Mapping found for operation " + op.toJSON());
+ // c8yAgent.updateOperationStatus(tenant, op, OperationStatus.FAILED,
+ // "No Mapping found for operation " + op.toJSON());
}
} catch (InterruptedException e) {
- //c8yAgent.updateOperationStatus(tenant, op, OperationStatus.FAILED, e.getLocalizedMessage());
+ // c8yAgent.updateOperationStatus(tenant, op, OperationStatus.FAILED,
+ // e.getLocalizedMessage());
} catch (ExecutionException e) {
- //c8yAgent.updateOperationStatus(tenant, op, OperationStatus.FAILED, e.getLocalizedMessage());
+ // c8yAgent.updateOperationStatus(tenant, op, OperationStatus.FAILED,
+ // e.getLocalizedMessage());
}
}
return futureProcessingResult;
diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/outbound/BasePayloadProcessorOutbound.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/outbound/BasePayloadProcessorOutbound.java
index cc580aae..5a2a6f30 100644
--- a/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/outbound/BasePayloadProcessorOutbound.java
+++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/outbound/BasePayloadProcessorOutbound.java
@@ -30,6 +30,7 @@
import dynamic.mapping.model.Mapping;
import dynamic.mapping.model.MappingSubstitution;
import lombok.extern.slf4j.Slf4j;
+import dynamic.mapping.configuration.ServiceConfiguration;
import dynamic.mapping.connector.core.client.AConnectorClient;
import dynamic.mapping.core.C8YAgent;
import dynamic.mapping.core.ConfigurationRegistry;
@@ -45,7 +46,7 @@
import org.springframework.web.bind.annotation.RequestMethod;
import java.io.IOException;
-import java.text.MessageFormat;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -65,9 +66,6 @@ public BasePayloadProcessorOutbound(ConfigurationRegistry configurationRegistry,
protected AConnectorClient connectorClient;
- public static String TOKEN_DEVICE_TOPIC = "_DEVICE_IDENT_";
- public static String TOKEN_TOPIC_LEVEL = "_TOPIC_LEVEL_";
-
public abstract ProcessingContext deserializePayload(ProcessingContext context, C8YMessage c8yMessage)
throws IOException;
@@ -79,8 +77,8 @@ public ProcessingContext substituteInTargetAndSend(ProcessingContext conte
*/
Mapping mapping = context.getMapping();
String tenant = context.getTenant();
+ ServiceConfiguration serviceConfiguration = context.getServiceConfiguration();
- // if there are to little device idenfified then we replicate the first device
Map> postProcessingCache = context.getPostProcessingCache();
Set pathTargets = postProcessingCache.keySet();
@@ -91,9 +89,22 @@ public ProcessingContext substituteInTargetAndSend(ProcessingContext conte
* is required in the payload for a substitution
*/
List splitTopicExAsList = Mapping.splitTopicExcludingSeparatorAsList(context.getTopic());
- payloadTarget.set(Mapping.TOKEN_TOPIC_LEVEL, splitTopicExAsList);
+ payloadTarget.put("$", Mapping.TOKEN_TOPIC_LEVEL, splitTopicExAsList);
+ if (mapping.supportsMessageContext) {
+ Map cod = new HashMap() {
+ {
+ put(Mapping.CONTEXT_DATA_KEY_NAME, "dummy");
+ }
+ };
+ payloadTarget.put("$", Mapping.TOKEN_CONTEXT_DATA, cod);
+ }
+ if (serviceConfiguration.logPayload || mapping.debug) {
+ String patchedPayloadTarget = payloadTarget.jsonString();
+ log.info("Tenant {} - Patched payload: {} {} {} {}", tenant, patchedPayloadTarget,
+ serviceConfiguration.logPayload, mapping.debug, serviceConfiguration.logPayload || mapping.debug);
+ }
- String deviceSource = "undefined";
+ String deviceSource = context.getSource();
for (String pathTarget : pathTargets) {
MappingSubstitution.SubstituteValue substituteValue = new MappingSubstitution.SubstituteValue(
@@ -131,7 +142,14 @@ public ProcessingContext substituteInTargetAndSend(ProcessingContext conte
context.setResolvedPublishTopic(context.getMapping().getPublishTopic());
}
// remove TOPIC_LEVEL
- payloadTarget.delete(Mapping.TOKEN_TOPIC_LEVEL);
+ payloadTarget.delete("$." + Mapping.TOKEN_TOPIC_LEVEL);
+ if (mapping.supportsMessageContext) {
+ String key = payloadTarget
+ .read(String.format("$.%s.%s", Mapping.TOKEN_CONTEXT_DATA, Mapping.CONTEXT_DATA_KEY_NAME));
+ context.setKey(key.getBytes());
+ // remove TOKEN_CONTEXT_DATA
+ payloadTarget.delete("$." + Mapping.TOKEN_CONTEXT_DATA);
+ }
var newPredecessor = context.addRequest(
new C8YRequest(predecessor, RequestMethod.POST, deviceSource, mapping.externalIdType,
payloadTarget.jsonString(),
@@ -140,7 +158,7 @@ public ProcessingContext