TB-201 Update JeMPI Docker Images Environment Variables

client-registry-jempi/docker-compose.api.yml

@@ -4,17 +4,18 @@ services:
   jempi-api:
     image: jembi/jempi-api:${JEMPI_API_IMAGE_TAG}
     environment:
-      KC_REALM_NAME: ${KC_REALM_NAME}
-      KC_JEMPI_CLIENT_ID: ${KC_JEMPI_CLIENT_ID}
-      KC_JEMPI_CLIENT_SECRET: ${KC_JEMPI_CLIENT_SECRET}
-      KC_API_URL: ${KC_API_URL}
-      KC_JEMPI_ROOT_URL: ${KC_JEMPI_ROOT_URL}
-      JEMPI_SESSION_SECRET: ${JEMPI_SESSION_SECRET}
-      kafka.bootstrap.servers: ${KAFKA_HOSTS}
-      JEMPI_FILE_IMPORT_MAX_SIZE_BYTE: ${JEMPI_FILE_IMPORT_MAX_SIZE_BYTE}
-      JEMPI_SESSION_SECURE: ${JEMPI_SESSION_SECURE}
-      JEMPI_SESSION_DOMAIN_NAME: ${DOMAIN_NAME}
-      postgres.server: ${JEMPI_REPMGR_PARTNER_NODES}
+      POSTGRESQL_USER: ${POSTGRESQL_USERNAME} 
+      POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} 
+      POSTGRESQL_DATABASE: ${POSTGRESQL_DATABASE}
+      KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_HOSTS}
+      KAFKA_APPLICATION_ID: ${KAFKA_APPLICATION_ID_API}
+      DGRAPH_HOSTS: ${DGRAPH_HOSTS}
+      DGRAPH_PORTS: ${DGRAPH_PORTS}
+      HTTP_SERVER_PORT: ${HTTP_SERVER_PORT}
+      LOG4J2_LEVEL: ${LOG4J2_LEVEL}
+      LINKER_HOST: ${LINKER_HOST}
+    volumes:
+      - "jempi-shared-data:/app/csv"
     deploy:
       replicas: ${JEMPI_API_INSTANCES}
       resources:
@@ -24,18 +25,16 @@ services:
           memory: ${JEMPI_API_MEMORY_RESERVE}
     networks:
       reverse-proxy:
-      keycloak:
       kafka:
       default:
 
+volumes:
+  jempi-shared-data: 
 
 networks:
   reverse-proxy:
     name: reverse-proxy_public
     external: true
-  keycloak:
-    name: keycloak_public
-    external: true
   kafka:
     name: kafka_public
     external: true

client-registry-jempi/docker-compose.combined-dev.yml

@@ -1,12 +1,6 @@
 version: '3.9'
 
 services:
-  jempi-sync-receiver:
-    ports:
-      - published: 50040
-        target: 50000
-        protocol: tcp
-        mode: host
 
   jempi-controller:
     ports:

client-registry-jempi/docker-compose.combined.yml

@@ -4,7 +4,11 @@ services:
   jempi-async-receiver:
     image: jembi/jempi-async-receiver:${JEMPI_ASYNC_RECEIVER_IMAGE_TAG}
     environment:
-      kafka.bootstrap.servers: ${KAFKA_HOSTS}
+      LOG4J2_LEVEL: ${LOG4J2_LEVEL} 
+      KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_HOSTS}
+      KAFKA_CLIENT_ID: ${KAFKA_CLIENT_ID_ASYNC}
+    volumes:
+      - "jempi-shared-data:/app/csv"
     deploy:
       replicas: ${JEMPI_ASYNC_RECEIVER_INSTANCES}
       resources:
@@ -16,40 +20,36 @@ services:
       kafka:
       default:
 
-  jempi-sync-receiver:
-    image: jembi/jempi-sync-receiver:${JEMPI_SYNC_RECEIVER_IMAGE_TAG}
+  jempi-etl:
+    image: jembi/jempi-etl:${JEMPI_ETL_IMAGE_TAG}
     environment:
-      kafka.bootstrap.servers: ${KAFKA_HOSTS}
+      LOG4J2_LEVEL: ${LOG4J2_LEVEL} 
+      KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_HOSTS}
+      KAFKA_APPLICATION_ID: ${KAFKA_APPLICATION_ID_ETL}
     deploy:
-      replicas: ${JEMPI_SYNC_RECEIVER_INSTANCES}
+      replicas: ${JEMPI_ETL_INSTANCES}
       resources:
         limits:
-          memory: ${JEMPI_SYNC_RECEIVER_MEMORY_LIMIT}
+          memory: ${JEMPI_ETL_MEMORY_LIMIT}
         reservations:
-          memory: ${JEMPI_SYNC_RECEIVER_MEMORY_RESERVE}
-    networks:
-      kafka:
-      default:
-
-  jempi-pre-processor:
-    image: jembi/jempi-pre-processor:${JEMPI_PRE_PROCESSOR_IMAGE_TAG}
-    environment:
-      kafka.bootstrap.servers: ${KAFKA_HOSTS}
-    deploy:
-      replicas: ${JEMPI_PRE_PROCESSOR_INSTANCES}
-      resources:
-        limits:
-          memory: ${JEMPI_PRE_PROCESSOR_MEMORY_LIMIT}
-        reservations:
-          memory: ${JEMPI_PRE_PROCESSOR_MEMORY_RESERVE}
+          memory: ${JEMPI_ETL_MEMORY_RESERVE}
     networks:
       kafka:
       default:
 
   jempi-controller:
     image: jembi/jempi-controller:${JEMPI_CONTROLLER_IMAGE_TAG}
     environment:
-      kafka.bootstrap.servers: ${KAFKA_HOSTS}
+      POSTGRESQL_USER: ${POSTGRESQL_USERNAME} 
+      POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} 
+      POSTGRESQL_DATABASE: ${POSTGRESQL_DATABASE}
+      POSTGRESQL_HOST: ${POSTGRESQL_HOST}
+      KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_HOSTS}
+      KAFKA_APPLICATION_ID: ${KAFKA_APPLICATION_ID_CTRL}
+      KAFKA_CLIENT_ID: ${KAFKA_CLIENT_ID_CTRL}
+      HTTP_SERVER_PORT: ${HTTP_SERVER_PORT}
+      LOG4J2_LEVEL: ${LOG4J2_LEVEL}
+      LINKER_HOST: ${LINKER_HOST}
     deploy:
       replicas: ${JEMPI_CONTROLLER_INSTANCES}
       resources:
@@ -61,25 +61,22 @@ services:
       kafka:
       default:
 
-  jempi-em-calculator:
-    image: jembi/jempi-em-calculator:${JEMPI_EM_CALCULATOR_IMAGE_TAG}
-    environment:
-      kafka.bootstrap.servers: ${KAFKA_HOSTS}
-    deploy:
-      replicas: ${JEMPI_EM_CALCULATOR_INSTANCES}
-      resources:
-        limits:
-          memory: ${JEMPI_EM_CALCULATOR_MEMORY_LIMIT}
-        reservations:
-          memory: ${JEMPI_EM_CALCULATOR_MEMORY_RESERVE}
-    networks:
-      kafka:
-      default:
-
   jempi-linker:
     image: jembi/jempi-linker:${JEMPI_LINKER_IMAGE_TAG}
     environment:
-      kafka.bootstrap.servers: ${KAFKA_HOSTS}
+      POSTGRESQL_USER: ${POSTGRESQL_USERNAME} 
+      POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} 
+      POSTGRESQL_DATABASE: ${POSTGRESQL_DATABASE}
+      KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_HOSTS}
+      KAFKA_APPLICATION_ID_INTERACTIONS: ${KAFKA_APPLICATION_ID_INTERACTIONS}
+      KAFKA_APPLICATION_ID_MU: ${KAFKA_APPLICATION_ID_MU}
+      KAFKA_CLIENT_ID_NOTIFICATIONS: ${KAFKA_CLIENT_ID_NOTIFICATIONS}
+      DGRAPH_HOSTS: ${DGRAPH_HOSTS}
+      DGRAPH_PORTS: ${DGRAPH_PORTS}
+      HTTP_SERVER_PORT: ${HTTP_SERVER_PORT}
+      LOG4J2_LEVEL: ${LOG4J2_LEVEL}
+      LINKER_MATCH_THRESHOLD: ${LINKER_MATCH_THRESHOLD}
+      LINKER_MATCH_THRESHOLD_MARGIN: ${LINKER_MATCH_THRESHOLD_MARGIN}
     deploy:
       replicas: ${JEMPI_LINKER_INSTANCES}
       resources:
@@ -120,6 +117,7 @@ services:
 
 volumes:
   jempi-psql-01-data:
+  jempi-shared-data:
 
 networks:
   kafka:

client-registry-jempi/docker-compose.web.yml

@@ -9,7 +9,6 @@ services:
       REACT_APP_KC_FRONTEND_URL: ${KC_FRONTEND_URL}
       REACT_APP_KC_REALM_NAME: ${KC_REALM_NAME}
       REACT_APP_KC_JEMPI_CLIENT_ID: ${KC_JEMPI_CLIENT_ID}
-    command: sh -c "yarn build && serve -s build"
     deploy:
       replicas: ${JEMPI_WEB_INSTANCES}
       placement:
@@ -21,14 +20,10 @@ services:
           memory: ${JEMPI_WEB_MEMORY_RESERVE}
     networks:
       reverse-proxy:
-      keycloak:
       default:
 
 networks:
   reverse-proxy:
     name: reverse-proxy_public
     external: true
-  keycloak:
-    name: keycloak_public
-    external: true
   default:

client-registry-jempi/importer/jempi_psql_init_db.sql

@@ -19,13 +19,15 @@ CREATE TABLE IF NOT EXISTS Notification_State
 CREATE TABLE IF NOT EXISTS Notification
 (
     Id uuid DEFAULT gen_random_uuid() PRIMARY KEY,
-    Type_Id uuid,
+    Type VARCHAR(50),
     Created date,
     Reviewd_By uuid,
     Reviewed_At timestamp without time zone,
-    State_Id uuid,
+    State VARCHAR(50),
     Patient_Id VARCHAR(50),
-    Names VARCHAR(100)
+    Names VARCHAR(100),
+    Golden_Id VARCHAR(50),
+    Score Numeric
 );
 
 CREATE TABLE IF NOT EXISTS Action

client-registry-jempi/importer/mapping-mediator/docker-compose.config.yml

@@ -0,0 +1,63 @@
+version: '3.9'
+
+services:
+  mapping-mediator-config-importer:
+    image: jembi/instantohie-config-importer
+    deploy:
+      restart_policy:
+        condition: none
+    configs:
+      - source: mediator-config-metadata.js
+        target: /metadata.js
+      - source: mediator-config-register.json
+        target: /register.json
+      - source: mediator-config-register-response.json
+        target: /register-response.json
+      - source: mediator-config-update-response.json
+        target: /update-response.json
+      - source: mediator-config-update.json
+        target: /update.json
+      - source: mediator-config-search.json
+        target: /search.json
+    networks:
+      mapping-mediator:
+    # This command will only attempt to import the config when the uptime responds with a 2xx
+    command: sh -c "wait-on -t 60000 http-get://openhim-mapping-mediator:3003/uptime && node /metadata.js"
+
+configs:
+  mediator-config-metadata.js:
+    file: ./metadata.js
+    name: mediator-config-metadata.js-${mediator_config_metadata_js_DIGEST:?err}
+    labels:
+      name: jempi
+  mediator-config-register.json:
+    file: ./register.json
+    name: mediator-config-register.json-${mediator_config_register_json_DIGEST:?err}
+    labels:
+      name: jempi
+  mediator-config-register-response.json:
+    file: ./register-response.json
+    name: mediator-config-register-response.json-${mediator_config_register_response_json_DIGEST:?err}
+    labels:
+      name: jempi
+  mediator-config-update-response.json:
+    file: ./update-response.json
+    name: mediator-config-update-response.json-${mediator_config_update_response_json_DIGEST:?err}
+    labels:
+      name: jempi
+  mediator-config-update.json:
+    file: ./update.json
+    name: mediator-config-update.json-${mediator_config_update_json_DIGEST:?err}
+    labels:
+      name: jempi
+  mediator-config-search.json:
+    file: ./search.json
+    name: mediator-config-search.json-${mediator_config_search_json_DIGEST:?err}
+    labels:
+      name: jempi
+
+
+networks:
+  mapping-mediator:
+    name: openhim_mapping_mediator_public
+    external: true

client-registry-jempi/importer/mapping-mediator/metadata.js

@@ -0,0 +1,92 @@
+'use strict'
+
+const axios = require('axios')
+const fs = require('fs')
+const path = require('path')
+
+const MEDIATOR_HOSTNAME = process.env.MEDIATOR_HOST_NAME || 'openhim-mapping-mediator'
+const MEDIATOR_API_PORT = process.env.MEDIATOR_API_PORT || 3003
+
+// Function for sending importing the configuration
+const sendRequest = async (data, method, endpointId) => {
+  const url = endpointId
+    ? `http://${MEDIATOR_HOSTNAME}:${MEDIATOR_API_PORT}/endpoints/${endpointId}`
+    : `http://${MEDIATOR_HOSTNAME}:${MEDIATOR_API_PORT}/endpoints`
+
+  const options = {
+    url: url,
+    method: method,
+    headers: {
+      'Content-Type': 'application/json'
+    },
+    data: JSON.stringify(data)
+  }
+
+  try {
+    const response = await axios(options)
+
+    console.log(
+      `Successfully Imported OpenHIM Mediator Config.\n\nImport summary:${JSON.stringify(
+        response.data
+      )}`
+    )
+  } catch (error) {
+    throw new Error(
+      `Failed to import OpenHIM Mediator config: ${error.message}`
+    )
+  }
+}
+
+const getEndpoints = async (callback) => {
+  const options = {
+    url: `http://${MEDIATOR_HOSTNAME}:${MEDIATOR_API_PORT}/endpoints`,
+    method: 'get',
+    headers: {
+      'Content-Type': 'application/json'
+    }
+  }
+
+  try {
+    const response = await axios(options)
+    callback(null, response.data)
+  } catch (error) {
+    callback(
+      new Error(
+        `Failed to fetch OpenHIM Mediator Mapping endpoints: ${error.response.data}`
+      )
+    )
+  }
+}
+
+const importMetaData = async () => {
+  // get the endpoints list incase we need to do a update instead of a create
+  // If the endpoint already exists, perform an update
+  getEndpoints((_error, endpoints) => {
+    const dirPath = path.resolve(__dirname)
+    const files = fs.readdirSync(dirPath)
+    files.reduce((_acc, curr) => {
+      const jsonRegex = /(.*?(\bjson\b)[^$]*)/
+
+      if (curr.match(jsonRegex)) {
+        let method = 'post'
+        let endpointId = null
+
+        const jsonData = JSON.parse(
+          fs.readFileSync(path.join(dirPath, curr), 'utf8')
+        )
+        const matchingEndpoint = endpoints.filter(
+          (endpoint) => endpoint.endpoint.pattern === jsonData.endpoint.pattern
+        )[0]
+
+        if (matchingEndpoint) {
+          endpointId = matchingEndpoint._id
+          method = 'put'
+        }
+
+        sendRequest(jsonData, method, endpointId)
+      }
+    }, {})
+  })
+}
+
+importMetaData()