Merge pull request #229 from jembi/PLAT-706-add-postgres-jempi

Plat 706 add postgres service to Jempi
jembi · May 8, 2023 · 265c574 · 265c574
2 parents b90ce87 + cac83cb
commit 265c574
Show file tree

Hide file tree

Showing 22 changed files with 264 additions and 45 deletions.
diff --git a/.env.cluster b/.env.cluster
@@ -28,14 +28,14 @@ REPMGR_PARTNER_NODES=santempi-psql-1,santempi-psql-2,santempi-psql-3
 # Reverse Proxy - Nginx
 REVERSE_PROXY_INSTANCES=3
 DOMAIN_NAME=domain
-SUBDOMAINS=openhimcomms.domain,openhimcore.domain,openhimconsole.domain,kibana.domain,reports.domain,santewww.domain,santempi.domain,superset.domain,keycloak.domain,grafana.domain,minio.domain,jempi.domain
+SUBDOMAINS=openhimcomms.domain,openhimcore.domain,openhimconsole.domain,kibana.domain,reports.domain,santewww.domain,santempi.domain,superset.domain,keycloak.domain,grafana.domain,minio.domain,jempi-web.domain,jempi-api.domain
 STAGING=true
 INSECURE=false
 
 # Analytics Datastore - Elastic Search
 ES_HEAP_SIZE=-Xms8192m -Xmx8192m
 ES_LEADER_NODE=analytics-datastore-elastic-search-01
-ES_HOSTS="analytics-datastore-elastic-search-01:9200","analytics-datastore-elastic-search-02:9200","analytics-datastore-elastic-search-03:9200"
+ES_HOSTS="\"analytics-datastore-elastic-search-01:9200","analytics-datastore-elastic-search-02:9200","analytics-datastore-elastic-search-03:9200\""
 
 # Analytics Datastore - Clickhouse
 CLICKHOUSE_HOST=analytics-datastore-clickhouse-01
@@ -72,12 +72,15 @@ HAPI_PROXY_INSTANCES=3
 # Identity Access Manager - Keycloak
 KC_FRONTEND_URL=https://keycloak.domain
 KC_GRAFANA_ROOT_URL=https://grafana.domain
+KC_JEMPI_ROOT_URL=https://jempi-web.domain
 KC_SUPERSET_ROOT_URL=https://superset.domain
 KC_OPENHIM_ROOT_URL=https://openhimconsole.domain
 GF_SERVER_DOMAIN=grafana.domain
 
 # Client Registry - JeMPI
 JEMPI_WEB_INSTANCES=3
+REACT_APP_JEMPI_BASE_URL=https://jempi-api.domain/JeMPI
+JEMPI_SESSION_SECURE=true
 
 # Resource limits
 OPENHIM_MEMORY_LIMIT=4G

diff --git a/.env.remote b/.env.remote
@@ -10,7 +10,7 @@ MOCK_SERVER_PORT=<port>
 POSTGRES_REPLICA_SET=postgres-1:5432,postgres-2:5432,postgres-3:5432
 
 # Analytics Datastore - Elastic Search
-ES_HOSTS="analytics-datastore-elastic-search-01:9200","analytics-datastore-elastic-search-02:9200","analytics-datastore-elastic-search-03:9200"
+ES_HOSTS="\"analytics-datastore-elastic-search-01:9200","analytics-datastore-elastic-search-02:9200","analytics-datastore-elastic-search-03:9200\""
 
 # Analytics Datastore - Clickhouse
 CLICKHOUSE_HOST=analytics-datastore-clickhouse-01

diff --git a/client-registry-jempi/docker-compose.api-dev.yml b/client-registry-jempi/docker-compose.api-dev.yml
@@ -5,5 +5,4 @@ services:
     ports:
       - published: 50000
         target: 50000
-        protocol: tcp
         mode: host
diff --git a/client-registry-jempi/docker-compose.api.yml b/client-registry-jempi/docker-compose.api.yml
@@ -12,6 +12,9 @@ services:
       JEMPI_SESSION_SECRET: ${JEMPI_SESSION_SECRET}
       kafka.bootstrap.servers: ${KAFKA_HOSTS}
       JEMPI_FILE_IMPORT_MAX_SIZE_BYTE: ${JEMPI_FILE_IMPORT_MAX_SIZE_BYTE}
+      JEMPI_SESSION_SECURE: ${JEMPI_SESSION_SECURE}
+      JEMPI_SESSION_DOMAIN_NAME: ${DOMAIN_NAME}
+      postgres.server: jempi-postgresql
     deploy:
       replicas: 1
       resources:
@@ -20,11 +23,16 @@ services:
         reservations:
           memory: ${JEMPI_API_MEMORY_RESERVE}
     networks:
+      reverse-proxy:
       keycloak:
       kafka:
       default:
 
+
 networks:
+  reverse-proxy:
+    name: reverse-proxy_public
+    external: true
   keycloak:
     name: keycloak_public
     external: true

diff --git a/client-registry-jempi/docker-compose.combined-dev.yml b/client-registry-jempi/docker-compose.combined-dev.yml
@@ -21,3 +21,9 @@ services:
         target: 50000
         protocol: tcp
         mode: host
+
+  jempi-postgresql:
+    ports:
+      - published: 5435
+        target: 5432
+        mode: host
diff --git a/client-registry-jempi/docker-compose.combined.yml b/client-registry-jempi/docker-compose.combined.yml
@@ -91,8 +91,43 @@ services:
       kafka:
       default:
 
+  jempi-postgresql:
+    image: bitnami/postgresql:15.2.0
+    environment:
+      POSTGRESQL_USERNAME: ${POSTGRESQL_USERNAME}
+      POSTGRESQL_DATABASE: ${POSTGRESQL_DATABASE}
+      ALLOW_EMPTY_PASSWORD: "yes"
+    deploy:
+      mode: replicated
+      replicas: 1
+      resources:
+        limits:
+          cpus: ${JEMPI_POSTGRES_CPU_LIMIT}
+          memory: ${JEMPI_POSTGRES_MEMORY_LIMIT}
+        reservations:
+          cpus: ${JEMPI_POSTGRES_CPU_RESERVE}
+          memory: ${JEMPI_POSTGRES_MEMORY_RESERVE}
+    volumes:
+      - "jempi-psql-01-data:/bitnami/postgresql"
+    configs:
+      - target: /docker-entrypoint-initdb.d/jempi_psql_init_db.sql
+        source: jempi_psql_init_db.sql
+    networks:
+      kafka:
+      default:
+
+volumes:
+  jempi-psql-01-data:
+
 networks:
   kafka:
     name: kafka_public
     external: true
   defualt:
+
+configs:
+  jempi_psql_init_db.sql:
+    file: ./importer/jempi_psql_init_db.sql
+    name: jempi_psql_init_db.sql-${jempi_psql_init_db_sql_DIGEST:?err}
+    labels:
+      name: jempi
diff --git a/client-registry-jempi/docker-compose.dgraph.yml b/client-registry-jempi/docker-compose.dgraph.yml
@@ -3,6 +3,7 @@ version: '3.9'
 services:
   jempi-alpha-01:
     image: dgraph/dgraph:v22.0.0
+    hostname: alpha-01
     volumes:
       - jempi-alpha-01-data:/dgraph
     deploy:
@@ -18,6 +19,7 @@ services:
 
   jempi-alpha-02:
     image: dgraph/dgraph:v22.0.0
+    hostname: alpha-02
     volumes:
       - jempi-alpha-02-data:/dgraph
     deploy:
@@ -33,6 +35,7 @@ services:
 
   jempi-alpha-03:
     image: dgraph/dgraph:v22.0.0
+    hostname: alpha-03
     volumes:
       - jempi-alpha-03-data:/dgraph
     deploy:

diff --git a/client-registry-jempi/docker-compose.web.yml b/client-registry-jempi/docker-compose.web.yml
@@ -6,9 +6,10 @@ services:
     environment:
       REACT_APP_JEMPI_BASE_URL: ${REACT_APP_JEMPI_BASE_URL}
       REACT_APP_MOCK_BACKEND: ${REACT_APP_MOCK_BACKEND}
-      KC_FRONTEND_URL: ${KC_FRONTEND_URL}
-      KC_REALM_NAME: ${KC_REALM_NAME}
-      KC_JEMPI_CLIENT_ID: ${KC_JEMPI_CLIENT_ID}
+      REACT_APP_KC_FRONTEND_URL: ${KC_FRONTEND_URL}
+      REACT_APP_KC_REALM_NAME: ${KC_REALM_NAME}
+      REACT_APP_KC_JEMPI_CLIENT_ID: ${KC_JEMPI_CLIENT_ID}
+    command: sh -c "yarn build && serve -s build"
     deploy:
       replicas: ${JEMPI_WEB_INSTANCES}
       placement:
@@ -23,6 +24,7 @@ services:
       keycloak:
       default:
 
+
 networks:
   reverse-proxy:
     name: reverse-proxy_public

diff --git a/client-registry-jempi/importer/docker-compose.config.yml b/client-registry-jempi/importer/docker-compose.config.yml
@@ -23,7 +23,7 @@ configs:
     file: ./kafka.js
     name: jempi-kafka-config.js-${jempi_kafka_config_DIGEST:?err}
     labels:
-      name: jempi-kafka
+      name: jempi
 
 networks:
   kafka:

diff --git a/client-registry-jempi/importer/jempi_psql_init_db.sql b/client-registry-jempi/importer/jempi_psql_init_db.sql
@@ -0,0 +1,80 @@
+CREATE TABLE IF NOT EXISTS Notification_Type
+(
+    Id uuid DEFAULT gen_random_uuid() PRIMARY KEY,
+    Type VARCHAR(50)
+);
+
+CREATE TABLE IF NOT EXISTS Action_Type
+(
+    Id UUID DEFAULT gen_random_uuid() PRIMARY KEY UNIQUE,
+    Type VARCHAR(50)
+);
+
+CREATE TABLE IF NOT EXISTS Notification_State
+(
+    Id UUID DEFAULT gen_random_uuid() PRIMARY KEY,
+    State VARCHAR(50)
+);
+
+CREATE TABLE IF NOT EXISTS Notification
+(
+    Id uuid DEFAULT gen_random_uuid() PRIMARY KEY,
+    Type_Id uuid,
+    Created date,
+    Reviewd_By uuid,
+    Reviewed_At timestamp without time zone,
+    State_Id uuid,
+    Patient_Id VARCHAR(50),
+    Names VARCHAR(100)
+);
+
+CREATE TABLE IF NOT EXISTS Action
+(
+    Id UUID DEFAULT gen_random_uuid() PRIMARY KEY,
+    Notification_Id UUID,
+    Action_Type_Id UUID,
+    Date date,
+    CONSTRAINT FK_Notification
+      FOREIGN KEY(Notification_Id) 
+	    REFERENCES Notification(Id),
+    CONSTRAINT FK_Action_Type
+      FOREIGN KEY(Action_Type_Id) 
+	    REFERENCES Action_Type(Id)
+);
+
+CREATE TABLE IF NOT EXISTS Match
+(
+    Notification_Id UUID,
+    Score Numeric,
+    Golden_Id VARCHAR(50),
+    CONSTRAINT FK_Notification
+      FOREIGN KEY(Notification_Id) 
+	    REFERENCES Notification(Id)
+);
+
+CREATE TABLE IF NOT EXISTS candidates
+(
+    Notification_Id UUID,
+    Score Numeric,
+    Golden_Id VARCHAR(50),
+    CONSTRAINT FK_Notification
+      FOREIGN KEY(Notification_Id) 
+	    REFERENCES Notification(Id)
+);
+
+CREATE TABLE IF NOT EXISTS users
+(
+    id UUID DEFAULT gen_random_uuid() PRIMARY KEY UNIQUE,
+    given_name VARCHAR(255),
+    family_name VARCHAR(255),
+    email VARCHAR(255) UNIQUE,
+    username VARCHAR(255) UNIQUE
+);
+
+INSERT INTO Notification_State(State)
+VALUES ('New'), ('Seen'), ('Actioned'), ('Accepted'), ('Pending');
+
+INSERT INTO Notification_Type(Type)
+VALUES ('THRESHOLD'), ('MARGIN'), ('UPDATE');
+
+\dt;
diff --git a/client-registry-jempi/package-metadata.json b/client-registry-jempi/package-metadata.json
@@ -4,7 +4,11 @@
   "description": "A patient matching and deduplicater for the platform",
   "type": "infrastructure",
   "version": "0.0.1",
-  "dependencies": ["interoperability-layer-openhim", "message-bus-kafka"],
+  "dependencies": [
+    "interoperability-layer-openhim",
+    "message-bus-kafka",
+    "identity-access-manager-keycloak"
+  ],
   "environmentVariables": {
     "KAFKA_HOSTS": "kafka-01:9092",
     "JEMPI_ZERO_01_MEMORY_LIMIT": "3G",
@@ -32,17 +36,17 @@
     "JEMPI_API_MEMORY_LIMIT": "3G",
     "JEMPI_API_MEMORY_RESERVE": "500M",
     "JEMPI_KAFKA_TOPICS": "JeMPI-async-preprocessor,JeMPI-patient-controller,JeMPI-patient-em,JeMPI-patient-linker,JeMPI-mu-linker,JeMPI-notifications",
-    "JEMPI_ASYNC_RECEIVER_IMAGE_TAG": "0.1.1",
-    "JEMPI_SYNC_RECEIVER_IMAGE_TAG": "0.1.1",
-    "JEMPI_PRE_PROCESSOR_IMAGE_TAG": "0.1.1",
-    "JEMPI_CONTROLLER_IMAGE_TAG": "0.1.1",
-    "JEMPI_EM_CALCULATOR_IMAGE_TAG": "0.1.1",
-    "JEMPI_LINKER_IMAGE_TAG": "0.1.1",
-    "JEMPI_API_IMAGE_TAG": "0.1.1",
+    "JEMPI_ASYNC_RECEIVER_IMAGE_TAG": "0.2.0",
+    "JEMPI_SYNC_RECEIVER_IMAGE_TAG": "0.2.0",
+    "JEMPI_PRE_PROCESSOR_IMAGE_TAG": "0.2.0",
+    "JEMPI_CONTROLLER_IMAGE_TAG": "0.2.0",
+    "JEMPI_EM_CALCULATOR_IMAGE_TAG": "0.2.0",
+    "JEMPI_LINKER_IMAGE_TAG": "0.2.0",
+    "JEMPI_API_IMAGE_TAG": "0.2.0",
     "JEMPI_OPENHIM_PASSWORD": "instant101",
     "JEMPI_SESSION_SECRET": "c05ll3lesrinf39t7mc5h6un6r0c69lgfno69dsak3vabeqamouq4328cuaekros401ajdpkh60rrt",
     "JEMPI_FILE_IMPORT_MAX_SIZE_BYTE": 128000000,
-    "REACT_APP_JEMPI_BASE_URL": "http://jempi-api:50000/JeMPI",
+    "REACT_APP_JEMPI_BASE_URL": "http://localhost:50000/JeMPI",
     "REACT_APP_MOCK_BACKEND": "false",
     "JEMPI_WEB_VERSION": "latest",
     "JEMPI_WEB_INSTANCES": 1,
@@ -53,6 +57,14 @@
     "KC_API_URL": "http://identity-access-manager-keycloak:8080",
     "KC_JEMPI_CLIENT_ID": "jempi-oauth",
     "KC_JEMPI_CLIENT_SECRET": "Tbe3llP5OJIlqUjz7K1wPp8YDAdCOEMn",
-    "KC_JEMPI_ROOT_URL": "http://localhost:3033"
+    "KC_JEMPI_ROOT_URL": "http://localhost:3033",
+    "POSTGRESQL_DATABASE": "notifications",
+    "POSTGRESQL_USERNAME": "postgres",
+    "JEMPI_POSTGRES_CPU_LIMIT": "0",
+    "JEMPI_POSTGRES_CPU_RESERVE": "0.05",
+    "JEMPI_POSTGRES_MEMORY_LIMIT": "3G",
+    "JEMPI_POSTGRES_MEMORY_RESERVE": "500M",
+    "JEMPI_SESSION_SECURE": false,
+    "DOMAIN_NAME": ""
   }
 }
diff --git a/client-registry-jempi/swarm.sh b/client-registry-jempi/swarm.sh
@@ -95,7 +95,7 @@ function destroy_package() {
     log warn "Volumes are only deleted on the host on which the command is run. Postgres volumes on other nodes are not deleted"
   fi
 
-  docker::prune_configs "jempi-kafka"
+  docker::prune_configs "jempi"
 }
 
 main() {

diff --git a/identity-access-manager-keycloak/package-metadata.json b/identity-access-manager-keycloak/package-metadata.json
@@ -26,7 +26,7 @@
     "KC_GRAFANA_CLIENT_SECRET": "CV14QfwnpYFj1IH5dK5lScPNCYAIYP1c",
     "KC_GRAFANA_ROOT_URL": "http://localhost:3000",
     "KC_GRAFANA_CLIENT_ROLES": "admin,editor,viewer",
-    "KC_JEMPI_SSO_ENABLED": "false",
+    "KC_JEMPI_SSO_ENABLED": "true",
     "KC_JEMPI_CLIENT_ID": "jempi-oauth",
     "KC_JEMPI_CLIENT_SECRET": "Tbe3llP5OJIlqUjz7K1wPp8YDAdCOEMn",
     "KC_JEMPI_ROOT_URL": "http://localhost:3033",

diff --git a/infrastructure/ansible/inventories/development/group_vars/all.yml b/infrastructure/ansible/inventories/development/group_vars/all.yml
@@ -14,15 +14,15 @@ sudoers:
     key: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINGNVNlWwpQKVXPSngEOOdJjuCyVEQoBXv+nHqmcM6vW
   - name: [email protected]
     username: barrydwyer
-    state: present
+    state: absent
     key: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICw4vvV33xxrTzxAiPoRhuwyWUzmgrP5NPo2n4bmKx7P
   - name: [email protected]
     username: marklabuschagne
-    state: present
+    state: absent
     key: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHa1ghm9JsRl6JDtWU8AV+U70jehqlWaXmpjRg3afqB4
   - name: [email protected]
     username: castellogovender
-    state: present
+    state: absent
     key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDWl3Vc+v0fiuVBIVTDehGGUgZqfvn0JG2oNS7oMV9X2s3p8VwndKm6/oa5bOO4+sp9Z/8ychvo7S2MNXBRQAzTEXo5Oej5E/aZqaFzvpjgI79cnSidIHTzOUw0rUk2EwpSCpJVSXCjSjxGLbbLI35+KltLVoNXGmpNFq4Xi7XwT9aXIm5KQcbumV1NFRGPl+XiaRHioo37gFtub9XjODdQFfs9KGMS2oEuzzFFaXj5unUDZo+lZlEp67HHwq6EpDLR5xf7l96xBvMpQ/MS2pTxCoHVqXx3IDL5CXBBrgUKS8lUuhPPUJxPeARz4DuU7SmWxhNwTDspAEROQ8Rsq6M+kvdslFVeJ4O5M2Rs75mDTUCAkmZTixkTMuMmWqBuQT3zPA7d8Gf0YLhKfwgyH8y5YeJgpU01o2wbH6em/n2isI+pSIatdUFQAKuFdzJiawQJEB5UobKwN2tKVP8ImjrfN+q8nuqHZwTTGW0AaCrb985aQFTgMbYwQeR3mkBNOE0=
   - name: [email protected]
     username: lumieremondo
@@ -52,15 +52,15 @@ docker_users:
     key: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINGNVNlWwpQKVXPSngEOOdJjuCyVEQoBXv+nHqmcM6vW
   - name: [email protected]
     username: barrydwyer
-    state: present
+    state: absent
     key: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICw4vvV33xxrTzxAiPoRhuwyWUzmgrP5NPo2n4bmKx7P
   - name: [email protected]
     username: marklabuschagne
-    state: present
+    state: absent
     key: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHa1ghm9JsRl6JDtWU8AV+U70jehqlWaXmpjRg3afqB4
   - name: [email protected]
     username: castellogovender
-    state: present
+    state: absent
     key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDWl3Vc+v0fiuVBIVTDehGGUgZqfvn0JG2oNS7oMV9X2s3p8VwndKm6/oa5bOO4+sp9Z/8ychvo7S2MNXBRQAzTEXo5Oej5E/aZqaFzvpjgI79cnSidIHTzOUw0rUk2EwpSCpJVSXCjSjxGLbbLI35+KltLVoNXGmpNFq4Xi7XwT9aXIm5KQcbumV1NFRGPl+XiaRHioo37gFtub9XjODdQFfs9KGMS2oEuzzFFaXj5unUDZo+lZlEp67HHwq6EpDLR5xf7l96xBvMpQ/MS2pTxCoHVqXx3IDL5CXBBrgUKS8lUuhPPUJxPeARz4DuU7SmWxhNwTDspAEROQ8Rsq6M+kvdslFVeJ4O5M2Rs75mDTUCAkmZTixkTMuMmWqBuQT3zPA7d8Gf0YLhKfwgyH8y5YeJgpU01o2wbH6em/n2isI+pSIatdUFQAKuFdzJiawQJEB5UobKwN2tKVP8ImjrfN+q8nuqHZwTTGW0AaCrb985aQFTgMbYwQeR3mkBNOE0=
   - name: [email protected]
     username: lumieremondo

diff --git a/infrastructure/ansible/inventories/development/hosts b/infrastructure/ansible/inventories/development/hosts
@@ -1,8 +1,8 @@
 [leader]
-13.244.111.91 name=node-1
+{ip_1} name=node-1
 
 [managers]
-13.244.239.144 name=node-2
-13.246.13.60 name=node-3
+{ip_2} name=node-2
+{ip_3} name=node-3
 
 [workers]
diff --git a/infrastructure/ansible/inventories/production/hosts b/infrastructure/ansible/inventories/production/hosts
@@ -1,7 +1,8 @@
 [leader]
-
+{ip_1} name=node-1
 
 [managers]
-
+{ip_2} name=node-2
+{ip_3} name=node-3
 
 [workers]