Merge branch 'main' into api-config-importer-latest

jembi · May 31, 2024 · c106f67 · c106f67
2 parents 1d7d79e + 71faa49
commit c106f67
Show file tree

Hide file tree

Showing 47 changed files with 665 additions and 452 deletions.
diff --git a/.env.cluster b/.env.cluster
@@ -45,6 +45,9 @@ ES_HOSTS="\"analytics-datastore-elastic-search-01:9200","analytics-datastore-ela
 # Analytics Datastore - Clickhouse
 CLICKHOUSE_HOST=analytics-datastore-clickhouse-01
 
+# Dashboard Vusualizer - Superset
+SUPERSET_POSTGRESQL_URL=pgpool-1:5432,pgpool-2:5432,pgpool-3:5432
+
 # Dashboard Visualiser - Kibana
 KIBANA_INSTANCES=1
 
@@ -100,6 +103,7 @@ JEMPI_CONTROLLER_INSTANCES=1
 JEMPI_EM_CALCULATOR_INSTANCES=1
 JEMPI_LINKER_INSTANCES=1
 JEMPI_API_INSTANCES=1
+JEMPI_POSTGRES_DB=pgpool-1,pgpool-2,pgpool-3
 
 # Resource limits
 OPENHIM_MEMORY_LIMIT=4G
@@ -110,5 +114,5 @@ KAFDROP_MEMORY_LIMIT=500M
 
 # PG Pool
 # The postgres users have to be specified so that they can be added to the pg_pool authentication interceptor
-PGPOOL_POSTGRES_CUSTOM_USERS=hapi,keycloak
-PGPOOL_POSTGRES_CUSTOM_PASSWORDS=instant101,instant101
+PGPOOL_POSTGRES_CUSTOM_USERS=hapi,keycloak,jempi,superset
+PGPOOL_POSTGRES_CUSTOM_PASSWORDS=instant101,instant101,instant101,instant101
diff --git a/.github/workflows/run-tests.sh b/.github/workflows/run-tests.sh
@@ -25,43 +25,47 @@ for package in "${CHANGED_FILES[@]}"; do
     fi
 done
 
+function run_test() {
+   DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud HOST=$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":$1
+}
+
+# Run the basic funtional end to end tests for the CDR recipe
+run_test "recipe"
+
 if [[ ${#changed_packages[@]} -eq 0 ]] || [[ "${!changed_packages[*]}" == *"utils"* ]] || [[ "${!changed_packages[*]}" == *"features/steps"* ]] || [[ "${!changed_packages[*]}" == *"infrastructure"* ]] ; then
-    openhim_ran="true"
-    DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":openhim
+    openhimRan="true"
+    run_test "openhim"
 else
     for folder_name in "${!changed_packages[@]}"; do
         echo "$folder_name was changed"
 
         if [[ $folder_name == *"clickhouse"* ]]; then
-            DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":clickhouse
+            run_test "clickhouse"
         elif [[ $folder_name == *"elastic"* ]] || [[ $folder_name == *"kibana"* ]] || [[ $folder_name == *"logstash"* ]]; then
-            DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":elk
+            run_test "elk"
         elif [[ $folder_name == *"kafka"* ]] || [[ $folder_name == *"monitoring"* ]]; then
-            DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":kafka
+            run_test "kafka"
         elif [[ $folder_name == *"openhim"* ]] && [[ $openhimRan == "false" ]]; then
             openhimRan="true"
-            DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":openhim
+            run_test "openhim"
         elif [[ $folder_name == *"reverse-proxy"* ]]; then
-            DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":nginx
+            run_test "nginx"
         elif [[ $folder_name == *"hapi"* ]]; then
-            DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":hapi
+            run_test "hapi"
         elif [[ $folder_name == *"santempi"* ]]; then
-            DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":sante
+            run_test "sante"
         elif [[ $folder_name == *"monitoring"* ]]; then
-            DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":monitoring
+            run_test "monitoring"
         elif [[ $folder_name == *"keycloak"* ]]; then
-            DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":keycloak
+            run_test "keycloak"
         elif [[ $folder_name == *"superset"* ]] && [[ $NODE_MODE == "single" ]]; then
-            DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":superset
+            run_test "superset"
         elif [[ $folder_name == *"jsreport"* ]] && [[ $NODE_MODE == "single" ]]; then
-            DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":jsreport
+            run_test "jsreport"
         elif [[ $folder_name == *"mpi-mediator"* ]] && [[ $NODE_MODE == "single" ]]; then
-            DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":mpi-mediator
+            run_test "mpi-mediator"
         elif [[ $folder_name == *"jempi"* ]] && [[ $NODE_MODE == "single" ]]; then
-            DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":jempi
+            run_test "jempi"
         fi
     done
 fi
-
-# Run the basic funtional end to end tests for the CDR recipe
-DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud HOST=$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":recipe
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -10,7 +10,9 @@
   },
   "json.schemas": [
     {
-      "fileMatch": ["package-metadata.json"],
+      "fileMatch": [
+        "package-metadata.json"
+      ],
       "url": "https://raw.githubusercontent.com/openhie/package-starter-kit/main/schema/package-metadata.schema.json"
     }
   ],

diff --git a/analytics-datastore-clickhouse/docker-compose.cluster.yml b/analytics-datastore-clickhouse/docker-compose.cluster.yml
@@ -1,17 +1,17 @@
-version: '3.9'
+version: "3.9"
 
 services:
   analytics-datastore-clickhouse-01:
-    image: clickhouse/clickhouse-server
+    image: ${CLICKHOUSE_IMAGE}
     ulimits:
       noFile: 262144
     volumes:
       - clickhouse-data-01:/var/lib/clickhouse/
     hostname: analytics-datastore-clickhouse-01
-    deploy: 
-      placement: 
-        constraints: 
-          - "node.labels.name==node-1"
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==${ANALYTICS_DATASTORE_CLICKHOUSE_01_PLACEMENT}"
     configs:
       - target: /etc/clickhouse-server/config.d/docker_related_config.xml
         source: docker_related_config.xml
@@ -36,12 +36,12 @@ services:
       default:
 
   analytics-datastore-clickhouse-02:
-    image: clickhouse/clickhouse-server
+    image: ${CLICKHOUSE_IMAGE}
     hostname: analytics-datastore-clickhouse-02
-    deploy: 
-      placement: 
-        constraints: 
-          - "node.labels.name==node-2"
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==${ANALYTICS_DATASTORE_CLICKHOUSE_02_PLACEMENT}"
     ulimits:
       noFile: 262144
     volumes:
@@ -70,16 +70,16 @@ services:
       default:
 
   analytics-datastore-clickhouse-03:
-    image: clickhouse/clickhouse-server
+    image: ${CLICKHOUSE_IMAGE}
     hostname: analytics-datastore-clickhouse-03
-    deploy: 
-      placement: 
-        constraints: 
-          - "node.labels.name==node-3"
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==${ANALYTICS_DATASTORE_CLICKHOUSE_03_PLACEMENT}"
     ulimits:
       noFile: 262144
     volumes:
-      - clickhouse-data-03:/var/lib/clickhouse/        
+      - clickhouse-data-03:/var/lib/clickhouse/
     configs:
       - target: /etc/clickhouse-server/config.d/docker_related_config.xml
         source: docker_related_config.xml
@@ -109,7 +109,7 @@ services:
     ulimits:
       noFile: 262144
     volumes:
-      - clickhouse-data-04:/var/lib/clickhouse/        
+      - clickhouse-data-04:/var/lib/clickhouse/
     configs:
       - target: /etc/clickhouse-server/config.d/docker_related_config.xml
         source: docker_related_config.xml
@@ -136,77 +136,77 @@ volumes:
   clickhouse-data-02:
   clickhouse-data-03:
   clickhouse-data-04:
-  
+
 configs:
   docker_related_config.xml:
     file: ./cluster_configs/docker_related_config.xml
     name: docker_related_config.xml-${docker_related_config_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_enable_keeper_01.xml:
     file: ./cluster_configs/enable_keeper_01.xml
     name: enable_keeper_01.xml-${enable_keeper_01_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_enable_keeper_02.xml:
     file: ./cluster_configs/enable_keeper_02.xml
     name: enable_keeper_02.xml-${enable_keeper_02_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_enable_keeper_03.xml:
     file: ./cluster_configs/enable_keeper_03.xml
     name: enable_keeper_03.xml-${enable_keeper_03_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_macros_01.xml:
     file: ./cluster_configs/macros_01.xml
     name: macros_01.xml-${macros_01_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_macros_02.xml:
     file: ./cluster_configs/macros_02.xml
     name: macros_02.xml-${macros_02_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_macros_03.xml:
     file: ./cluster_configs/macros_03.xml
     name: macros_03.xml-${macros_03_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_macros_04.xml:
     file: ./cluster_configs/macros_04.xml
     name: macros_04.xml-${macros_04_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_remote_servers.xml:
     file: ./cluster_configs/remote_servers.xml
     name: remote_servers.xml-${remote_servers_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_use_keeper.xml:
     file: ./cluster_configs/use_keeper.xml
     name: use_keeper.xml-${use_keeper_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_metric_log.xml:
     file: ./general_configs/metric_log.xml
     name: metric_log.xml.xml-${metric_log_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_part_log.xml:
     file: ./general_configs/part_log.xml
     name: part_log.xml.xml-${part_log_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_query_log.xml:
     file: ./general_configs/query_log.xml
     name: query_log.xml.xml-${query_log_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_trace_log.xml:
     file: ./general_configs/trace_log.xml
     name: trace_log.xml.xml-${trace_log_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
 
 networks:

diff --git a/analytics-datastore-clickhouse/docker-compose.yml b/analytics-datastore-clickhouse/docker-compose.yml
@@ -1,8 +1,8 @@
-version: '3.9'
+version: "3.9"
 
 services:
   analytics-datastore-clickhouse:
-    image: clickhouse/clickhouse-server
+    image: ${CLICKHOUSE_IMAGE}
     ulimits:
       noFile: 262144
     volumes:
@@ -15,7 +15,7 @@ services:
       - target: /etc/clickhouse-server/config.d/query_log.xml
         source: clickhouse_query_log.xml
       - target: /etc/clickhouse-server/config.d/trace_log.xml
-        source: clickhouse_trace_log.xml  
+        source: clickhouse_trace_log.xml
     networks:
       public:
       default:
@@ -27,22 +27,22 @@ configs:
   clickhouse_metric_log.xml:
     file: ./general_configs/metric_log.xml
     name: metric_log.xml.xml-${metric_log_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_part_log.xml:
     file: ./general_configs/part_log.xml
     name: part_log.xml.xml-${part_log_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_query_log.xml:
     file: ./general_configs/query_log.xml
     name: query_log.xml.xml-${query_log_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_trace_log.xml:
     file: ./general_configs/trace_log.xml
     name: trace_log.xml.xml-${trace_log_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
 
 networks:

diff --git a/analytics-datastore-clickhouse/package-metadata.json b/analytics-datastore-clickhouse/package-metadata.json
@@ -7,6 +7,10 @@
   "dependencies": [],
   "environmentVariables": {
     "CLICKHOUSE_HOST": "analytics-datastore-clickhouse",
-    "CLICKHOUSE_PORT": "8123"
+    "CLICKHOUSE_PORT": "8123",
+    "CLICKHOUSE_IMAGE": "clickhouse/clickhouse-server:23.8.14.6",
+    "ANALYTICS_DATASTORE_CLICKHOUSE_01_PLACEMENT": "node-1",
+    "ANALYTICS_DATASTORE_CLICKHOUSE_02_PLACEMENT": "node-2",
+    "ANALYTICS_DATASTORE_CLICKHOUSE_03_PLACEMENT": "node-3"
   }
 }
diff --git a/analytics-datastore-elastic-search/docker-compose.certs.yml b/analytics-datastore-elastic-search/docker-compose.certs.yml
@@ -2,7 +2,7 @@ version: "3.9"
 
 services:
   create_certs:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.0
+    image: ${ES_IMAGE}
     deploy:
       placement:
         constraints: