E2e tests for recipes

.env.cluster

@@ -16,6 +16,7 @@ OPENHIM_MEDIATOR_API_PORT=443
 MONGO_SET_COUNT=3
 OPENHIM_MONGO_URL=mongodb://mongo-1:27017,mongo-2:27017,mongo-3:27017/openhim?replicaSet=mongo-set
 OPENHIM_MONGO_ATNAURL=mongodb://mongo-1:27017,mongo-2:27017,mongo-3:27017/openhim?replicaSet=mongo-set
+KAFKA_BROKERS=kafka-01,kafka-02,kafka-03
 
 # FHIR Datastore - HAPI FHIR
 HAPI_FHIR_INSTANCES=3
@@ -63,9 +64,13 @@ JS_REPORT_PACKAGE_PATH=
 # Message Bus Kafka
 # Topics should comma seperated, optional include partion and repliction values
 #   e.g. <topic>:<partions>:<replicationFactor> -> test:3:2 (defaults to <topics>:3:1)
-KAFKA_TOPICS=2xx,reprocess,3xx,metrics:3:3
+KAFKA_TOPICS=2xx,2xx-async,reprocess,3xx,metrics:3:3,patient,observation
 KAFKA_HOSTS=kafka-01:9092,kafka-02:9092,kafka-03:9092
 
+# Kafka consumer mapper
+KAFKA_HOST=kafka-01,kafka-02,kafka-03
+KAFKA_PORT=9092
+
 # MPI MEDIATOR
 MPI_MEDIATOR_INSTANCES=3
 

.github/workflows/ci-cluster.yml

@@ -35,6 +35,7 @@ jobs:
         shell: bash
         env:
           SSH_KEY: ${{ secrets.DEPLOY_KEY_PUB }}
+      - run: cp ../../test/cucumber/features/resources/testsSecurity.tf ./security.tf
       - run: terraform init
       - run: terraform apply -auto-approve
       - name: Upload terraform state
@@ -72,7 +73,7 @@ jobs:
   run-e2e-tests:
     runs-on: ubuntu-20.04
     needs: configure-e2e-server
-    timeout-minutes: 120
+    timeout-minutes: 160
     steps:
       - uses: actions/checkout@v3
         with:

.github/workflows/ci.yml

@@ -36,6 +36,7 @@ jobs:
         shell: bash
         env:
           SSH_KEY: ${{ secrets.DEPLOY_KEY_PUB }}
+      - run: cp ../../test/cucumber/features/resources/testsSecurity.tf ./security.tf
       - run: terraform init
       - run: terraform apply -auto-approve
       - name: Upload terraform state
@@ -71,7 +72,7 @@ jobs:
   run-e2e-tests:
     runs-on: ubuntu-20.04
     needs: configure-e2e-server
-    timeout-minutes: 80
+    timeout-minutes: 120
     steps:
       - uses: actions/checkout@v3
         with:

.github/workflows/run-tests.sh

@@ -25,14 +25,8 @@ for package in "${CHANGED_FILES[@]}"; do
     fi
 done
 
-if [[ ${#changed_packages[@]} -eq 0 ]] || [[ "${!changed_packages[*]}" == *"utils"* ]]; then
-    DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE"
-elif [[ "${!changed_packages[*]}" == *"features/single-mode"* ]] && [[ $NODE_MODE == "single" ]]; then
-    DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:single
-elif [[ "${!changed_packages[*]}" == *"features/cluster-mode"* ]] && [[ $NODE_MODE == "cluster" ]]; then
-    DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:cluster
-elif [[ "${!changed_packages[*]}" == *"infrastructure"* ]] && [[ $openhimRan == "false" ]]; then
-    openhimRan="true"
+if [[ ${#changed_packages[@]} -eq 0 ]] || [[ "${!changed_packages[*]}" == *"utils"* ]] || [[ "${!changed_packages[*]}" == *"features/steps"* ]] || [[ "${!changed_packages[*]}" == *"infrastructure"* ]] ; then
+    openhim_ran="true"
     DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":openhim
 else
     for folder_name in "${!changed_packages[@]}"; do
@@ -68,3 +62,6 @@ else
         fi
     done
 fi
+
+# Run the basic funtional end to end tests for the CDR recipe
+DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud HOST=$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":recipe

cdr-dw.env

@@ -18,3 +18,5 @@ KC_JEMPI_SSO_ENABLED=true
 REACT_APP_JEMPI_BASE_API_PORT=50001
 KC_SUPERSET_SSO_ENABLED=true
 KC_GRAFANA_SSO_ENABLED=true
+
+INSECURE_PORTS=5001:5001-80:80-8080:8080-3003:3003-8124:8123-5601:5601-5488:5488-3000:3000-9200:9200-8089:8089-9001:9001-3033:3033-50000:50000

config.yaml

@@ -27,20 +27,20 @@ packages:
 profiles:
   - name: cdr-dw
     packages:
+      - message-bus-kafka
       - interoperability-layer-openhim
       - reverse-proxy-nginx
       - fhir-datastore-hapi-fhir
-      - message-bus-kafka
       - job-scheduler-ofelia
-      - kafka-mapper-consumer
-      - kafka-unbundler-consumer
       - analytics-datastore-clickhouse
       - dashboard-visualiser-superset
       - monitoring
       - mpi-mediator
       - client-registry-jempi
       - identity-access-manager-keycloak
       - openhim-mapping-mediator
+      - kafka-mapper-consumer
+      - kafka-unbundler-consumer
     envFiles:
       - cdr-dw.env
 

dashboard-visualiser-superset/config/requirements-local.txt

@@ -1,5 +1,5 @@
 psycopg2>=2.9.9
 clickhouse-connect>=0.7.0
-flask-oidc>=1.3.0
-itsdangerous>=2.0.1
-flask_openid>=1.3.0
+flask-oidc==1.3.0
+itsdangerous==2.0.1
+flask_openid==1.3.0

infrastructure/terraform/security.tf

@@ -15,13 +15,6 @@ resource "aws_security_group" "docker_swarm_sg" {
     cidr_blocks = [
       "0.0.0.0/0"]
   }
-  ingress {
-    from_port = 80
-    to_port = 80
-    protocol = "tcp"
-    cidr_blocks = [
-      "0.0.0.0/0"]
-  }
   ingress {
     from_port = 443
     to_port = 443

mpi-mediator/docker-compose.yml

@@ -2,7 +2,7 @@ version: '3.9'
 
 services:
   mpi-mediator:
-    image: jembi/mpi-mediator:v2.1.0
+    image: jembi/mpi-mediator:v2.1.1
     networks:
       openhim:
       kafka:

mpi-mediator/importer/volume/openhim-import.json

@@ -189,6 +189,73 @@
         "id": "65d47e98223a0672c3c9808b",
         "name": "Super User"
       }
+    },
+    {
+      "name": "JeMPI Patient links endpoint in fhir",
+      "description": "JeMPI Patient endpoint for retrieving  using the patient interaction id",
+      "urlPattern": "^/fhir/links/Patient/?[^/]*$",
+      "isAsynchronousProcess": false,
+      "methods": [
+        "GET",
+        "POST",
+        "DELETE",
+        "PUT",
+        "OPTIONS",
+        "HEAD",
+        "TRACE",
+        "CONNECT",
+        "PATCH"
+      ],
+      "type": "http",
+      "tcpPort": null,
+      "tcpHost": null,
+      "pollingSchedule": null,
+      "requestBody": true,
+      "responseBody": true,
+      "allow": [
+        "instant"
+      ],
+      "whitelist": [],
+      "authType": "private",
+      "routes": [
+        {
+          "name": "Generic Mapping Mediator",
+          "type": "http",
+          "status": "enabled",
+          "secured": false,
+          "host": "openhim-mapping-mediator",
+          "port": 3003,
+          "path": "",
+          "pathTransform": "",
+          "primary": true,
+          "username": "",
+          "password": "",
+          "forwardAuthHeader": true,
+          "waitPrimaryResponse": false,
+          "statusCodesCheck": "2**"
+        }
+      ],
+      "matchContentTypes": [],
+      "matchContentRegex": null,
+      "matchContentXpath": null,
+      "matchContentJson": null,
+      "matchContentValue": null,
+      "properties": [],
+      "txViewAcl": [],
+      "txViewFullAcl": [],
+      "txRerunAcl": [],
+      "alerts": [],
+      "status": "enabled",
+      "rewriteUrls": false,
+      "addAutoRewriteRules": true,
+      "rewriteUrlsConfig": [],
+      "autoRetryEnabled": false,
+      "autoRetryPeriodMinutes": 60,
+      "updatedBy": {
+        "id": "661e3c1206d33f9dfbcb3094",
+        "name": "test test"
+      },
+      "priority": 1
     }
   ],
   "Mediators": [

test/cucumber/features/cluster-mode/recipe.cluster.feature

@@ -0,0 +1,34 @@
+Feature: CDR recipe?
+    Does the recipe work as expected  
+
+Scenario: Init the CDR recipe
+    Given I use parameters "package init -p cdr-dw --dev --env-file=.env.cluster"
+    When I launch the platform with params
+    Then The service "mongo-1" should be started with 1 replica
+    And The service "openhim-core" should be started with 3 replica
+    And The service "openhim-console" should be started with 3 replica
+    And The service "kafka-unbundler-consumer" should be started with 1 replica
+    And The service "kafka-mapper-consumer" should be started with 1 replica
+
+Scenario: Send Fhir bundle and store the clinical data in the Fhir datastore, and the patient info in the CR
+    Given I have configured the cdr
+    When I send a fhir patient bundle
+    Then the clinical data should be stored in hapi fhir
+    And the patient data in the Jempi client registry
+    And the data should be stored in clickhouse
+
+Scenario: Fetch International Patient summary (IPS)
+    When I send a fhir patient bundle
+    And I then send a fhir patient summary request
+    Then I should get a successful summary response
+
+Scenario: Fetch everything for a patient (all the clinical data)
+    When I send a fhir patient bundle
+    And I then send a request for all the patient's clinical data
+    Then I should get a successful everything response
+
+Scenario: Destroy the services
+    Given I use parameters "package remove -p cdr-dw --env-file=.env.cluster"
+    When I launch the platform with params
+    Then There should be 0 service
+    And There should be 0 volume