From 6103191ed71cb13f7928001e7305215cfe606b23 Mon Sep 17 00:00:00 2001 From: Arran Standish Date: Wed, 5 Jul 2023 08:47:13 +0200 Subject: [PATCH 01/20] Add pgpool as a psql connection manager --- .env.cluster | 2 +- analytics-datastore-elastic-search/swarm.sh | 2 +- .../config/custom_pgpool.conf | 24 +++++ .../config/pgpool_node_id0 | 1 + .../config/pgpool_node_id1 | 1 + .../config/pgpool_node_id2 | 1 + .../docker-compose-postgres.cluster.yml | 99 ++++++++++++++++++- .../package-metadata.json | 5 +- 8 files changed, 131 insertions(+), 4 deletions(-) create mode 100644 fhir-datastore-hapi-fhir/config/custom_pgpool.conf create mode 100644 fhir-datastore-hapi-fhir/config/pgpool_node_id0 create mode 100644 fhir-datastore-hapi-fhir/config/pgpool_node_id1 create mode 100644 fhir-datastore-hapi-fhir/config/pgpool_node_id2 diff --git a/.env.cluster b/.env.cluster index c13bed3d..9aa4ba8e 100644 --- a/.env.cluster +++ b/.env.cluster @@ -23,7 +23,7 @@ REPMGR_PARTNER_NODES=postgres-1,postgres-2,postgres-3 POSTGRES_REPLICA_SET=postgres-1:5432,postgres-2:5432,postgres-3:5432 # Sante Datastore - Sante MPI -REPMGR_PARTNER_NODES=santempi-psql-1,santempi-psql-2,santempi-psql-3 +SANTEMPI_REPMGR_PARTNER_NODES=santempi-psql-1,santempi-psql-2,santempi-psql-3 # Reverse Proxy - Nginx REVERSE_PROXY_INSTANCES=3 diff --git a/analytics-datastore-elastic-search/swarm.sh b/analytics-datastore-elastic-search/swarm.sh index 5eace35d..6c7bef4a 100644 --- a/analytics-datastore-elastic-search/swarm.sh +++ b/analytics-datastore-elastic-search/swarm.sh @@ -78,7 +78,7 @@ function create_certs() { function add_docker_configs() { local -r TIMESTAMP="$(date "+%Y%m%d%H%M%S")" - local -r path_config_certs="/usr/share/elasticsearch/config/certs/" + local -r path_config_certs="/usr/share/elasticsearch/config/certs" log info "Creating configs..." diff --git a/fhir-datastore-hapi-fhir/config/custom_pgpool.conf b/fhir-datastore-hapi-fhir/config/custom_pgpool.conf new file mode 100644 index 00000000..32ef1f77 --- /dev/null +++ b/fhir-datastore-hapi-fhir/config/custom_pgpool.conf @@ -0,0 +1,24 @@ +failover_on_backend_error='on' +detach_false_primary='on' +auto_failback='on' +health_check_period=25 +health_check_max_retries=3 +health_check_user='postgres' + +use_watchdog='on' +hostname0='pgpool-1' +wd_port0=9000 +pgpool_port0=9999 +hostname1='pgpool-2' +wd_port1=9000 +pgpool_port1=9999 +hostname2='pgpool-3' +wd_port2=9000 +pgpool_port2=9999 +heartbeat_hostname0='pgpool-1' +heartbeat_port0=9694 +heartbeat_hostname1='pgpool-2' +heartbeat_port1=9694 +heartbeat_hostname2='pgpool-3' +heartbeat_port2=9694 +trusted_servers='pgpool-1,pgpool-2,pgpool-3' diff --git a/fhir-datastore-hapi-fhir/config/pgpool_node_id0 b/fhir-datastore-hapi-fhir/config/pgpool_node_id0 new file mode 100644 index 00000000..573541ac --- /dev/null +++ b/fhir-datastore-hapi-fhir/config/pgpool_node_id0 @@ -0,0 +1 @@ +0 diff --git a/fhir-datastore-hapi-fhir/config/pgpool_node_id1 b/fhir-datastore-hapi-fhir/config/pgpool_node_id1 new file mode 100644 index 00000000..d00491fd --- /dev/null +++ b/fhir-datastore-hapi-fhir/config/pgpool_node_id1 @@ -0,0 +1 @@ +1 diff --git a/fhir-datastore-hapi-fhir/config/pgpool_node_id2 b/fhir-datastore-hapi-fhir/config/pgpool_node_id2 new file mode 100644 index 00000000..0cfbf088 --- /dev/null +++ b/fhir-datastore-hapi-fhir/config/pgpool_node_id2 @@ -0,0 +1 @@ +2 diff --git a/fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml b/fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml index 1cd86405..26efbd3b 100644 --- a/fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml +++ b/fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml @@ -21,6 +21,7 @@ services: REPMGR_NODE_NAME: postgres-2 REPMGR_PRIMARY_HOST: ${REPMGR_PRIMARY_HOST} REPMGR_PARTNER_NODES: ${REPMGR_PARTNER_NODES} + POSTGRESQL_POSTGRES_PASSWORD: ${HAPI_POSTGRES_PASSWORD} volumes: - 'hapi-postgres-2-data:/bitnami/postgresql' deploy: @@ -40,7 +41,6 @@ services: hapi-fhir-postgres: pg_backup_net: {} - postgres-3: image: bitnami/postgresql-repmgr:14 environment: @@ -53,6 +53,7 @@ services: REPMGR_NODE_NAME: postgres-3 REPMGR_PRIMARY_HOST: ${REPMGR_PRIMARY_HOST} REPMGR_PARTNER_NODES: ${REPMGR_PARTNER_NODES} + POSTGRESQL_POSTGRES_PASSWORD: ${HAPI_POSTGRES_PASSWORD} volumes: - 'hapi-postgres-3-data:/bitnami/postgresql' deploy: @@ -72,6 +73,102 @@ services: hapi-fhir-postgres: pg_backup_net: {} + pgpool-1: + image: bitnami/pgpool:4.4.3 + deploy: + placement: + max_replicas_per_node: 1 + configs: + - target: /config/custom_pgpool.conf + source: pgpool.conf + - target: /opt/bitnami/pgpool/conf/pgpool_node_id + source: pgpool_node_id0.conf + environment: + PGPOOL_BACKEND_NODES: ${HF_PGPOOL_NODES} + PGPOOL_BACKEND_APPLICATION_NAMES: ${REPMGR_PARTNER_NODES} + PGPOOL_SR_CHECK_USER: postgres + PGPOOL_SR_CHECK_PASSWORD: ${HAPI_POSTGRES_PASSWORD} + PGPOOL_POSTGRES_USERNAME: postgres + PGPOOL_POSTGRES_PASSWORD: ${HAPI_POSTGRES_PASSWORD} + PGPOOL_ADMIN_USERNAME: ${HF_PGPOOL_USERNAME} + PGPOOL_ADMIN_PASSWORD: ${HF_PGPOOL_PASSWORD} + PGPOOL_ENABLE_LOAD_BALANCING: "no" + PGPOOL_AUTO_FAILBACK: "yes" + PGPOOL_USER_CONF_FILE: "/config/custom_pgpool.conf" + networks: + default: + + pgpool-2: + image: bitnami/pgpool:4.4.3 + deploy: + placement: + max_replicas_per_node: 1 + configs: + - target: /config/custom_pgpool.conf + source: pgpool.conf + - target: /opt/bitnami/pgpool/conf/pgpool_node_id + source: pgpool_node_id1.conf + environment: + PGPOOL_BACKEND_NODES: ${HF_PGPOOL_NODES} + PGPOOL_BACKEND_APPLICATION_NAMES: ${REPMGR_PARTNER_NODES} + PGPOOL_SR_CHECK_USER: postgres + PGPOOL_SR_CHECK_PASSWORD: ${HAPI_POSTGRES_PASSWORD} + PGPOOL_POSTGRES_USERNAME: postgres + PGPOOL_POSTGRES_PASSWORD: ${HAPI_POSTGRES_PASSWORD} + PGPOOL_ADMIN_USERNAME: ${HF_PGPOOL_USERNAME} + PGPOOL_ADMIN_PASSWORD: ${HF_PGPOOL_PASSWORD} + PGPOOL_ENABLE_LOAD_BALANCING: "no" + PGPOOL_AUTO_FAILBACK: "yes" + PGPOOL_USER_CONF_FILE: "/config/custom_pgpool.conf" + networks: + default: + + pgpool-3: + image: bitnami/pgpool:4.4.3 + deploy: + placement: + max_replicas_per_node: 1 + configs: + - target: /config/custom_pgpool.conf + source: pgpool.conf + - target: /opt/bitnami/pgpool/conf/pgpool_node_id + source: pgpool_node_id2.conf + environment: + PGPOOL_BACKEND_NODES: ${HF_PGPOOL_NODES} + PGPOOL_BACKEND_APPLICATION_NAMES: ${REPMGR_PARTNER_NODES} + PGPOOL_SR_CHECK_USER: postgres + PGPOOL_SR_CHECK_PASSWORD: ${HAPI_POSTGRES_PASSWORD} + PGPOOL_POSTGRES_USERNAME: postgres + PGPOOL_POSTGRES_PASSWORD: ${HAPI_POSTGRES_PASSWORD} + PGPOOL_ADMIN_USERNAME: ${HF_PGPOOL_USERNAME} + PGPOOL_ADMIN_PASSWORD: ${HF_PGPOOL_PASSWORD} + PGPOOL_ENABLE_LOAD_BALANCING: "no" + PGPOOL_AUTO_FAILBACK: "yes" + PGPOOL_USER_CONF_FILE: "/config/custom_pgpool.conf" + networks: + default: + +configs: + pgpool.conf: + name: pgpool_conf-${pgpool_conf_DIGEST:?err} + file: ./config/custom_pgpool.conf + labels: + name: hapi-fhir + pgpool_node_id0.conf: + name: pgpool_node_id0-${pgpool_node_id0_DIGEST:?err} + file: ./config/pgpool_node_id0 + labels: + name: hapi-fhir + pgpool_node_id1.conf: + name: pgpool_node_id1-${pgpool_node_id1_DIGEST:?err} + file: ./config/pgpool_node_id1 + labels: + name: hapi-fhir + pgpool_node_id2.conf: + name: pgpool_node_id2-${pgpool_node_id2_DIGEST:?err} + file: ./config/pgpool_node_id2 + labels: + name: hapi-fhir volumes: hapi-postgres-2-data: diff --git a/fhir-datastore-hapi-fhir/package-metadata.json b/fhir-datastore-hapi-fhir/package-metadata.json index 719e386d..ba610825 100644 --- a/fhir-datastore-hapi-fhir/package-metadata.json +++ b/fhir-datastore-hapi-fhir/package-metadata.json @@ -22,6 +22,9 @@ "HF_POSTGRES_MEMORY_RESERVE": "500M", "HF_POSTGRESQL_PASSWORD": "instant101", "HF_POSTGRESQL_USERNAME": "admin", - "HF_POSTGRESQL_DATABASE": "hapi" + "HF_POSTGRESQL_DATABASE": "hapi", + "HF_PGPOOL_NODES": "0:postgres-1:5432,1:postgres-2:5432,2:postgres-3:5432", + "HF_PGPOOL_USERNAME": "pgadmin", + "HF_PGPOOL_PASSWORD": "dev_password_only" } } From 3d5fbdca6173e16775982ae9779681dba68e9fd8 Mon Sep 17 00:00:00 2001 From: Arran Standish Date: Wed, 5 Jul 2023 08:50:34 +0200 Subject: [PATCH 02/20] Add pgpool services to hapi-fhir cluster tests --- .../cluster-mode/hapi-fhir.cluster.feature | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/test/cucumber/features/cluster-mode/hapi-fhir.cluster.feature b/test/cucumber/features/cluster-mode/hapi-fhir.cluster.feature index 9da91314..523a2c82 100644 --- a/test/cucumber/features/cluster-mode/hapi-fhir.cluster.feature +++ b/test/cucumber/features/cluster-mode/hapi-fhir.cluster.feature @@ -13,17 +13,24 @@ Feature: Fhir Datastore HAPI-FHIR? And The service "postgres-3" should be started with 1 replica And The service "postgres-3" should be connected to the networks | hapi-fhir_postgres_public | hapi-fhir_default | pg_backup | + And The service "postgres-3" should be started with 1 replica + And The service "pgpool-1" should be connected to the networks + | hapi-fhir_default | + And The service "pgpool-2" should be connected to the networks + | hapi-fhir_default | + And The service "pgpool-3" should be connected to the networks + | hapi-fhir_default | And The service "hapi-fhir" should be started with 3 replicas And The service "hapi-fhir" should be connected to the networks | mpi_public | hapi-fhir_public | hapi-fhir_default | - And There should be 4 services + And There should be 7 services And The service "hapi-fhir" should have healthy containers Scenario: Init Message Bus Helper Hapi Proxy Given I use parameters "package init -n=message-bus-helper-hapi-proxy --only --dev --env-file=.env.cluster" When I launch the platform with params Then The service "hapi-proxy" should be started with 3 replicas - And There should be 5 services + And There should be 8 services And The service "hapi-proxy" should be connected to the networks | hapi-fhir_public | kafka_public | openhim_public | @@ -33,6 +40,9 @@ Feature: Fhir Datastore HAPI-FHIR? Then The service "postgres-1" should be removed And The service "postgres-2" should be removed And The service "postgres-3" should be removed + And The service "pgpool-1" should be removed + And The service "pgpool-2" should be removed + And The service "pgpool-3" should be removed And The service "hapi-fhir" should be removed And The service "hapi-proxy" should be removed And There should be 0 service From 2e87e2931040be6f2184a90b4024757ea9614f0e Mon Sep 17 00:00:00 2001 From: Arran Standish Date: Wed, 5 Jul 2023 09:17:12 +0200 Subject: [PATCH 03/20] Add resource limits to pgpool --- .../docker-compose-postgres.cluster.yml | 21 +++++++++++++++++++ .../package-metadata.json | 6 +++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml b/fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml index 26efbd3b..ce673ded 100644 --- a/fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml +++ b/fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml @@ -78,6 +78,13 @@ services: deploy: placement: max_replicas_per_node: 1 + resources: + limits: + cpus: ${HF_PGPOOL_CPU_LIMIT} + memory: ${HF_PGPOOL_MEMORY_LIMIT} + reservations: + cpus: ${HF_PGPOOL_CPU_RESERVE} + memory: ${HF_PGPOOL_MEMORY_RESERVE} configs: - target: /config/custom_pgpool.conf source: pgpool.conf @@ -103,6 +110,13 @@ services: deploy: placement: max_replicas_per_node: 1 + resources: + limits: + cpus: ${HF_PGPOOL_CPU_LIMIT} + memory: ${HF_PGPOOL_MEMORY_LIMIT} + reservations: + cpus: ${HF_PGPOOL_CPU_RESERVE} + memory: ${HF_PGPOOL_MEMORY_RESERVE} configs: - target: /config/custom_pgpool.conf source: pgpool.conf @@ -128,6 +142,13 @@ services: deploy: placement: max_replicas_per_node: 1 + resources: + limits: + cpus: ${HF_PGPOOL_CPU_LIMIT} + memory: ${HF_PGPOOL_MEMORY_LIMIT} + reservations: + cpus: ${HF_PGPOOL_CPU_RESERVE} + memory: ${HF_PGPOOL_MEMORY_RESERVE} configs: - target: /config/custom_pgpool.conf source: pgpool.conf diff --git a/fhir-datastore-hapi-fhir/package-metadata.json b/fhir-datastore-hapi-fhir/package-metadata.json index ba610825..7108c91b 100644 --- a/fhir-datastore-hapi-fhir/package-metadata.json +++ b/fhir-datastore-hapi-fhir/package-metadata.json @@ -25,6 +25,10 @@ "HF_POSTGRESQL_DATABASE": "hapi", "HF_PGPOOL_NODES": "0:postgres-1:5432,1:postgres-2:5432,2:postgres-3:5432", "HF_PGPOOL_USERNAME": "pgadmin", - "HF_PGPOOL_PASSWORD": "dev_password_only" + "HF_PGPOOL_PASSWORD": "dev_password_only", + "HF_PGPOOL_CPU_LIMIT": "0", + "HF_PGPOOL_MEMORY_LIMIT": "250M", + "HF_PGPOOL_CPU_RESERVE": "0.05", + "HF_PGPOOL_MEMORY_RESERVE": "50M" } } From 945d2656bbaa41eb73d2ef4e6b0ad4ede5f822e9 Mon Sep 17 00:00:00 2001 From: Arran Standish Date: Fri, 7 Jul 2023 07:59:33 +0200 Subject: [PATCH 04/20] Update connection string info to point to pgpool instead for cluster mode --- .env.cluster | 4 ++-- .../docker-compose-postgres.cluster.yml | 6 ++++++ fhir-datastore-hapi-fhir/docker-compose.yml | 4 ++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.env.cluster b/.env.cluster index 9aa4ba8e..0cc6c357 100644 --- a/.env.cluster +++ b/.env.cluster @@ -4,7 +4,7 @@ CLUSTERED_MODE=true # Log -DEBUG=1 +DEBUG=0 BASHLOG_FILE=0 BASHLOG_FILE_PATH=platform.log @@ -20,7 +20,7 @@ OPENHIM_MONGO_ATNAURL=mongodb://mongo-1:27017,mongo-2:27017,mongo-3:27017/openhi # FHIR Datastore - HAPI FHIR HAPI_FHIR_INSTANCES=3 REPMGR_PARTNER_NODES=postgres-1,postgres-2,postgres-3 -POSTGRES_REPLICA_SET=postgres-1:5432,postgres-2:5432,postgres-3:5432 +POSTGRES_REPLICA_SET=pgpool-1:5432,pgpool-2:5432,pgpool-3:5432 # Sante Datastore - Sante MPI SANTEMPI_REPMGR_PARTNER_NODES=santempi-psql-1,santempi-psql-2,santempi-psql-3 diff --git a/fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml b/fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml index ce673ded..2adffd78 100644 --- a/fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml +++ b/fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml @@ -99,6 +99,8 @@ services: PGPOOL_POSTGRES_PASSWORD: ${HAPI_POSTGRES_PASSWORD} PGPOOL_ADMIN_USERNAME: ${HF_PGPOOL_USERNAME} PGPOOL_ADMIN_PASSWORD: ${HF_PGPOOL_PASSWORD} + PGPOOL_POSTGRES_CUSTOM_USERS: ${HF_POSTGRESQL_USERNAME} + PGPOOL_POSTGRES_CUSTOM_PASSWORDS: ${HF_POSTGRESQL_PASSWORD} PGPOOL_ENABLE_LOAD_BALANCING: "no" PGPOOL_AUTO_FAILBACK: "yes" PGPOOL_USER_CONF_FILE: "/config/custom_pgpool.conf" @@ -131,6 +133,8 @@ services: PGPOOL_POSTGRES_PASSWORD: ${HAPI_POSTGRES_PASSWORD} PGPOOL_ADMIN_USERNAME: ${HF_PGPOOL_USERNAME} PGPOOL_ADMIN_PASSWORD: ${HF_PGPOOL_PASSWORD} + PGPOOL_POSTGRES_CUSTOM_USERS: ${HF_POSTGRESQL_USERNAME} + PGPOOL_POSTGRES_CUSTOM_PASSWORDS: ${HF_POSTGRESQL_PASSWORD} PGPOOL_ENABLE_LOAD_BALANCING: "no" PGPOOL_AUTO_FAILBACK: "yes" PGPOOL_USER_CONF_FILE: "/config/custom_pgpool.conf" @@ -163,6 +167,8 @@ services: PGPOOL_POSTGRES_PASSWORD: ${HAPI_POSTGRES_PASSWORD} PGPOOL_ADMIN_USERNAME: ${HF_PGPOOL_USERNAME} PGPOOL_ADMIN_PASSWORD: ${HF_PGPOOL_PASSWORD} + PGPOOL_POSTGRES_CUSTOM_USERS: ${HF_POSTGRESQL_USERNAME} + PGPOOL_POSTGRES_CUSTOM_PASSWORDS: ${HF_POSTGRESQL_PASSWORD} PGPOOL_ENABLE_LOAD_BALANCING: "no" PGPOOL_AUTO_FAILBACK: "yes" PGPOOL_USER_CONF_FILE: "/config/custom_pgpool.conf" diff --git a/fhir-datastore-hapi-fhir/docker-compose.yml b/fhir-datastore-hapi-fhir/docker-compose.yml index 57b73e7e..e6ffe82b 100644 --- a/fhir-datastore-hapi-fhir/docker-compose.yml +++ b/fhir-datastore-hapi-fhir/docker-compose.yml @@ -9,8 +9,8 @@ services: default: environment: - spring.datasource.url=jdbc:postgresql://${POSTGRES_REPLICA_SET}/hapi?targetServerType=primary - - spring.datasource.username=admin - - spring.datasource.password=instant101 + - spring.datasource.username=${HF_POSTGRESQL_USERNAME} + - spring.datasource.password=${HF_POSTGRESQL_PASSWORD} - spring.datasource.driverClassName=org.postgresql.Driver - spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.PostgreSQL95Dialect - hapi.fhir.allow_external_references=true From d03b90ba9eeb947c81d98dbd53f62d2c9516f37c Mon Sep 17 00:00:00 2001 From: Arran Standish Date: Fri, 7 Jul 2023 09:48:33 +0200 Subject: [PATCH 05/20] Increase available postgres connections and add 5min idle + lieftime limit --- fhir-datastore-hapi-fhir/config/custom_pgpool.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fhir-datastore-hapi-fhir/config/custom_pgpool.conf b/fhir-datastore-hapi-fhir/config/custom_pgpool.conf index 32ef1f77..88a7ce68 100644 --- a/fhir-datastore-hapi-fhir/config/custom_pgpool.conf +++ b/fhir-datastore-hapi-fhir/config/custom_pgpool.conf @@ -4,6 +4,11 @@ auto_failback='on' health_check_period=25 health_check_max_retries=3 health_check_user='postgres' +reserved_connections=1 +max_pool=6 +client_idle_limit=300 +connection_life_time=300 +child_max_connections=25 use_watchdog='on' hostname0='pgpool-1' From 19ab0a2367ce85fbde9b672483d55a5c5c44e86e Mon Sep 17 00:00:00 2001 From: Arran Standish Date: Fri, 7 Jul 2023 09:49:01 +0200 Subject: [PATCH 06/20] Increase memory limit a bit as 250 was getting killed due to OOM --- fhir-datastore-hapi-fhir/package-metadata.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fhir-datastore-hapi-fhir/package-metadata.json b/fhir-datastore-hapi-fhir/package-metadata.json index 7108c91b..2f0d75b5 100644 --- a/fhir-datastore-hapi-fhir/package-metadata.json +++ b/fhir-datastore-hapi-fhir/package-metadata.json @@ -27,7 +27,7 @@ "HF_PGPOOL_USERNAME": "pgadmin", "HF_PGPOOL_PASSWORD": "dev_password_only", "HF_PGPOOL_CPU_LIMIT": "0", - "HF_PGPOOL_MEMORY_LIMIT": "250M", + "HF_PGPOOL_MEMORY_LIMIT": "500M", "HF_PGPOOL_CPU_RESERVE": "0.05", "HF_PGPOOL_MEMORY_RESERVE": "50M" } From 955824918d80921f0cfac0910707ff7236b90045 Mon Sep 17 00:00:00 2001 From: Arran Standish Date: Mon, 10 Jul 2023 12:13:58 +0200 Subject: [PATCH 07/20] Update healthcheck to declare hapi-fhir dead if cannot connect to postgres --- fhir-datastore-hapi-fhir/docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fhir-datastore-hapi-fhir/docker-compose.yml b/fhir-datastore-hapi-fhir/docker-compose.yml index e6ffe82b..43fe2bd6 100644 --- a/fhir-datastore-hapi-fhir/docker-compose.yml +++ b/fhir-datastore-hapi-fhir/docker-compose.yml @@ -30,10 +30,10 @@ services: cpus: ${HAPI_FHIR_CPU_RESERVE} memory: ${HAPI_FHIR_MEMORY_RESERVE} healthcheck: - test: /bin/wget --no-verbose --tries=1 --spider http://localhost:8080 + test: /bin/wget --no-verbose --tries=1 --spider http://localhost:8080/fhir/Organization?identifier=urn:healthcheck interval: 30s - timeout: 5s - retries: 5 + timeout: 30s + retries: 2 start_period: 2m networks: From 9e0d7494565fb94a3b3b468fbe2007dc8bcbc565 Mon Sep 17 00:00:00 2001 From: nour-borgi Date: Mon, 10 Jul 2023 16:01:09 +0100 Subject: [PATCH 08/20] Limit memory usage of JVM in Hapi proxy to 90% --- message-bus-helper-hapi-proxy/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/message-bus-helper-hapi-proxy/docker-compose.yml b/message-bus-helper-hapi-proxy/docker-compose.yml index 0ee3f1f0..8dc29e46 100644 --- a/message-bus-helper-hapi-proxy/docker-compose.yml +++ b/message-bus-helper-hapi-proxy/docker-compose.yml @@ -8,6 +8,7 @@ services: HAPI_SERVER_URL: ${HAPI_SERVER_URL} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_HOSTS} HAPI_SERVER_VALIDATE_FORMAT: ${HAPI_SERVER_VALIDATE_FORMAT} + JAVA_OPTS: "-XX:MaxRAMPercentage=90" deploy: placement: max_replicas_per_node: 1 From fa6ff41110cea12ddc18613d98668552f58a74d4 Mon Sep 17 00:00:00 2001 From: Arran Standish Date: Tue, 11 Jul 2023 09:12:14 +0200 Subject: [PATCH 09/20] Rename POSTGRES_REPLICA_SET to be more descriptive --- .env.cluster | 2 +- .env.remote | 3 +-- .../docker-compose-postgres.cluster.yml | 7 +++---- fhir-datastore-hapi-fhir/docker-compose.yml | 2 +- fhir-datastore-hapi-fhir/package-metadata.json | 2 +- .../features/cluster-mode/hapi-fhir.cluster.feature | 10 +++++----- 6 files changed, 12 insertions(+), 14 deletions(-) diff --git a/.env.cluster b/.env.cluster index 0cc6c357..af5dd7e7 100644 --- a/.env.cluster +++ b/.env.cluster @@ -20,7 +20,7 @@ OPENHIM_MONGO_ATNAURL=mongodb://mongo-1:27017,mongo-2:27017,mongo-3:27017/openhi # FHIR Datastore - HAPI FHIR HAPI_FHIR_INSTANCES=3 REPMGR_PARTNER_NODES=postgres-1,postgres-2,postgres-3 -POSTGRES_REPLICA_SET=pgpool-1:5432,pgpool-2:5432,pgpool-3:5432 +HAPI_DB_SET=pgpool-1:5432,pgpool-2:5432,pgpool-3:5432 # Sante Datastore - Sante MPI SANTEMPI_REPMGR_PARTNER_NODES=santempi-psql-1,santempi-psql-2,santempi-psql-3 diff --git a/.env.remote b/.env.remote index 8dea7b25..a56ff521 100644 --- a/.env.remote +++ b/.env.remote @@ -7,7 +7,6 @@ OPENHIM_TRANSACTION_API_PORT= OPENHIM_MEDIATOR_API_PORT= MOCK_SERVER_HOST= MOCK_SERVER_PORT= -POSTGRES_REPLICA_SET=postgres-1:5432,postgres-2:5432,postgres-3:5432 # Analytics Datastore - Elastic Search ES_HOSTS="\"analytics-datastore-elastic-search-01:9200","analytics-datastore-elastic-search-02:9200","analytics-datastore-elastic-search-03:9200\"" @@ -17,7 +16,7 @@ CLICKHOUSE_HOST=analytics-datastore-clickhouse-01 # FHIR Datastore - HAPI FHIR REPMGR_PARTNER_NODES=postgres-1,postgres-2,postgres-3 -POSTGRES_REPLICA_SET=postgres-1:5432,postgres-2:5432,postgres-3:5432 +HAPI_DB_SET=postgres-1:5432,postgres-2:5432,postgres-3:5432 # Interoperability Layer - OpenHIM MONGO_SET_COUNT=3 diff --git a/fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml b/fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml index 2adffd78..0579037e 100644 --- a/fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml +++ b/fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml @@ -38,8 +38,6 @@ services: memory: ${HF_POSTGRES_MEMORY_RESERVE} networks: default: - hapi-fhir-postgres: - pg_backup_net: {} postgres-3: image: bitnami/postgresql-repmgr:14 @@ -70,8 +68,6 @@ services: memory: ${HF_POSTGRES_MEMORY_RESERVE} networks: default: - hapi-fhir-postgres: - pg_backup_net: {} pgpool-1: image: bitnami/pgpool:4.4.3 @@ -106,6 +102,7 @@ services: PGPOOL_USER_CONF_FILE: "/config/custom_pgpool.conf" networks: default: + pg_backup_net: pgpool-2: image: bitnami/pgpool:4.4.3 @@ -140,6 +137,7 @@ services: PGPOOL_USER_CONF_FILE: "/config/custom_pgpool.conf" networks: default: + pg_backup_net: pgpool-3: image: bitnami/pgpool:4.4.3 @@ -174,6 +172,7 @@ services: PGPOOL_USER_CONF_FILE: "/config/custom_pgpool.conf" networks: default: + pg_backup_net: configs: pgpool.conf: diff --git a/fhir-datastore-hapi-fhir/docker-compose.yml b/fhir-datastore-hapi-fhir/docker-compose.yml index 43fe2bd6..217f9eed 100644 --- a/fhir-datastore-hapi-fhir/docker-compose.yml +++ b/fhir-datastore-hapi-fhir/docker-compose.yml @@ -8,7 +8,7 @@ services: public: default: environment: - - spring.datasource.url=jdbc:postgresql://${POSTGRES_REPLICA_SET}/hapi?targetServerType=primary + - spring.datasource.url=jdbc:postgresql://${HAPI_DB_SET}/hapi?targetServerType=primary - spring.datasource.username=${HF_POSTGRESQL_USERNAME} - spring.datasource.password=${HF_POSTGRESQL_PASSWORD} - spring.datasource.driverClassName=org.postgresql.Driver diff --git a/fhir-datastore-hapi-fhir/package-metadata.json b/fhir-datastore-hapi-fhir/package-metadata.json index 2f0d75b5..d47bf8c6 100644 --- a/fhir-datastore-hapi-fhir/package-metadata.json +++ b/fhir-datastore-hapi-fhir/package-metadata.json @@ -14,7 +14,7 @@ "HAPI_FHIR_CPU_RESERVE": "0.05", "HAPI_FHIR_MEMORY_LIMIT": "3G", "HAPI_FHIR_MEMORY_RESERVE": "500M", - "POSTGRES_REPLICA_SET": "postgres-1:5432", + "HAPI_DB_SET": "postgres-1:5432", "HAPI_POSTGRES_PASSWORD": "postgres", "HF_POSTGRES_CPU_LIMIT": "0", "HF_POSTGRES_CPU_RESERVE": "0.05", diff --git a/test/cucumber/features/cluster-mode/hapi-fhir.cluster.feature b/test/cucumber/features/cluster-mode/hapi-fhir.cluster.feature index 523a2c82..70f68256 100644 --- a/test/cucumber/features/cluster-mode/hapi-fhir.cluster.feature +++ b/test/cucumber/features/cluster-mode/hapi-fhir.cluster.feature @@ -9,17 +9,17 @@ Feature: Fhir Datastore HAPI-FHIR? | hapi-fhir_postgres_public | hapi-fhir_default | pg_backup | And The service "postgres-2" should be started with 1 replica And The service "postgres-2" should be connected to the networks - | hapi-fhir_postgres_public | hapi-fhir_default | pg_backup | + | hapi-fhir_default | And The service "postgres-3" should be started with 1 replica And The service "postgres-3" should be connected to the networks - | hapi-fhir_postgres_public | hapi-fhir_default | pg_backup | + | hapi-fhir_default | And The service "postgres-3" should be started with 1 replica And The service "pgpool-1" should be connected to the networks - | hapi-fhir_default | + | hapi-fhir_default| pg_backup | And The service "pgpool-2" should be connected to the networks - | hapi-fhir_default | + | hapi-fhir_default| pg_backup | And The service "pgpool-3" should be connected to the networks - | hapi-fhir_default | + | hapi-fhir_default| pg_backup | And The service "hapi-fhir" should be started with 3 replicas And The service "hapi-fhir" should be connected to the networks | mpi_public | hapi-fhir_public | hapi-fhir_default | From 7bb2f9a41d63e44b46891606e9b7b178e0f68f03 Mon Sep 17 00:00:00 2001 From: Arran Standish Date: Tue, 11 Jul 2023 10:02:17 +0200 Subject: [PATCH 10/20] Reintroduce the postgres_replica_set environment variable --- .env.cluster | 1 + .env.remote | 1 + .../docker-compose-postgres.cluster.yml | 5 ++--- fhir-datastore-hapi-fhir/package-metadata.json | 1 + .../features/cluster-mode/hapi-fhir.cluster.feature | 10 +++++----- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.env.cluster b/.env.cluster index af5dd7e7..6a849f6d 100644 --- a/.env.cluster +++ b/.env.cluster @@ -20,6 +20,7 @@ OPENHIM_MONGO_ATNAURL=mongodb://mongo-1:27017,mongo-2:27017,mongo-3:27017/openhi # FHIR Datastore - HAPI FHIR HAPI_FHIR_INSTANCES=3 REPMGR_PARTNER_NODES=postgres-1,postgres-2,postgres-3 +POSTGRES_REPLICA_SET=postgres-1:5432,postgres-2:5432,postgres-3:5432 HAPI_DB_SET=pgpool-1:5432,pgpool-2:5432,pgpool-3:5432 # Sante Datastore - Sante MPI diff --git a/.env.remote b/.env.remote index a56ff521..882cf140 100644 --- a/.env.remote +++ b/.env.remote @@ -16,6 +16,7 @@ CLICKHOUSE_HOST=analytics-datastore-clickhouse-01 # FHIR Datastore - HAPI FHIR REPMGR_PARTNER_NODES=postgres-1,postgres-2,postgres-3 +POSTGRES_REPLICA_SET=postgres-1:5432,postgres-2:5432,postgres-3:5432 HAPI_DB_SET=postgres-1:5432,postgres-2:5432,postgres-3:5432 # Interoperability Layer - OpenHIM diff --git a/fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml b/fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml index 0579037e..87ea782d 100644 --- a/fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml +++ b/fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml @@ -38,6 +38,7 @@ services: memory: ${HF_POSTGRES_MEMORY_RESERVE} networks: default: + pg_backup_net: postgres-3: image: bitnami/postgresql-repmgr:14 @@ -68,6 +69,7 @@ services: memory: ${HF_POSTGRES_MEMORY_RESERVE} networks: default: + pg_backup_net: pgpool-1: image: bitnami/pgpool:4.4.3 @@ -102,7 +104,6 @@ services: PGPOOL_USER_CONF_FILE: "/config/custom_pgpool.conf" networks: default: - pg_backup_net: pgpool-2: image: bitnami/pgpool:4.4.3 @@ -137,7 +138,6 @@ services: PGPOOL_USER_CONF_FILE: "/config/custom_pgpool.conf" networks: default: - pg_backup_net: pgpool-3: image: bitnami/pgpool:4.4.3 @@ -172,7 +172,6 @@ services: PGPOOL_USER_CONF_FILE: "/config/custom_pgpool.conf" networks: default: - pg_backup_net: configs: pgpool.conf: diff --git a/fhir-datastore-hapi-fhir/package-metadata.json b/fhir-datastore-hapi-fhir/package-metadata.json index d47bf8c6..af39a8f0 100644 --- a/fhir-datastore-hapi-fhir/package-metadata.json +++ b/fhir-datastore-hapi-fhir/package-metadata.json @@ -14,6 +14,7 @@ "HAPI_FHIR_CPU_RESERVE": "0.05", "HAPI_FHIR_MEMORY_LIMIT": "3G", "HAPI_FHIR_MEMORY_RESERVE": "500M", + "POSTGRES_REPLICA_SET": "postgres-1:5432", "HAPI_DB_SET": "postgres-1:5432", "HAPI_POSTGRES_PASSWORD": "postgres", "HF_POSTGRES_CPU_LIMIT": "0", diff --git a/test/cucumber/features/cluster-mode/hapi-fhir.cluster.feature b/test/cucumber/features/cluster-mode/hapi-fhir.cluster.feature index 70f68256..ba7ac63b 100644 --- a/test/cucumber/features/cluster-mode/hapi-fhir.cluster.feature +++ b/test/cucumber/features/cluster-mode/hapi-fhir.cluster.feature @@ -9,17 +9,17 @@ Feature: Fhir Datastore HAPI-FHIR? | hapi-fhir_postgres_public | hapi-fhir_default | pg_backup | And The service "postgres-2" should be started with 1 replica And The service "postgres-2" should be connected to the networks - | hapi-fhir_default | + | hapi-fhir_default | pg_backup | And The service "postgres-3" should be started with 1 replica And The service "postgres-3" should be connected to the networks - | hapi-fhir_default | + | hapi-fhir_default | pg_backup | And The service "postgres-3" should be started with 1 replica And The service "pgpool-1" should be connected to the networks - | hapi-fhir_default| pg_backup | + | hapi-fhir_default | And The service "pgpool-2" should be connected to the networks - | hapi-fhir_default| pg_backup | + | hapi-fhir_default | And The service "pgpool-3" should be connected to the networks - | hapi-fhir_default| pg_backup | + | hapi-fhir_default | And The service "hapi-fhir" should be started with 3 replicas And The service "hapi-fhir" should be connected to the networks | mpi_public | hapi-fhir_public | hapi-fhir_default | From 2a4bfa999898f4bc29aa58d10438aa938d28c503 Mon Sep 17 00:00:00 2001 From: Arran Standish Date: Tue, 11 Jul 2023 12:22:41 +0200 Subject: [PATCH 11/20] Update version of platform image --- config.yaml | 2 +- remote-img-load.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config.yaml b/config.yaml index bdff8925..d02396dd 100644 --- a/config.yaml +++ b/config.yaml @@ -1,5 +1,5 @@ projectName: platform -image: jembi/platform:2.3.1 +image: jembi/platform:2.4.0 logPath: /tmp/logs packages: diff --git a/remote-img-load.sh b/remote-img-load.sh index 9621ed37..9dca9ea8 100755 --- a/remote-img-load.sh +++ b/remote-img-load.sh @@ -21,5 +21,5 @@ fi ./build-image.sh "$TAG_NAME" echo "Transfering image to ${HOST} ..." -docker save jembi/platform:latest | bzip2 | ssh "${USER}@${HOST}" docker load +docker save jembi/platform:$TAG_NAME | bzip2 | ssh "${USER}@${HOST}" docker load echo "Image transfered" From a6b3003d6532ffd47384395d7f500fe91259c264 Mon Sep 17 00:00:00 2001 From: Arran Standish Date: Tue, 11 Jul 2023 13:53:06 +0200 Subject: [PATCH 12/20] Update github actions to update config.yaml image tag to match the remote loaded one --- .github/workflows/ci-cluster.yml | 2 ++ .github/workflows/ci.yml | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-cluster.yml b/.github/workflows/ci-cluster.yml index c3c3386d..b99a22e0 100644 --- a/.github/workflows/ci-cluster.yml +++ b/.github/workflows/ci-cluster.yml @@ -93,6 +93,8 @@ jobs: - run: ssh-keyscan -H ${{ github.run_id }}.jembi.cloud >> ~/.ssh/known_hosts - run: ./remote-img-load.sh ${{ github.run_id }}.jembi.cloud - run: sed -i 's/domain/${{ github.run_id }}.jembi.cloud/g' .env.cluster + - run: >- + sed -i 's/^image: jembi\/platform:.*$/image: jembi\/platform:${{ github.run_id }}.jembi.cloud/' config.yaml - name: Install dependencies working-directory: ./test/cucumber run: yarn diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d50365a9..eb00c7a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,7 +95,8 @@ jobs: - run: sed -i 's/OPENHIM_MEDIATOR_API_PORT=8080/OPENHIM_MEDIATOR_API_PORT=443/g' .env.local - run: sed -i 's/INSECURE=true/INSECURE=false/g' .env.local - run: sed -i 's/domain/${{ github.run_id }}.jembi.cloud/g' .env.local - + - run: >- + sed -i 's/^image: jembi\/platform:.*$/image: jembi\/platform:${{ github.run_id }}.jembi.cloud/' config.yaml - name: Install dependencies working-directory: ./test/cucumber run: yarn From a58230d73d61f40298725720913a0b0f27d9350e Mon Sep 17 00:00:00 2001 From: nour-borgi Date: Tue, 11 Jul 2023 14:37:55 +0100 Subject: [PATCH 13/20] Fix JDBV connection issue Hapi Fhir --- fhir-datastore-hapi-fhir/docker-compose.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/fhir-datastore-hapi-fhir/docker-compose.yml b/fhir-datastore-hapi-fhir/docker-compose.yml index 57b73e7e..468e224c 100644 --- a/fhir-datastore-hapi-fhir/docker-compose.yml +++ b/fhir-datastore-hapi-fhir/docker-compose.yml @@ -12,6 +12,13 @@ services: - spring.datasource.username=admin - spring.datasource.password=instant101 - spring.datasource.driverClassName=org.postgresql.Driver + - spring.datasource.test-on-borrow=true + - spring.datasource.validation-query=select 1 + - spring.datasource.hikari.maximumPoolSize=1000 + - spring.datasource.hikari.minimumIdle=10 + - spring.datasource.hikari.idleTimeout=00000 + - spring.datasource.hikari.connectionTimeout=600000 + - spring.datasource.hikari.leakDetectionThreshold=500000 - spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.PostgreSQL95Dialect - hapi.fhir.allow_external_references=true - hapi.fhir.bulk_export_enabled=true From 1a674cfcf07c8dd3730c8d57226381b074299a80 Mon Sep 17 00:00:00 2001 From: nour-borgi Date: Tue, 11 Jul 2023 16:22:22 +0100 Subject: [PATCH 14/20] Update values --- fhir-datastore-hapi-fhir/docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fhir-datastore-hapi-fhir/docker-compose.yml b/fhir-datastore-hapi-fhir/docker-compose.yml index 468e224c..fa18c182 100644 --- a/fhir-datastore-hapi-fhir/docker-compose.yml +++ b/fhir-datastore-hapi-fhir/docker-compose.yml @@ -16,9 +16,9 @@ services: - spring.datasource.validation-query=select 1 - spring.datasource.hikari.maximumPoolSize=1000 - spring.datasource.hikari.minimumIdle=10 - - spring.datasource.hikari.idleTimeout=00000 - - spring.datasource.hikari.connectionTimeout=600000 - - spring.datasource.hikari.leakDetectionThreshold=500000 + - spring.datasource.hikari.idleTimeout=30000 + - spring.datasource.hikari.connectionTimeout=800000 + - spring.datasource.hikari.leakDetectionThreshold=900000 - spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.PostgreSQL95Dialect - hapi.fhir.allow_external_references=true - hapi.fhir.bulk_export_enabled=true From 1936fc99f5f164f214407cb0610aa0496d3ef99a Mon Sep 17 00:00:00 2001 From: nour-borgi Date: Wed, 12 Jul 2023 11:41:11 +0100 Subject: [PATCH 15/20] Fine tune config hapi fhir --- fhir-datastore-hapi-fhir/docker-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fhir-datastore-hapi-fhir/docker-compose.yml b/fhir-datastore-hapi-fhir/docker-compose.yml index 1aa8a6ca..f8ed433c 100644 --- a/fhir-datastore-hapi-fhir/docker-compose.yml +++ b/fhir-datastore-hapi-fhir/docker-compose.yml @@ -14,11 +14,11 @@ services: - spring.datasource.driverClassName=org.postgresql.Driver - spring.datasource.test-on-borrow=true - spring.datasource.validation-query=select 1 - - spring.datasource.hikari.maximumPoolSize=1000 + - spring.datasource.hikari.maximumPoolSize=500 - spring.datasource.hikari.minimumIdle=10 - - spring.datasource.hikari.idleTimeout=30000 - - spring.datasource.hikari.connectionTimeout=800000 - - spring.datasource.hikari.leakDetectionThreshold=900000 + - spring.datasource.hikari.idleTimeout=600000 + - spring.datasource.hikari.connectionTimeout=600000 + - spring.datasource.hikari.leakDetectionThreshold=600000 - spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.PostgreSQL95Dialect - hapi.fhir.allow_external_references=true - hapi.fhir.bulk_export_enabled=true From 49ab46d3cdac229c956782640455e187080e751a Mon Sep 17 00:00:00 2001 From: Arran Standish Date: Mon, 17 Jul 2023 09:27:29 +0200 Subject: [PATCH 16/20] Update hikari settings to more realistic values --- fhir-datastore-hapi-fhir/docker-compose.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fhir-datastore-hapi-fhir/docker-compose.yml b/fhir-datastore-hapi-fhir/docker-compose.yml index 217f9eed..9c3002ac 100644 --- a/fhir-datastore-hapi-fhir/docker-compose.yml +++ b/fhir-datastore-hapi-fhir/docker-compose.yml @@ -13,6 +13,10 @@ services: - spring.datasource.password=${HF_POSTGRESQL_PASSWORD} - spring.datasource.driverClassName=org.postgresql.Driver - spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.PostgreSQL95Dialect + - spring.datasource.hikari.maximumPoolSize=250 + - spring.datasource.hikari.minimumIdle=20 + - spring.datasource.hikari.connectionTimeout=60000 + - spring.datasource.hikari.idleTimeout=600000 - hapi.fhir.allow_external_references=true - hapi.fhir.bulk_export_enabled=true - hapi.fhir.enable_repository_validating_interceptor=true From 2ef926e10ae822077fed0678d6118d958cfaeb39 Mon Sep 17 00:00:00 2001 From: Arran Standish Date: Mon, 17 Jul 2023 09:27:59 +0200 Subject: [PATCH 17/20] Parameterise hapi java options --- fhir-datastore-hapi-fhir/docker-compose.yml | 4 ++-- fhir-datastore-hapi-fhir/package-metadata.json | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/fhir-datastore-hapi-fhir/docker-compose.yml b/fhir-datastore-hapi-fhir/docker-compose.yml index 9c3002ac..c9ba03f0 100644 --- a/fhir-datastore-hapi-fhir/docker-compose.yml +++ b/fhir-datastore-hapi-fhir/docker-compose.yml @@ -20,8 +20,8 @@ services: - hapi.fhir.allow_external_references=true - hapi.fhir.bulk_export_enabled=true - hapi.fhir.enable_repository_validating_interceptor=true - - JAVA_TOOL_OPTIONS=-Xmx2g - - CATALINA_OPTS=-Xmx2g + - JAVA_TOOL_OPTIONS=${HF_JAVA_OPTS} + - CATALINA_OPTS=${HF_JAVA_OPTS} deploy: replicas: ${HAPI_FHIR_INSTANCES} placement: diff --git a/fhir-datastore-hapi-fhir/package-metadata.json b/fhir-datastore-hapi-fhir/package-metadata.json index af39a8f0..c2aa75bb 100644 --- a/fhir-datastore-hapi-fhir/package-metadata.json +++ b/fhir-datastore-hapi-fhir/package-metadata.json @@ -30,6 +30,7 @@ "HF_PGPOOL_CPU_LIMIT": "0", "HF_PGPOOL_MEMORY_LIMIT": "500M", "HF_PGPOOL_CPU_RESERVE": "0.05", - "HF_PGPOOL_MEMORY_RESERVE": "50M" + "HF_PGPOOL_MEMORY_RESERVE": "50M", + "HF_JAVA_OPTS": "-Xmx2g" } } From 548c5852f6d0bfb9e3b58fb29366385803507465 Mon Sep 17 00:00:00 2001 From: Arran Standish Date: Mon, 17 Jul 2023 09:31:29 +0200 Subject: [PATCH 18/20] Parameterise hapi max replica nodes to allow more than 3 total instances --- fhir-datastore-hapi-fhir/docker-compose.yml | 2 +- fhir-datastore-hapi-fhir/package-metadata.json | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/fhir-datastore-hapi-fhir/docker-compose.yml b/fhir-datastore-hapi-fhir/docker-compose.yml index c9ba03f0..3c104282 100644 --- a/fhir-datastore-hapi-fhir/docker-compose.yml +++ b/fhir-datastore-hapi-fhir/docker-compose.yml @@ -25,7 +25,7 @@ services: deploy: replicas: ${HAPI_FHIR_INSTANCES} placement: - max_replicas_per_node: 1 + max_replicas_per_node: ${HAPI_FHIR_MAX_REPLICAS_PER_NODE} resources: limits: cpus: ${HAPI_FHIR_CPU_LIMIT} diff --git a/fhir-datastore-hapi-fhir/package-metadata.json b/fhir-datastore-hapi-fhir/package-metadata.json index c2aa75bb..e267fb88 100644 --- a/fhir-datastore-hapi-fhir/package-metadata.json +++ b/fhir-datastore-hapi-fhir/package-metadata.json @@ -10,6 +10,7 @@ "REPMGR_PARTNER_NODES": "postgres-1", "REPMGR_PASSWORD": "instant101", "HAPI_FHIR_INSTANCES": "1", + "HAPI_FHIR_MAX_REPLICAS_PER_NODE": "1", "HAPI_FHIR_CPU_LIMIT": "0", "HAPI_FHIR_CPU_RESERVE": "0.05", "HAPI_FHIR_MEMORY_LIMIT": "3G", From 106b2b1741defda4d4cfd63b6883273925f719ae Mon Sep 17 00:00:00 2001 From: Arran Standish Date: Mon, 17 Jul 2023 09:33:07 +0200 Subject: [PATCH 19/20] Fix pull down merge issue --- fhir-datastore-hapi-fhir/docker-compose.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/fhir-datastore-hapi-fhir/docker-compose.yml b/fhir-datastore-hapi-fhir/docker-compose.yml index 794a1b7a..3c104282 100644 --- a/fhir-datastore-hapi-fhir/docker-compose.yml +++ b/fhir-datastore-hapi-fhir/docker-compose.yml @@ -12,13 +12,6 @@ services: - spring.datasource.username=${HF_POSTGRESQL_USERNAME} - spring.datasource.password=${HF_POSTGRESQL_PASSWORD} - spring.datasource.driverClassName=org.postgresql.Driver - - spring.datasource.test-on-borrow=true - - spring.datasource.validation-query=select 1 - - spring.datasource.hikari.maximumPoolSize=500 - - spring.datasource.hikari.minimumIdle=10 - - spring.datasource.hikari.idleTimeout=600000 - - spring.datasource.hikari.connectionTimeout=600000 - - spring.datasource.hikari.leakDetectionThreshold=600000 - spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.PostgreSQL95Dialect - spring.datasource.hikari.maximumPoolSize=250 - spring.datasource.hikari.minimumIdle=20 From 47193f72935c6ecd0172004062eed0ee2563a1cb Mon Sep 17 00:00:00 2001 From: Arran Standish Date: Wed, 26 Jul 2023 08:59:18 +0200 Subject: [PATCH 20/20] Remove CI sed changes as remote-image-load was updated instead --- .env.cluster | 2 +- .github/workflows/ci-cluster.yml | 2 -- .github/workflows/ci.yml | 2 -- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.env.cluster b/.env.cluster index 6a849f6d..5e8e70b7 100644 --- a/.env.cluster +++ b/.env.cluster @@ -4,7 +4,7 @@ CLUSTERED_MODE=true # Log -DEBUG=0 +DEBUG=1 BASHLOG_FILE=0 BASHLOG_FILE_PATH=platform.log diff --git a/.github/workflows/ci-cluster.yml b/.github/workflows/ci-cluster.yml index b99a22e0..c3c3386d 100644 --- a/.github/workflows/ci-cluster.yml +++ b/.github/workflows/ci-cluster.yml @@ -93,8 +93,6 @@ jobs: - run: ssh-keyscan -H ${{ github.run_id }}.jembi.cloud >> ~/.ssh/known_hosts - run: ./remote-img-load.sh ${{ github.run_id }}.jembi.cloud - run: sed -i 's/domain/${{ github.run_id }}.jembi.cloud/g' .env.cluster - - run: >- - sed -i 's/^image: jembi\/platform:.*$/image: jembi\/platform:${{ github.run_id }}.jembi.cloud/' config.yaml - name: Install dependencies working-directory: ./test/cucumber run: yarn diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb00c7a2..7ed7cedd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,8 +95,6 @@ jobs: - run: sed -i 's/OPENHIM_MEDIATOR_API_PORT=8080/OPENHIM_MEDIATOR_API_PORT=443/g' .env.local - run: sed -i 's/INSECURE=true/INSECURE=false/g' .env.local - run: sed -i 's/domain/${{ github.run_id }}.jembi.cloud/g' .env.local - - run: >- - sed -i 's/^image: jembi\/platform:.*$/image: jembi\/platform:${{ github.run_id }}.jembi.cloud/' config.yaml - name: Install dependencies working-directory: ./test/cucumber run: yarn