diff --git a/.env.cluster b/.env.cluster index c13bed3d..5e8e70b7 100644 --- a/.env.cluster +++ b/.env.cluster @@ -21,9 +21,10 @@ OPENHIM_MONGO_ATNAURL=mongodb://mongo-1:27017,mongo-2:27017,mongo-3:27017/openhi HAPI_FHIR_INSTANCES=3 REPMGR_PARTNER_NODES=postgres-1,postgres-2,postgres-3 POSTGRES_REPLICA_SET=postgres-1:5432,postgres-2:5432,postgres-3:5432 +HAPI_DB_SET=pgpool-1:5432,pgpool-2:5432,pgpool-3:5432 # Sante Datastore - Sante MPI -REPMGR_PARTNER_NODES=santempi-psql-1,santempi-psql-2,santempi-psql-3 +SANTEMPI_REPMGR_PARTNER_NODES=santempi-psql-1,santempi-psql-2,santempi-psql-3 # Reverse Proxy - Nginx REVERSE_PROXY_INSTANCES=3 diff --git a/.env.remote b/.env.remote index 8dea7b25..882cf140 100644 --- a/.env.remote +++ b/.env.remote @@ -7,7 +7,6 @@ OPENHIM_TRANSACTION_API_PORT= OPENHIM_MEDIATOR_API_PORT= MOCK_SERVER_HOST= MOCK_SERVER_PORT= -POSTGRES_REPLICA_SET=postgres-1:5432,postgres-2:5432,postgres-3:5432 # Analytics Datastore - Elastic Search ES_HOSTS="\"analytics-datastore-elastic-search-01:9200","analytics-datastore-elastic-search-02:9200","analytics-datastore-elastic-search-03:9200\"" @@ -18,6 +17,7 @@ CLICKHOUSE_HOST=analytics-datastore-clickhouse-01 # FHIR Datastore - HAPI FHIR REPMGR_PARTNER_NODES=postgres-1,postgres-2,postgres-3 POSTGRES_REPLICA_SET=postgres-1:5432,postgres-2:5432,postgres-3:5432 +HAPI_DB_SET=postgres-1:5432,postgres-2:5432,postgres-3:5432 # Interoperability Layer - OpenHIM MONGO_SET_COUNT=3 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d50365a9..7ed7cedd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,7 +95,6 @@ jobs: - run: sed -i 's/OPENHIM_MEDIATOR_API_PORT=8080/OPENHIM_MEDIATOR_API_PORT=443/g' .env.local - run: sed -i 's/INSECURE=true/INSECURE=false/g' .env.local - run: sed -i 's/domain/${{ github.run_id }}.jembi.cloud/g' .env.local - - name: Install dependencies working-directory: ./test/cucumber run: yarn diff --git a/analytics-datastore-elastic-search/swarm.sh b/analytics-datastore-elastic-search/swarm.sh index 5eace35d..6c7bef4a 100644 --- a/analytics-datastore-elastic-search/swarm.sh +++ b/analytics-datastore-elastic-search/swarm.sh @@ -78,7 +78,7 @@ function create_certs() { function add_docker_configs() { local -r TIMESTAMP="$(date "+%Y%m%d%H%M%S")" - local -r path_config_certs="/usr/share/elasticsearch/config/certs/" + local -r path_config_certs="/usr/share/elasticsearch/config/certs" log info "Creating configs..." diff --git a/config.yaml b/config.yaml index fe22ac16..d02396dd 100644 --- a/config.yaml +++ b/config.yaml @@ -1,5 +1,5 @@ projectName: platform -image: jembi/platform:2.3.3 +image: jembi/platform:2.4.0 logPath: /tmp/logs packages: diff --git a/fhir-datastore-hapi-fhir/config/custom_pgpool.conf b/fhir-datastore-hapi-fhir/config/custom_pgpool.conf new file mode 100644 index 00000000..88a7ce68 --- /dev/null +++ b/fhir-datastore-hapi-fhir/config/custom_pgpool.conf @@ -0,0 +1,29 @@ +failover_on_backend_error='on' +detach_false_primary='on' +auto_failback='on' +health_check_period=25 +health_check_max_retries=3 +health_check_user='postgres' +reserved_connections=1 +max_pool=6 +client_idle_limit=300 +connection_life_time=300 +child_max_connections=25 + +use_watchdog='on' +hostname0='pgpool-1' +wd_port0=9000 +pgpool_port0=9999 +hostname1='pgpool-2' +wd_port1=9000 +pgpool_port1=9999 +hostname2='pgpool-3' +wd_port2=9000 +pgpool_port2=9999 +heartbeat_hostname0='pgpool-1' +heartbeat_port0=9694 +heartbeat_hostname1='pgpool-2' +heartbeat_port1=9694 +heartbeat_hostname2='pgpool-3' +heartbeat_port2=9694 +trusted_servers='pgpool-1,pgpool-2,pgpool-3' diff --git a/fhir-datastore-hapi-fhir/config/pgpool_node_id0 b/fhir-datastore-hapi-fhir/config/pgpool_node_id0 new file mode 100644 index 00000000..573541ac --- /dev/null +++ b/fhir-datastore-hapi-fhir/config/pgpool_node_id0 @@ -0,0 +1 @@ +0 diff --git a/fhir-datastore-hapi-fhir/config/pgpool_node_id1 b/fhir-datastore-hapi-fhir/config/pgpool_node_id1 new file mode 100644 index 00000000..d00491fd --- /dev/null +++ b/fhir-datastore-hapi-fhir/config/pgpool_node_id1 @@ -0,0 +1 @@ +1 diff --git a/fhir-datastore-hapi-fhir/config/pgpool_node_id2 b/fhir-datastore-hapi-fhir/config/pgpool_node_id2 new file mode 100644 index 00000000..0cfbf088 --- /dev/null +++ b/fhir-datastore-hapi-fhir/config/pgpool_node_id2 @@ -0,0 +1 @@ +2 diff --git a/fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml b/fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml index 1cd86405..87ea782d 100644 --- a/fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml +++ b/fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml @@ -21,6 +21,7 @@ services: REPMGR_NODE_NAME: postgres-2 REPMGR_PRIMARY_HOST: ${REPMGR_PRIMARY_HOST} REPMGR_PARTNER_NODES: ${REPMGR_PARTNER_NODES} + POSTGRESQL_POSTGRES_PASSWORD: ${HAPI_POSTGRES_PASSWORD} volumes: - 'hapi-postgres-2-data:/bitnami/postgresql' deploy: @@ -37,9 +38,7 @@ services: memory: ${HF_POSTGRES_MEMORY_RESERVE} networks: default: - hapi-fhir-postgres: - pg_backup_net: {} - + pg_backup_net: postgres-3: image: bitnami/postgresql-repmgr:14 @@ -53,6 +52,7 @@ services: REPMGR_NODE_NAME: postgres-3 REPMGR_PRIMARY_HOST: ${REPMGR_PRIMARY_HOST} REPMGR_PARTNER_NODES: ${REPMGR_PARTNER_NODES} + POSTGRESQL_POSTGRES_PASSWORD: ${HAPI_POSTGRES_PASSWORD} volumes: - 'hapi-postgres-3-data:/bitnami/postgresql' deploy: @@ -69,9 +69,131 @@ services: memory: ${HF_POSTGRES_MEMORY_RESERVE} networks: default: - hapi-fhir-postgres: - pg_backup_net: {} + pg_backup_net: + + pgpool-1: + image: bitnami/pgpool:4.4.3 + deploy: + placement: + max_replicas_per_node: 1 + resources: + limits: + cpus: ${HF_PGPOOL_CPU_LIMIT} + memory: ${HF_PGPOOL_MEMORY_LIMIT} + reservations: + cpus: ${HF_PGPOOL_CPU_RESERVE} + memory: ${HF_PGPOOL_MEMORY_RESERVE} + configs: + - target: /config/custom_pgpool.conf + source: pgpool.conf + - target: /opt/bitnami/pgpool/conf/pgpool_node_id + source: pgpool_node_id0.conf + environment: + PGPOOL_BACKEND_NODES: ${HF_PGPOOL_NODES} + PGPOOL_BACKEND_APPLICATION_NAMES: ${REPMGR_PARTNER_NODES} + PGPOOL_SR_CHECK_USER: postgres + PGPOOL_SR_CHECK_PASSWORD: ${HAPI_POSTGRES_PASSWORD} + PGPOOL_POSTGRES_USERNAME: postgres + PGPOOL_POSTGRES_PASSWORD: ${HAPI_POSTGRES_PASSWORD} + PGPOOL_ADMIN_USERNAME: ${HF_PGPOOL_USERNAME} + PGPOOL_ADMIN_PASSWORD: ${HF_PGPOOL_PASSWORD} + PGPOOL_POSTGRES_CUSTOM_USERS: ${HF_POSTGRESQL_USERNAME} + PGPOOL_POSTGRES_CUSTOM_PASSWORDS: ${HF_POSTGRESQL_PASSWORD} + PGPOOL_ENABLE_LOAD_BALANCING: "no" + PGPOOL_AUTO_FAILBACK: "yes" + PGPOOL_USER_CONF_FILE: "/config/custom_pgpool.conf" + networks: + default: + + pgpool-2: + image: bitnami/pgpool:4.4.3 + deploy: + placement: + max_replicas_per_node: 1 + resources: + limits: + cpus: ${HF_PGPOOL_CPU_LIMIT} + memory: ${HF_PGPOOL_MEMORY_LIMIT} + reservations: + cpus: ${HF_PGPOOL_CPU_RESERVE} + memory: ${HF_PGPOOL_MEMORY_RESERVE} + configs: + - target: /config/custom_pgpool.conf + source: pgpool.conf + - target: /opt/bitnami/pgpool/conf/pgpool_node_id + source: pgpool_node_id1.conf + environment: + PGPOOL_BACKEND_NODES: ${HF_PGPOOL_NODES} + PGPOOL_BACKEND_APPLICATION_NAMES: ${REPMGR_PARTNER_NODES} + PGPOOL_SR_CHECK_USER: postgres + PGPOOL_SR_CHECK_PASSWORD: ${HAPI_POSTGRES_PASSWORD} + PGPOOL_POSTGRES_USERNAME: postgres + PGPOOL_POSTGRES_PASSWORD: ${HAPI_POSTGRES_PASSWORD} + PGPOOL_ADMIN_USERNAME: ${HF_PGPOOL_USERNAME} + PGPOOL_ADMIN_PASSWORD: ${HF_PGPOOL_PASSWORD} + PGPOOL_POSTGRES_CUSTOM_USERS: ${HF_POSTGRESQL_USERNAME} + PGPOOL_POSTGRES_CUSTOM_PASSWORDS: ${HF_POSTGRESQL_PASSWORD} + PGPOOL_ENABLE_LOAD_BALANCING: "no" + PGPOOL_AUTO_FAILBACK: "yes" + PGPOOL_USER_CONF_FILE: "/config/custom_pgpool.conf" + networks: + default: + + pgpool-3: + image: bitnami/pgpool:4.4.3 + deploy: + placement: + max_replicas_per_node: 1 + resources: + limits: + cpus: ${HF_PGPOOL_CPU_LIMIT} + memory: ${HF_PGPOOL_MEMORY_LIMIT} + reservations: + cpus: ${HF_PGPOOL_CPU_RESERVE} + memory: ${HF_PGPOOL_MEMORY_RESERVE} + configs: + - target: /config/custom_pgpool.conf + source: pgpool.conf + - target: /opt/bitnami/pgpool/conf/pgpool_node_id + source: pgpool_node_id2.conf + environment: + PGPOOL_BACKEND_NODES: ${HF_PGPOOL_NODES} + PGPOOL_BACKEND_APPLICATION_NAMES: ${REPMGR_PARTNER_NODES} + PGPOOL_SR_CHECK_USER: postgres + PGPOOL_SR_CHECK_PASSWORD: ${HAPI_POSTGRES_PASSWORD} + PGPOOL_POSTGRES_USERNAME: postgres + PGPOOL_POSTGRES_PASSWORD: ${HAPI_POSTGRES_PASSWORD} + PGPOOL_ADMIN_USERNAME: ${HF_PGPOOL_USERNAME} + PGPOOL_ADMIN_PASSWORD: ${HF_PGPOOL_PASSWORD} + PGPOOL_POSTGRES_CUSTOM_USERS: ${HF_POSTGRESQL_USERNAME} + PGPOOL_POSTGRES_CUSTOM_PASSWORDS: ${HF_POSTGRESQL_PASSWORD} + PGPOOL_ENABLE_LOAD_BALANCING: "no" + PGPOOL_AUTO_FAILBACK: "yes" + PGPOOL_USER_CONF_FILE: "/config/custom_pgpool.conf" + networks: + default: +configs: + pgpool.conf: + name: pgpool_conf-${pgpool_conf_DIGEST:?err} + file: ./config/custom_pgpool.conf + labels: + name: hapi-fhir + pgpool_node_id0.conf: + name: pgpool_node_id0-${pgpool_node_id0_DIGEST:?err} + file: ./config/pgpool_node_id0 + labels: + name: hapi-fhir + pgpool_node_id1.conf: + name: pgpool_node_id1-${pgpool_node_id1_DIGEST:?err} + file: ./config/pgpool_node_id1 + labels: + name: hapi-fhir + pgpool_node_id2.conf: + name: pgpool_node_id2-${pgpool_node_id2_DIGEST:?err} + file: ./config/pgpool_node_id2 + labels: + name: hapi-fhir volumes: hapi-postgres-2-data: diff --git a/fhir-datastore-hapi-fhir/docker-compose.yml b/fhir-datastore-hapi-fhir/docker-compose.yml index 95e0158e..8546fd48 100644 --- a/fhir-datastore-hapi-fhir/docker-compose.yml +++ b/fhir-datastore-hapi-fhir/docker-compose.yml @@ -8,20 +8,24 @@ services: public: default: environment: - - spring.datasource.url=jdbc:postgresql://${POSTGRES_REPLICA_SET}/hapi?targetServerType=primary - - spring.datasource.username=admin - - spring.datasource.password=instant101 + - spring.datasource.url=jdbc:postgresql://${HAPI_DB_SET}/hapi?targetServerType=primary + - spring.datasource.username=${HF_POSTGRESQL_USERNAME} + - spring.datasource.password=${HF_POSTGRESQL_PASSWORD} - spring.datasource.driverClassName=org.postgresql.Driver - spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.PostgreSQL95Dialect + - spring.datasource.hikari.maximumPoolSize=250 + - spring.datasource.hikari.minimumIdle=20 + - spring.datasource.hikari.connectionTimeout=60000 + - spring.datasource.hikari.idleTimeout=600000 - hapi.fhir.allow_external_references=true - hapi.fhir.bulk_export_enabled=true - hapi.fhir.enable_repository_validating_interceptor=true - - JAVA_TOOL_OPTIONS=-Xmx2g - - CATALINA_OPTS=-Xmx2g + - JAVA_TOOL_OPTIONS=${HF_JAVA_OPTS} + - CATALINA_OPTS=${HF_JAVA_OPTS} deploy: replicas: ${HAPI_FHIR_INSTANCES} placement: - max_replicas_per_node: 1 + max_replicas_per_node: ${HAPI_FHIR_MAX_REPLICAS_PER_NODE} resources: limits: cpus: ${HAPI_FHIR_CPU_LIMIT} @@ -30,10 +34,10 @@ services: cpus: ${HAPI_FHIR_CPU_RESERVE} memory: ${HAPI_FHIR_MEMORY_RESERVE} healthcheck: - test: /bin/wget --no-verbose --tries=1 --spider http://localhost:8080 + test: /bin/wget --no-verbose --tries=1 --spider http://localhost:8080/fhir/Organization?identifier=urn:healthcheck interval: 30s - timeout: 5s - retries: 5 + timeout: 30s + retries: 2 start_period: 2m networks: diff --git a/fhir-datastore-hapi-fhir/package-metadata.json b/fhir-datastore-hapi-fhir/package-metadata.json index 719e386d..e267fb88 100644 --- a/fhir-datastore-hapi-fhir/package-metadata.json +++ b/fhir-datastore-hapi-fhir/package-metadata.json @@ -10,11 +10,13 @@ "REPMGR_PARTNER_NODES": "postgres-1", "REPMGR_PASSWORD": "instant101", "HAPI_FHIR_INSTANCES": "1", + "HAPI_FHIR_MAX_REPLICAS_PER_NODE": "1", "HAPI_FHIR_CPU_LIMIT": "0", "HAPI_FHIR_CPU_RESERVE": "0.05", "HAPI_FHIR_MEMORY_LIMIT": "3G", "HAPI_FHIR_MEMORY_RESERVE": "500M", "POSTGRES_REPLICA_SET": "postgres-1:5432", + "HAPI_DB_SET": "postgres-1:5432", "HAPI_POSTGRES_PASSWORD": "postgres", "HF_POSTGRES_CPU_LIMIT": "0", "HF_POSTGRES_CPU_RESERVE": "0.05", @@ -22,6 +24,14 @@ "HF_POSTGRES_MEMORY_RESERVE": "500M", "HF_POSTGRESQL_PASSWORD": "instant101", "HF_POSTGRESQL_USERNAME": "admin", - "HF_POSTGRESQL_DATABASE": "hapi" + "HF_POSTGRESQL_DATABASE": "hapi", + "HF_PGPOOL_NODES": "0:postgres-1:5432,1:postgres-2:5432,2:postgres-3:5432", + "HF_PGPOOL_USERNAME": "pgadmin", + "HF_PGPOOL_PASSWORD": "dev_password_only", + "HF_PGPOOL_CPU_LIMIT": "0", + "HF_PGPOOL_MEMORY_LIMIT": "500M", + "HF_PGPOOL_CPU_RESERVE": "0.05", + "HF_PGPOOL_MEMORY_RESERVE": "50M", + "HF_JAVA_OPTS": "-Xmx2g" } } diff --git a/message-bus-helper-hapi-proxy/docker-compose.yml b/message-bus-helper-hapi-proxy/docker-compose.yml index 0ee3f1f0..8dc29e46 100644 --- a/message-bus-helper-hapi-proxy/docker-compose.yml +++ b/message-bus-helper-hapi-proxy/docker-compose.yml @@ -8,6 +8,7 @@ services: HAPI_SERVER_URL: ${HAPI_SERVER_URL} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_HOSTS} HAPI_SERVER_VALIDATE_FORMAT: ${HAPI_SERVER_VALIDATE_FORMAT} + JAVA_OPTS: "-XX:MaxRAMPercentage=90" deploy: placement: max_replicas_per_node: 1 diff --git a/test/cucumber/features/cluster-mode/hapi-fhir.cluster.feature b/test/cucumber/features/cluster-mode/hapi-fhir.cluster.feature index 9da91314..ba7ac63b 100644 --- a/test/cucumber/features/cluster-mode/hapi-fhir.cluster.feature +++ b/test/cucumber/features/cluster-mode/hapi-fhir.cluster.feature @@ -9,21 +9,28 @@ Feature: Fhir Datastore HAPI-FHIR? | hapi-fhir_postgres_public | hapi-fhir_default | pg_backup | And The service "postgres-2" should be started with 1 replica And The service "postgres-2" should be connected to the networks - | hapi-fhir_postgres_public | hapi-fhir_default | pg_backup | + | hapi-fhir_default | pg_backup | And The service "postgres-3" should be started with 1 replica And The service "postgres-3" should be connected to the networks - | hapi-fhir_postgres_public | hapi-fhir_default | pg_backup | + | hapi-fhir_default | pg_backup | + And The service "postgres-3" should be started with 1 replica + And The service "pgpool-1" should be connected to the networks + | hapi-fhir_default | + And The service "pgpool-2" should be connected to the networks + | hapi-fhir_default | + And The service "pgpool-3" should be connected to the networks + | hapi-fhir_default | And The service "hapi-fhir" should be started with 3 replicas And The service "hapi-fhir" should be connected to the networks | mpi_public | hapi-fhir_public | hapi-fhir_default | - And There should be 4 services + And There should be 7 services And The service "hapi-fhir" should have healthy containers Scenario: Init Message Bus Helper Hapi Proxy Given I use parameters "package init -n=message-bus-helper-hapi-proxy --only --dev --env-file=.env.cluster" When I launch the platform with params Then The service "hapi-proxy" should be started with 3 replicas - And There should be 5 services + And There should be 8 services And The service "hapi-proxy" should be connected to the networks | hapi-fhir_public | kafka_public | openhim_public | @@ -33,6 +40,9 @@ Feature: Fhir Datastore HAPI-FHIR? Then The service "postgres-1" should be removed And The service "postgres-2" should be removed And The service "postgres-3" should be removed + And The service "pgpool-1" should be removed + And The service "pgpool-2" should be removed + And The service "pgpool-3" should be removed And The service "hapi-fhir" should be removed And The service "hapi-proxy" should be removed And There should be 0 service