diff --git a/openfn/docker-compose.yml b/openfn/docker-compose.yml index 48cdbf59..ba15de54 100644 --- a/openfn/docker-compose.yml +++ b/openfn/docker-compose.yml @@ -3,8 +3,7 @@ version: '3.9' services: openfn: image: ${OpenFn_IMAGE} - stop_grace_period: '3s' - tty: ${TTY:-false} + command: sh -c '/app/bin/lightning eval "Lightning.Release.migrate()" && /app/bin/lightning eval "Lightning.Demo.reset_demo()" && /app/bin/lightning start' deploy: resources: limits: @@ -21,6 +20,7 @@ services: - SECRET_KEY_BASE=${SECRET_KEY_BASE} - WORKER_RUNS_PRIVATE_KEY=${WORKER_RUNS_PRIVATE_KEY} - WORKER_SECRET=${WORKER_SECRET} + - KAFKA_TRIGGERS_ENABLED=${KAFKA_TRIGGERS_ENABLED} healthcheck: test: '${DOCKER_WEB_HEALTHCHECK_TEST:-curl localhost:4000/health_check}' interval: '10s' @@ -30,37 +30,7 @@ services: networks: - kafka_public - postgres - worker: - image: ${OpenFn_WORKER_IMAGE} - deploy: - resources: - limits: - cpus: '${DOCKER_WORKER_CPUS:-0}' - memory: '${DOCKER_WEB_MEMORY:-0}' - environment: - - WORKER_LIGHTNING_PUBLIC_KEY=${WORKER_LIGHTNING_PUBLIC_KEY} - - WORKER_SECRET=${WORKER_SECRET} - healthcheck: - test: - [ - 'CMD-SHELL', - 'curl -f http://openfn:${URL_PORT-4000}/health_check || exit 1' - ] - interval: 10s - timeout: 5s - retries: 3 - command: - [ - 'pnpm', - 'start:prod', - '-l', - 'ws://openfn:${URL_PORT-4000}/worker' - ] - stop_grace_period: '3s' - networks: - - kafka_public - - postgres networks: kafka_public: name: kafka_public diff --git a/openfn/importer/workflows/docker-compose.config.yml b/openfn/importer/workflows/docker-compose.config.yml new file mode 100644 index 00000000..19787b49 --- /dev/null +++ b/openfn/importer/workflows/docker-compose.config.yml @@ -0,0 +1,44 @@ +version: '3.9' + +services: + openfn_workflow_config: + image: node:18-alpine3.20 + command: sh -c "cd /workflows && ls && npm install -g @openfn/cli && ls && openfn deploy -c config.json --no-confirm" + configs: + - target: /workflows/cdr-ndr.yaml + source: cdr-ndr.yaml + - target: /workflows/config.json + source: config.json + - target: /workflows/project.yaml + source: project.yaml + networks: + postgres: + environment: + OPENFN_API_KEY: ${OPENFN_API_KEY} + OPENFN_API_URL: ${OPENFN_ENDPOINT} + deploy: + replicas: 1 + restart_policy: + condition: none + +networks: + postgres: + name: postgres_public + external: true + +configs: + cdr-ndr.yaml: + file: ./example/cdr-ndr.yaml + name: cdr-ndr.yaml-${cdr_ndr_yaml_DIGEST:?err} + labels: + name: cdrproject + config.json: + file: ./example/config.json + name: config.json-${config_json_DIGEST:?err} + labels: + name: configjson + project.yaml: + file: ./example/project.yaml + name: project.yaml-${project_yaml_DIGEST:?err} + labels: + name: project diff --git a/openfn/package-metadata.json b/openfn/package-metadata.json index b2495dbb..eadc289d 100644 --- a/openfn/package-metadata.json +++ b/openfn/package-metadata.json @@ -12,9 +12,9 @@ "LISTEN_ADDRESS": "0.0.0.0", "LOG_LEVEL": "debug", "ORIGINS": "http://localhost:4000", - "PRIMARY_ENCRYPTION_KEY": "secret_here", - "SECRET_KEY_BASE": "secret_key_here", - "WORKER_RUNS_PRIVATE_KEY": "", + "PRIMARY_ENCRYPTION_KEY": "KLu/IoZuaf+baDECd8wG4Z6auwNe6VAmwh9N8lWdJ1A=", + "SECRET_KEY_BASE": "jGDxZj2O+Qzegm5wcZ940RfWO4D6RyU8thNCr5BUpHNwa7UNV52M1/Sn+7RxiP+f", + "WORKER_RUNS_PRIVATE_KEY": "LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2Z0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktnd2dnU2tBZ0VBQW9JQkFRREVtR3drUW5pT0hqVCsKMnkyRHFvRUhyT3dLZFI2RW9RWG9DeDE4MytXZ3hNcGthTFZyOFViYVVVQWNISGgzUFp2Z2UwcEIzTWlCWWR5Kwp1ajM1am5uK2JIdk9OZGRldWxOUUdpczdrVFFHRU1nTSs0Njhldm5RS0h6R29DRUhabDlZV0s0MUd5SEZCZXppCnJiOGx2T1A1NEtSTS90aE5pVGtHaUIvTGFLMldLcTh0VmtoSHBvaFE3OGIyR21vNzNmcWtuSGZNWnc0ZE43d1MKdldOamZIN3QwSmhUdW9mTXludUxSWmdFYUhmTDlnbytzZ0thc0ZUTmVvdEZIQkYxQTJjUDJCakwzaUxad0hmdQozTzEwZzg0aGZlTzJqTWlsZlladHNDdmxDTE1EZWNrdFJGWFl6V0dWc25FcFNiOStjcWJWUXRvdEU4QklON09GClRmaEx2MG9uQWdNQkFBRUNnZ0VBV3dmZyt5RTBSVXBEYThiOVdqdzNKdUN4STE1NzFSbmliRUhKVTZzdzNyS0EKck9HM0w5WTI0cHhBdlVPSm5GMFFzbThrUVQ4RU1MU3B6RDdjdDVON2RZMngvaGY4TThhL0VSWXM4cFlYcXI5Vwpnbnh3NldGZ0R6elFHZ0RIaW0raXNudk5ucFdEbTRGVTRObG02d2g5MzVSZlA2KzVaSjJucEJpZjhFWDJLdE9rCklOSHRVbFcwNFlXeDEwS0pIWWhYNFlydXVjL3MraXBORzBCSDZEdlJaQzQxSWw0N1luaTg1OERaL0FaeVNZN1kKWTlTamNKQ0QvUHBENTlNQjlSanJDQjhweDBjWGlsVXBVZUJSYndGalVwbWZuVmhIa1hiYlM1U0hXWWM4K3pLRQp2ajFqSEpxc2UyR0hxK2lHL1V3NTZvcHNyM2x3dHBRUXpVcEJGblhMMFFLQmdRRDM5bkV3L1NNVGhCallSd1JGCkY2a2xOYmltU2RGOVozQlZleXhrT0dUeU5NSCtYckhsQjFpOXBRRHdtMit3V2RvcWg1ZFRFbEU5K1crZ0FhN0YKbXlWc2xPTW4wdnZ2cXY2Wkp5SDRtNTVKU0lWSzBzRjRQOTRMYkpNSStHUW5VNnRha3Y0V0FSMkpXaURabGxPdAp3R01EQWZqRVIrSEFZeUJDKzNDL25MNHF5d0tCZ1FESzk3NERtV0c4VDMzNHBiUFVEYnpDbG9oTlQ2UldxMXVwCmJSWng4ZGpzZU0vQ09kZnBUcmJuMnk5dVc3Q1pBNFVPQ2s4REcxZ3ZENVVDYlpEUVdMaUp5RzZGdG5OdGgvaU8KT1dJM0UyczZOS0VMMU1NVzh5QWZwNzV4Ung5cnNaQzI2UEtqQ0pWL2lTVjcyNlQ1ZTFzRG5sZUtBb0JFZnlDRgpvbEhhMmhybWxRS0JnUURHT1YyOWd1K1NmMng1SVRTWm8xT1ZxbitGZDhlZno1d3V5YnZ3Rm1Fa2V1YUdXZDh1CnJ4UFM3MkJ6K0Y1dUJUWngvMWtLa0w4Zm94TUlQN0FleW1zOWhUeWVybnkyMk9TVlBJSmN3dExqMUxTeDN3L0kKK0kyaVpsYVl1akVlZXpXbHY1S2R0cUNORjk3Zzh0ck1NTnMySVZKa1h1NXFwUk82V0ZXRzZGL2h4d0tCZ0hnNApHYUpFSFhIT204ekZTU2lYSW5FWGZKQmVWZmJIOUxqNzFrbVRlR3RJZTdhTlVHZnVxY1BYUGRiZUZGSHRsY2ZsCkx6dWwzS3V6VFExdEhGTnIyWkl5MTlQM1o1TSs4R2c5Y1FFeVRWYmlpV2xha2x0cmttRnRtQTI4bE0zVEZPWmkKUUNWMUZpZStjaWRVeC9qRnFma1F0c1VXQ2llSUxSazZOY1d0WGpXcEFvR0JBTGN6Y210VGlUUEFvWnk0MFV1QQpTOXpUd3RsamhmUWJEVTVjb21EcnlKcnFRU0VOdmQ2VW5HdW0zYVNnNk13dDc0NGxidDAyMC9mSGI0WTJkTGhMCmx4YWJ5b1dQUElRRUpLL1NNOGtURFEvYTRyME5tZzhuV3h5bGFLcHQ5WUhmZ2NYMkYzSzUrc0VSUGNFcVZlWFMKdWZkYXdYQVlFampZK3V2UHZ2YzU3RU1aCi0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K", "WORKER_SECRET": "secret_here", "POSTGRES_USER": "postgres", "POSTGRES_SERVICE": "postgres-1", @@ -24,8 +24,11 @@ "OpenFn_POSTGRESQL_DB": "lightning_dev", "OpenFn_POSTGRESQL_USERNAME": "openfn", "OpenFn_POSTGRESQL_PASSWORD": "instant101", - "WORKER_LIGHTNING_PUBLIC_KEY": "", - "OpenFn_IMAGE": "openfn/lightning:v2.6.0-kt9", - "OpenFn_WORKER_IMAGE": "openfn/ws-worker:latest" + "WORKER_LIGHTNING_PUBLIC_KEY": "LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUF4SmhzSkVKNGpoNDAvdHN0ZzZxQgpCNnpzQ25VZWhLRUY2QXNkZk4vbG9NVEtaR2kxYS9GRzJsRkFIQng0ZHoyYjRIdEtRZHpJZ1dIY3ZybzkrWTU1Ci9teDd6alhYWHJwVFVCb3JPNUUwQmhESURQdU92SHI1MENoOHhxQWhCMlpmV0ZpdU5Sc2h4UVhzNHEyL0piemoKK2VDa1RQN1lUWWs1Qm9nZnkyaXRsaXF2TFZaSVI2YUlVTy9HOWhwcU85MzZwSngzekdjT0hUZThFcjFqWTN4Kwo3ZENZVTdxSHpNcDdpMFdZQkdoM3kvWUtQcklDbXJCVXpYcUxSUndSZFFObkQ5Z1l5OTRpMmNCMzd0enRkSVBPCklYM2p0b3pJcFgyR2JiQXI1UWl6QTNuSkxVUlYyTTFobGJKeEtVbS9mbkttMVVMYUxSUEFTRGV6aFUzNFM3OUsKSndJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0tCg", + "OpenFn_IMAGE": "openfn/lightning:latest", + "OpenFn_WORKER_IMAGE": "openfn/ws-worker:latest", + "KAFKA_TRIGGERS_ENABLED": "true", + "OPENFN_API_KEY": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJKb2tlbiIsImlhdCI6MTY4ODAzNzE4NSwiaXNzIjoiSm9rZW4iLCJqdGkiOiIydG1ocG8zYm0xdmR0MDZvZDgwMDAwdTEiLCJuYmYiOjE2ODgwMzcxODUsInVzZXJfaWQiOiIzZjM3OGU2Yy02NjBhLTRiOTUtYWI5Ni02YmQwZGMyNjNkMzMifQ.J1FnACGpqtQbmXNvyUCwCY4mS5S6CohRU3Ey-N0prP4", + "OPENFN_ENDPOINT": "http://localhost:4000" } } diff --git a/openfn/swarm.sh b/openfn/swarm.sh index ce8ae7f1..42de3f7b 100644 --- a/openfn/swarm.sh +++ b/openfn/swarm.sh @@ -47,7 +47,16 @@ function initialize_package() { if [[ "${ACTION}" == "init" ]]; then docker::deploy_config_importer $STACK "$COMPOSE_FILE_PATH/importer/postgres/docker-compose.config.yml" "openfn_db_config" "openfn" fi - docker::deploy_service $STACK "${COMPOSE_FILE_PATH}" "docker-compose.yml" "$package_dev_compose_filename" + + docker::deploy_service $STACK "${COMPOSE_FILE_PATH}" "docker-compose.yml" "$package_dev_compose_filename" + + if docker service ps -q openfn_openfn &>/dev/null; then + echo "Service 'openfn_openfn' is running." + docker::deploy_config_importer $STACK "$COMPOSE_FILE_PATH/importer/workflows/docker-compose.config.yml" "openfn_workflow_config" "openfn_workflow" + else + echo "Service 'openfn_openfn' is not running." + log warn "Service 'interoperability-layer-openhim' does not appear to be running... skipping configuring of async/sync JeMPI channels" + fi ) || { log error "Failed to deploy package" exit 1