add workflow import via openfn cli

jembi · Aug 15, 2024 · 6259af7 · 6259af7
1 parent 40f1191
commit 6259af7
Show file tree

Hide file tree

Showing 4 changed files with 65 additions and 39 deletions.
diff --git a/openfn/docker-compose.yml b/openfn/docker-compose.yml
@@ -3,8 +3,7 @@ version: '3.9'
 services:
   openfn:
     image: ${OpenFn_IMAGE}
-    stop_grace_period: '3s'
-    tty: ${TTY:-false}
+    command: sh -c '/app/bin/lightning eval "Lightning.Release.migrate()" && /app/bin/lightning eval "Lightning.Demo.reset_demo()" && /app/bin/lightning start'
     deploy:
       resources:
         limits:
@@ -21,6 +20,7 @@ services:
       - SECRET_KEY_BASE=${SECRET_KEY_BASE}
       - WORKER_RUNS_PRIVATE_KEY=${WORKER_RUNS_PRIVATE_KEY}
       - WORKER_SECRET=${WORKER_SECRET}
+      - KAFKA_TRIGGERS_ENABLED=${KAFKA_TRIGGERS_ENABLED}
     healthcheck:
       test: '${DOCKER_WEB_HEALTHCHECK_TEST:-curl localhost:4000/health_check}'
       interval: '10s'
@@ -30,37 +30,7 @@ services:
     networks:
       - kafka_public
       - postgres
-  worker:
-    image: ${OpenFn_WORKER_IMAGE}
-    deploy:
-      resources:
-        limits:
-          cpus: '${DOCKER_WORKER_CPUS:-0}'
-          memory: '${DOCKER_WEB_MEMORY:-0}'
-    environment:
-      - WORKER_LIGHTNING_PUBLIC_KEY=${WORKER_LIGHTNING_PUBLIC_KEY}
-      - WORKER_SECRET=${WORKER_SECRET}
-    healthcheck:
-      test:
-        [
-          'CMD-SHELL',
-          'curl -f http://openfn:${URL_PORT-4000}/health_check || exit 1'
-        ]
-      interval: 10s
-      timeout: 5s
-      retries: 3
-    command:
-      [
-        'pnpm',
-        'start:prod',
-        '-l',
-        'ws://openfn:${URL_PORT-4000}/worker'
-      ]
-    stop_grace_period: '3s'
 
-    networks:
-      - kafka_public
-      - postgres
 networks:
   kafka_public:
     name: kafka_public

diff --git a/openfn/importer/workflows/docker-compose.config.yml b/openfn/importer/workflows/docker-compose.config.yml
@@ -0,0 +1,44 @@
+version: '3.9'
+
+services:
+  openfn_workflow_config:
+    image: node:18-alpine3.20
+    command: sh -c "cd /workflows && ls && npm install -g @openfn/cli && ls && openfn deploy -c config.json --no-confirm"
+    configs:
+      - target: /workflows/cdr-ndr.yaml
+        source: cdr-ndr.yaml
+      - target: /workflows/config.json
+        source: config.json
+      - target: /workflows/project.yaml
+        source: project.yaml
+    networks:
+      postgres:
+    environment:
+      OPENFN_API_KEY: ${OPENFN_API_KEY}
+      OPENFN_API_URL: ${OPENFN_ENDPOINT}
+    deploy:
+      replicas: 1
+      restart_policy:
+        condition: none
+
+networks:
+  postgres:
+    name: postgres_public
+    external: true
+
+configs:
+  cdr-ndr.yaml:
+    file: ./example/cdr-ndr.yaml
+    name: cdr-ndr.yaml-${cdr_ndr_yaml_DIGEST:?err}
+    labels:
+      name: cdrproject
+  config.json:
+    file: ./example/config.json
+    name: config.json-${config_json_DIGEST:?err}
+    labels:
+      name: configjson
+  project.yaml:
+    file: ./example/project.yaml
+    name: project.yaml-${project_yaml_DIGEST:?err}
+    labels:
+      name: project
diff --git a/openfn/package-metadata.json b/openfn/package-metadata.json
@@ -12,9 +12,9 @@
     "LISTEN_ADDRESS": "0.0.0.0",
     "LOG_LEVEL": "debug",
     "ORIGINS": "http://localhost:4000",
-    "PRIMARY_ENCRYPTION_KEY": "secret_here",
-    "SECRET_KEY_BASE": "secret_key_here",
-    "WORKER_RUNS_PRIVATE_KEY": "<RSA PRIVATE KEY HERE>",
+    "PRIMARY_ENCRYPTION_KEY": "KLu/IoZuaf+baDECd8wG4Z6auwNe6VAmwh9N8lWdJ1A=",
+    "SECRET_KEY_BASE": "jGDxZj2O+Qzegm5wcZ940RfWO4D6RyU8thNCr5BUpHNwa7UNV52M1/Sn+7RxiP+f",
+    "WORKER_RUNS_PRIVATE_KEY": "LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2Z0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktnd2dnU2tBZ0VBQW9JQkFRREVtR3drUW5pT0hqVCsKMnkyRHFvRUhyT3dLZFI2RW9RWG9DeDE4MytXZ3hNcGthTFZyOFViYVVVQWNISGgzUFp2Z2UwcEIzTWlCWWR5Kwp1ajM1am5uK2JIdk9OZGRldWxOUUdpczdrVFFHRU1nTSs0Njhldm5RS0h6R29DRUhabDlZV0s0MUd5SEZCZXppCnJiOGx2T1A1NEtSTS90aE5pVGtHaUIvTGFLMldLcTh0VmtoSHBvaFE3OGIyR21vNzNmcWtuSGZNWnc0ZE43d1MKdldOamZIN3QwSmhUdW9mTXludUxSWmdFYUhmTDlnbytzZ0thc0ZUTmVvdEZIQkYxQTJjUDJCakwzaUxad0hmdQozTzEwZzg0aGZlTzJqTWlsZlladHNDdmxDTE1EZWNrdFJGWFl6V0dWc25FcFNiOStjcWJWUXRvdEU4QklON09GClRmaEx2MG9uQWdNQkFBRUNnZ0VBV3dmZyt5RTBSVXBEYThiOVdqdzNKdUN4STE1NzFSbmliRUhKVTZzdzNyS0EKck9HM0w5WTI0cHhBdlVPSm5GMFFzbThrUVQ4RU1MU3B6RDdjdDVON2RZMngvaGY4TThhL0VSWXM4cFlYcXI5Vwpnbnh3NldGZ0R6elFHZ0RIaW0raXNudk5ucFdEbTRGVTRObG02d2g5MzVSZlA2KzVaSjJucEJpZjhFWDJLdE9rCklOSHRVbFcwNFlXeDEwS0pIWWhYNFlydXVjL3MraXBORzBCSDZEdlJaQzQxSWw0N1luaTg1OERaL0FaeVNZN1kKWTlTamNKQ0QvUHBENTlNQjlSanJDQjhweDBjWGlsVXBVZUJSYndGalVwbWZuVmhIa1hiYlM1U0hXWWM4K3pLRQp2ajFqSEpxc2UyR0hxK2lHL1V3NTZvcHNyM2x3dHBRUXpVcEJGblhMMFFLQmdRRDM5bkV3L1NNVGhCallSd1JGCkY2a2xOYmltU2RGOVozQlZleXhrT0dUeU5NSCtYckhsQjFpOXBRRHdtMit3V2RvcWg1ZFRFbEU5K1crZ0FhN0YKbXlWc2xPTW4wdnZ2cXY2Wkp5SDRtNTVKU0lWSzBzRjRQOTRMYkpNSStHUW5VNnRha3Y0V0FSMkpXaURabGxPdAp3R01EQWZqRVIrSEFZeUJDKzNDL25MNHF5d0tCZ1FESzk3NERtV0c4VDMzNHBiUFVEYnpDbG9oTlQ2UldxMXVwCmJSWng4ZGpzZU0vQ09kZnBUcmJuMnk5dVc3Q1pBNFVPQ2s4REcxZ3ZENVVDYlpEUVdMaUp5RzZGdG5OdGgvaU8KT1dJM0UyczZOS0VMMU1NVzh5QWZwNzV4Ung5cnNaQzI2UEtqQ0pWL2lTVjcyNlQ1ZTFzRG5sZUtBb0JFZnlDRgpvbEhhMmhybWxRS0JnUURHT1YyOWd1K1NmMng1SVRTWm8xT1ZxbitGZDhlZno1d3V5YnZ3Rm1Fa2V1YUdXZDh1CnJ4UFM3MkJ6K0Y1dUJUWngvMWtLa0w4Zm94TUlQN0FleW1zOWhUeWVybnkyMk9TVlBJSmN3dExqMUxTeDN3L0kKK0kyaVpsYVl1akVlZXpXbHY1S2R0cUNORjk3Zzh0ck1NTnMySVZKa1h1NXFwUk82V0ZXRzZGL2h4d0tCZ0hnNApHYUpFSFhIT204ekZTU2lYSW5FWGZKQmVWZmJIOUxqNzFrbVRlR3RJZTdhTlVHZnVxY1BYUGRiZUZGSHRsY2ZsCkx6dWwzS3V6VFExdEhGTnIyWkl5MTlQM1o1TSs4R2c5Y1FFeVRWYmlpV2xha2x0cmttRnRtQTI4bE0zVEZPWmkKUUNWMUZpZStjaWRVeC9qRnFma1F0c1VXQ2llSUxSazZOY1d0WGpXcEFvR0JBTGN6Y210VGlUUEFvWnk0MFV1QQpTOXpUd3RsamhmUWJEVTVjb21EcnlKcnFRU0VOdmQ2VW5HdW0zYVNnNk13dDc0NGxidDAyMC9mSGI0WTJkTGhMCmx4YWJ5b1dQUElRRUpLL1NNOGtURFEvYTRyME5tZzhuV3h5bGFLcHQ5WUhmZ2NYMkYzSzUrc0VSUGNFcVZlWFMKdWZkYXdYQVlFampZK3V2UHZ2YzU3RU1aCi0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K",
     "WORKER_SECRET": "secret_here",
     "POSTGRES_USER": "postgres",
     "POSTGRES_SERVICE": "postgres-1",
@@ -24,8 +24,11 @@
     "OpenFn_POSTGRESQL_DB": "lightning_dev",
     "OpenFn_POSTGRESQL_USERNAME": "openfn",
     "OpenFn_POSTGRESQL_PASSWORD": "instant101",
-    "WORKER_LIGHTNING_PUBLIC_KEY": "<RSA PUBLIC KEY HERE>",
-    "OpenFn_IMAGE": "openfn/lightning:v2.6.0-kt9",
-    "OpenFn_WORKER_IMAGE": "openfn/ws-worker:latest"
+    "WORKER_LIGHTNING_PUBLIC_KEY": "LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUF4SmhzSkVKNGpoNDAvdHN0ZzZxQgpCNnpzQ25VZWhLRUY2QXNkZk4vbG9NVEtaR2kxYS9GRzJsRkFIQng0ZHoyYjRIdEtRZHpJZ1dIY3ZybzkrWTU1Ci9teDd6alhYWHJwVFVCb3JPNUUwQmhESURQdU92SHI1MENoOHhxQWhCMlpmV0ZpdU5Sc2h4UVhzNHEyL0piemoKK2VDa1RQN1lUWWs1Qm9nZnkyaXRsaXF2TFZaSVI2YUlVTy9HOWhwcU85MzZwSngzekdjT0hUZThFcjFqWTN4Kwo3ZENZVTdxSHpNcDdpMFdZQkdoM3kvWUtQcklDbXJCVXpYcUxSUndSZFFObkQ5Z1l5OTRpMmNCMzd0enRkSVBPCklYM2p0b3pJcFgyR2JiQXI1UWl6QTNuSkxVUlYyTTFobGJKeEtVbS9mbkttMVVMYUxSUEFTRGV6aFUzNFM3OUsKSndJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0tCg",
+    "OpenFn_IMAGE": "openfn/lightning:latest",
+    "OpenFn_WORKER_IMAGE": "openfn/ws-worker:latest",
+    "KAFKA_TRIGGERS_ENABLED": "true",
+    "OPENFN_API_KEY": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJKb2tlbiIsImlhdCI6MTY4ODAzNzE4NSwiaXNzIjoiSm9rZW4iLCJqdGkiOiIydG1ocG8zYm0xdmR0MDZvZDgwMDAwdTEiLCJuYmYiOjE2ODgwMzcxODUsInVzZXJfaWQiOiIzZjM3OGU2Yy02NjBhLTRiOTUtYWI5Ni02YmQwZGMyNjNkMzMifQ.J1FnACGpqtQbmXNvyUCwCY4mS5S6CohRU3Ey-N0prP4",
+    "OPENFN_ENDPOINT": "http://localhost:4000"
   }
 }
diff --git a/openfn/swarm.sh b/openfn/swarm.sh
@@ -47,7 +47,16 @@ function initialize_package() {
     if [[ "${ACTION}" == "init" ]]; then
          docker::deploy_config_importer $STACK "$COMPOSE_FILE_PATH/importer/postgres/docker-compose.config.yml" "openfn_db_config" "openfn"
     fi
-    docker::deploy_service $STACK "${COMPOSE_FILE_PATH}" "docker-compose.yml" "$package_dev_compose_filename"
+
+     docker::deploy_service $STACK "${COMPOSE_FILE_PATH}" "docker-compose.yml" "$package_dev_compose_filename"
+
+    if docker service ps -q openfn_openfn &>/dev/null; then
+         echo "Service 'openfn_openfn' is running."
+         docker::deploy_config_importer $STACK "$COMPOSE_FILE_PATH/importer/workflows/docker-compose.config.yml" "openfn_workflow_config" "openfn_workflow"
+    else
+        echo "Service 'openfn_openfn' is not running."
+        log warn "Service 'interoperability-layer-openhim' does not appear to be running... skipping configuring of async/sync JeMPI channels"
+    fi
   ) || {
     log error "Failed to deploy package"
     exit 1