Add deployment configuration for running background job worker

.github/workflows/deploy.yml

@@ -63,7 +63,9 @@ jobs:
          s|^CKANEXT__SAML2AUTH__IDP_METADATA__REMOTE_URL=.*|CKANEXT__SAML2AUTH__IDP_METADATA__REMOTE_URL=${{ secrets.SAML_REMOTE_URL }}|g
          s|^CKANEXT__SAML2AUTH__ENTITY_ID=.*|CKANEXT__SAML2AUTH__ENTITY_ID=${{ secrets.SAML_IDENTITY_ID }}|g
          s|^CKAN_SMDH__AWS_STORAGE_BUCKET_NAME=.*|CKAN_SMDH__AWS_STORAGE_BUCKET_NAME=${{ secrets.S3_NAME }}|g
-         s|^CKAN_SMDH__AWS_STORAGE_BUCKET_REGION=.*|CKAN_SMDH__AWS_STORAGE_BUCKET_REGION=${{ secrets.S3_REG }}|g' \
+         s|^CKAN_SMDH__AWS_STORAGE_BUCKET_REGION=.*|CKAN_SMDH__AWS_STORAGE_BUCKET_REGION=${{ secrets.S3_REG }}|g
+         s|^CKANEXT__CLOUDSTORAGE__SYNC__QUEUE_NAME=.*|CKANEXT__CLOUDSTORAGE__SYNC__QUEUE_NAME=${{ secrets.CKANEXT__CLOUDSTORAGE__SYNC__QUEUE_NAME }}|g
+         s|^CKANEXT__CLOUDSTORAGE__SYNC__QUEUE_URL=.*|CKANEXT__CLOUDSTORAGE__SYNC__QUEUE_URL=${{ secrets.CKANEXT__CLOUDSTORAGE__SYNC__QUEUE_URL }}|g' \
          "contrib/docker-ckan/prod.env.template" > "contrib/docker-ckan/.env"
 
          echo INTERNAL_REG=${{ secrets.CONTAINER_REGISTRY_URL }} >> contrib/docker-ckan/.env

contrib/docker-ckan/base/Dockerfile

@@ -90,14 +90,17 @@ RUN mkdir -p ${CKAN_STORAGE_PATH} && \
 
 COPY ./contrib/docker-ckan/base/setup/prerun.py ${APP_DIR}
 COPY ./contrib/docker-ckan/base/setup/prerun_prod.py ${APP_DIR}
+COPY ./contrib/docker-ckan/base/setup/probe_worker.sh ${APP_DIR}
+COPY ./contrib/docker-ckan/base/setup/cron_service.conf ${APP_DIR}
+COPY ./contrib/docker-ckan/base/setup/start_cron.sh ${APP_DIR}
 COPY ./contrib/docker-ckan/base/setup/start_ckan.sh ${APP_DIR}
 ADD https://raw.githubusercontent.com/SmdhMdep/ckan-smdh/main/wsgi.py ${APP_DIR}
 RUN chmod 644 ${APP_DIR}/wsgi.py
 
 # Create entrypoint directory for children image scripts
 ONBUILD RUN mkdir /docker-entrypoint.d
 
-RUN chmod +x ${APP_DIR}/start_ckan.sh
+RUN chmod +x ${APP_DIR}/start_ckan.sh ${APP_DIR}/probe_worker.sh ${APP_DIR}/start_cron.sh
 
 EXPOSE 5000
 

contrib/docker-ckan/base/setup/cron_service.conf

@@ -0,0 +1,8 @@
+[program:cron]
+command=crond -f
+autostart=true
+autorestart=true
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0

contrib/docker-ckan/base/setup/probe_worker.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+log() {
+    echo "$(date -u -Iseconds) $1 $2" >&2
+}
+
+count=$(rq info --url "$CKAN_REDIS_URL" --only-workers --raw | wc -l)
+
+if [[ "$count" > 0 ]]
+then
+    log INFO "$count workers are running"
+    exit 0
+else
+    log ERROR "no workers are running"
+    exit 1
+fi

contrib/docker-ckan/base/setup/start_ckan.sh

@@ -107,15 +107,25 @@ echo "Loading Cloudstorage settings into ckan.ini"
 ckan config-tool $CKAN_INI \
     "ckanext.cloudstorage.driver = $CKANEXT__CLOUDSTORAGE__DRIVER" \
     "ckanext.cloudstorage.driver_options = $CKANEXT__CLOUDSTORAGE__DRIVER_OPTIONS" \
-    "ckanext.cloudstorage.container_name = $CKANEXT__CLOUDSTORAGE__CONTAINER_NAME"
+    "ckanext.cloudstorage.container_name = $CKANEXT__CLOUDSTORAGE__CONTAINER_NAME" \
+    "ckanext.cloudstorage.sync.queue_region = $CKANEXT__CLOUDSTORAGE__SYNC__QUEUE_REGION" \
+    "ckanext.cloudstorage.sync.queue_url = $CKANEXT__CLOUDSTORAGE__SYNC__QUEUE_URL"
 
 if [ $? -eq 0 ]
 then
     # Start supervisord
-    supervisord --configuration /etc/supervisord.conf &
+    supervisord --configuration /etc/supervisord.conf
     # Start uwsgi
-    echo "STARTING CKAN..."
-    uwsgi $UWSGI_OPTS
+    if [[ "$WORKER_PROCESS" != "true" ]]
+    then
+        echo "STARTING CKAN SERVER..."
+        uwsgi $UWSGI_OPTS
+    else
+        echo "STARTING CRON..."
+        ./start_cron.sh
+        echo "STARTING CKAN WORKER..."
+        su ckan -c "/usr/bin/ckan -c $CKAN_INI jobs worker"
+    fi
 else
   echo "[prerun] failed...not starting CKAN."
 fi

contrib/docker-ckan/base/setup/start_cron.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+crontab_file=$(mktemp)
+
+# add your cron jobs below
+cat <<EOF > $crontab_file
+# min   hour    day     month   weekday command
+*       *       *       *       *       ckan -c $CKAN_INI cloudstorage sync
+
+EOF
+
+echo "updating crontab file"
+
+crontab -l | cat - "$crontab_file" | crontab -
+
+crontab -l
+
+echo "starting cron service configuration"
+
+cat cron_service.conf >> /etc/supervisord.conf
+
+supervisorctl -c /etc/supervisord.conf reload

contrib/docker-ckan/base/setup/supervisord.conf

@@ -20,4 +20,4 @@ serverurl = unix:///tmp/supervisor.sock
 supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
 
 [include]
-files = /etc/supervisord.d/*.conf
+files = /etc/supervisord.d/*.conf

contrib/docker-ckan/dev-base/setup/start_ckan_development.sh

@@ -130,7 +130,9 @@ echo "Loading Cloudstorage settings into ckan.ini"
 ckan config-tool $CKAN_INI \
     "ckanext.cloudstorage.driver = $CKANEXT__CLOUDSTORAGE__DRIVER" \
     "ckanext.cloudstorage.driver_options = $CKANEXT__CLOUDSTORAGE__DRIVER_OPTIONS" \
-    "ckanext.cloudstorage.container_name = $CKANEXT__CLOUDSTORAGE__CONTAINER_NAME"
+    "ckanext.cloudstorage.container_name = $CKANEXT__CLOUDSTORAGE__CONTAINER_NAME" \
+    "ckanext.cloudstorage.sync.queue_region = $CKANEXT__CLOUDSTORAGE__SYNC__QUEUE_REGION" \
+    "ckanext.cloudstorage.sync.queue_url = $CKANEXT__CLOUDSTORAGE__SYNC__QUEUE_URL"
 
 # Run any startup scripts provided by images extending this one
 if [[ -d "/docker-entrypoint.d" ]]
@@ -149,4 +151,13 @@ fi
 supervisord --configuration /etc/supervisord.conf &
 
 # Start the development server as the ckan user with automatic reload
-su ckan -c "/usr/bin/ckan -c $CKAN_INI run -H 0.0.0.0"
+if [[ "$WORKER_PROCESS" != "true" ]]
+then
+    echo "STARTING CKAN SERVER..."
+    su ckan -c "/usr/bin/ckan -c $CKAN_INI run -H 0.0.0.0"
+else
+    echo "STARTING CRON..."
+    ./start_cron.sh
+    echo "STARTING CKAN WORKER..."
+    su ckan -c "/usr/bin/ckan -c $CKAN_INI jobs worker"
+fi

contrib/docker-ckan/dev.env.template

@@ -110,6 +110,9 @@ CKANEXT__DATASCI_SHARING__AWS_SECRET_ACCESS_KEY=$CKAN_SMDH__AWS_SECRET_ACCESS_KE
 CKANEXT__CLOUDSTORAGE__DRIVER=S3
 CKANEXT__CLOUDSTORAGE__CONTAINER_NAME=$CKAN_SMDH__AWS_STORAGE_BUCKET_NAME
 CKANEXT__CLOUDSTORAGE__DRIVER_OPTIONS={"key": "$CKAN_SMDH__AWS_ACCESS_KEY_ID", "secret": "$CKAN_SMDH__AWS_SECRET_ACCESS_KEY", "region": "$CKAN_SMDH__AWS_STORAGE_BUCKET_REGION"}
+CKANEXT__CLOUDSTORAGE__SYNC__QUEUE_REGION=$CKAN_SMDH__AWS_STORAGE_BUCKET_REGION
+CKANEXT__CLOUDSTORAGE__SYNC__QUEUE_NAME=
+CKANEXT__CLOUDSTORAGE__SYNC__QUEUE_URL=
 
 
 INTERNAL_REG=

contrib/docker-ckan/docker-compose.dev.yml

@@ -7,7 +7,7 @@ volumes:
 
 services:
 
-  ckan-dev:
+  ckan-dev: &ckan
     container_name: ${CKAN_CONTAINER_NAME}
     build:
       context: dev-deploy/
@@ -26,14 +26,30 @@ services:
         condition: service_healthy
     ports:
       - "0.0.0.0:${CKAN_PORT_HOST}:${CKAN_PORT}"
-
     volumes:
       - ckan_storage:/var/lib/ckan
       - ./src:/srv/app/src_extensions
     restart: unless-stopped
     healthcheck:
       test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"]
-
+
+  ckan-dev-worker:
+    <<: *ckan
+    container_name: ${WORKER_CONTAINER_NAME}
+    image: ${COMPOSE_PROJECT_NAME}-ckan-dev
+    depends_on:
+      db:
+        condition: service_healthy
+      solr:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    ports: []
+    environment:
+      - WORKER_PROCESS=true
+    healthcheck:
+      test: ["CMD", "/srv/app/probe_worker.sh"]
+
   datapusher:
     container_name: ${DATAPUSHER_CONTAINER_NAME}
     image: ckan/ckan-base-datapusher:${DATAPUSHER_VERSION}
@@ -66,6 +82,8 @@ services:
     restart: unless-stopped
     volumes:
       - solr_data:/var/solr
+    environment:
+      - SOLR_OPTS=-XX:-UseLargePages
     healthcheck:
       test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8983/solr/"]
 

contrib/docker-ckan/docker-compose.prod.yml

@@ -14,10 +14,16 @@ x-aws-cloudformation:
         Protocol: HTTPS
         Port: 443
       Type: AWS::ElasticLoadBalancingV2::Listener
+    # compose ecs integration doesn't handle 0 min percent for update config.
+    CkanworkerService:
+      Properties:
+        DeploymentConfiguration:
+          MaximumPercent: 100
+          MinimumHealthyPercent: 0
 
 services:
   ckan:
-    x-aws-role:
+    x-aws-role: &ckan-role
       Version: "2012-10-17"
       Statement:
         - Sid: S3BucketAccess
@@ -26,7 +32,7 @@ services:
             - s3:*
             - s3-object-lambda:*
           Resource:
-            - arn:aws:s3:::${CKAN_SMDH__AWS_STORAGE_BUCKET_NAME}
+            - arn:aws:s3:::${CKAN_SMDH__AWS_STORAGE_BUCKET_NAME}/*
         - Sid: DataSciSharingIAMAccess
           Effect: Allow
           Action:
@@ -37,7 +43,19 @@ services:
           Resource: "*"
         - Sid: DataSciSharingIAMManagedResourcesActions
           Effect: Allow
-          Action: iam:*
+          Action: [iam:*]
+          Resource:
+            - Fn::Join: [":", ["arn:aws:iam:", Ref: "AWS::AccountId", "policy/ckan-smdh/datasci-sharing/*"]]
+            - Fn::Join: [":", ["arn:aws:iam:", Ref: "AWS::AccountId", "group/ckan-smdh/datasci-sharing/*"]]
+        - Sid: SQSResourceManagement
+          Effect: Allow
+          Action: [sqs:*]
           Resource:
-            - arn:aws:iam::450869586150:policy/ckan-smdh/datasci-sharing/*
-            - arn:aws:iam::450869586150:group/ckan-smdh/datasci-sharing/*
+            Fn::Join: [":", [
+                "arn:aws:sqs:${CKANEXT__CLOUDSTORAGE__SYNC__QUEUE_REGION}",
+                Ref: "AWS::AccountId",
+                "${CKANEXT__CLOUDSTORAGE__SYNC__QUEUE_NAME}",
+              ]]
+
+  ckan-worker:
+    x-aws-role: *ckan-role

contrib/docker-ckan/docker-compose.yml

@@ -15,7 +15,7 @@ volumes:
       gid: 0
 
 services:
-  ckan:
+  ckan: &ckan
     container_name: ${CKAN_CONTAINER_NAME}
     image: ${INTERNAL_REG}/mdepckan/ckan-deploy:latest
     deploy:
@@ -51,7 +51,27 @@ services:
     healthcheck:
       test: ["CMD", "wget", "-qO", "/dev/null", "${CKAN_SITE_URL}/api/3/action/status_show"]
 
-
+  ckan-worker:
+    <<: *ckan
+    container_name: ${WORKER_CONTAINER_NAME}
+    image: ${INTERNAL_REG}/mdepckan/ckan-deploy:latest
+    depends_on:
+      solr:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    deploy:
+      replicas: 1
+      resources:
+          limits:
+            cpus: '4'
+            memory: 16GB
+    ports: []
+    environment:
+      - WORKER_PROCESS=true
+    healthcheck:
+      test: ["CMD", "/srv/app/probe_worker.sh"]
+
   datapusher:
     container_name: ${DATAPUSHER_CONTAINER_NAME}
     image: ckan/ckan-base-datapusher:${DATAPUSHER_VERSION}

contrib/docker-ckan/prod.env.template

@@ -115,3 +115,6 @@ CKANEXT__DATASCI_SHARING__BUCKET_NAME=$CKAN_SMDH__AWS_STORAGE_BUCKET_NAME
 CKANEXT__CLOUDSTORAGE__DRIVER=S3
 CKANEXT__CLOUDSTORAGE__DRIVER_OPTIONS={"region": "$CKAN_SMDH__AWS_STORAGE_BUCKET_REGION"}
 CKANEXT__CLOUDSTORAGE__CONTAINER_NAME=$CKAN_SMDH__AWS_STORAGE_BUCKET_NAME
+CKANEXT__CLOUDSTORAGE__SYNC__QUEUE_REGION=$CKAN_SMDH__AWS_STORAGE_BUCKET_REGION
+CKANEXT__CLOUDSTORAGE__SYNC__QUEUE_NAME=
+CKANEXT__CLOUDSTORAGE__SYNC__QUEUE_URL=