Combine CWL and modular CWL into a single container image

.github/workflows/build_docker_images.yml

@@ -62,26 +62,3 @@ jobs:
           push: true
           tags: ${{ env.REGISTRY }}/${{ env.SPS_DOCKER_CWL }}:${{ env.TAG }}
           labels: ${{ steps.metascheduler.outputs.labels }}
-  build-sps-docker-cwl-modular:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Log in to the Container registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Extract metadata (tags, labels) for SPS Docker CWL modular image
-        id: metascheduler
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.SPS_DOCKER_CWL_MODULAR }}
-      - name: Build and push SPS Docker CWL modular image
-        uses: docker/build-push-action@v5
-        with:
-          context: ./airflow/docker/cwl
-          file: airflow/docker/cwl/Dockerfile_modular
-          push: true
-          tags: ${{ env.REGISTRY }}/${{ env.SPS_DOCKER_CWL_MODULAR }}:${{ env.TAG }}
-          labels: ${{ steps.metascheduler.outputs.labels }}

airflow/dags/cwl_dag_modular.py

@@ -28,8 +28,8 @@
 from airflow import DAG
 
 # Task constants
-STAGE_IN_WORKFLOW = "https://raw.githubusercontent.com/unity-sds/unity-sps-workflows/refs/heads/220-stage-in-task/demos/cwl_dag_modular_stage_in.cwl"
-STAGE_OUT_WORKFLOW = "https://raw.githubusercontent.com/unity-sds/unity-sps-workflows/refs/heads/220-stage-in-task/demos/cwl_dag_modular_stage_out.cwl"
+STAGE_IN_WORKFLOW = "https://raw.githubusercontent.com/unity-sds/unity-sps-workflows/refs/heads/main/demos/cwl_dag_modular_stage_in.cwl"
+STAGE_OUT_WORKFLOW = "https://raw.githubusercontent.com/unity-sds/unity-sps-workflows/refs/heads/main/demos/cwl_dag_modular_stage_out.cwl"
 LOCAL_DIR = "/shared-task-data"
 
 # The path of the working directory where the CWL workflow is executed
@@ -217,11 +217,12 @@ def setup(ti=None, **context):
     task_id="cwl_task_processing",
     namespace=unity_sps_utils.POD_NAMESPACE,
     name="cwl-task-pod",
-    image=unity_sps_utils.SPS_DOCKER_CWL_IMAGE_MODULAR,
+    image=unity_sps_utils.SPS_DOCKER_CWL_IMAGE,
     service_account_name="airflow-worker",
     in_cluster=True,
     get_logs=True,
     startup_timeout_seconds=1800,
+    cmds=["/usr/share/cwl/docker_cwl_entrypoint_modular.sh"],
     arguments=[
         "-i",
         STAGE_IN_WORKFLOW,

airflow/docker/cwl/Dockerfile

@@ -4,7 +4,7 @@ FROM docker:25.0.3-dind
 
 # install Python
 RUN apk add --update --no-cache python3 && ln -sf python3 /usr/bin/python
-RUN apk add gcc musl-dev linux-headers python3-dev
+RUN apk add gcc musl-dev linux-headers python3-dev jq
 RUN apk add --no-cache python3 py3-pip
 RUN apk add vim
 
@@ -13,13 +13,16 @@ RUN mkdir /usr/share/cwl \
     && cd /usr/share/cwl \
     && python -m venv venv \
     && source venv/bin/activate \
-    && pip install cwltool cwl-runner docker boto3 awscli
+    && pip install cwltool cwl-runner docker boto3 awscli pyyaml
 
 # install nodejs to parse Javascript in CWL files
 RUN apk add --no-cache nodejs npm
 
 # script to execute a generic CWL workflow with arguments
 COPY docker_cwl_entrypoint.sh /usr/share/cwl/docker_cwl_entrypoint.sh
 
+# script to execute a generic CWL workflow with stage in, process, and stage out tasks
+COPY docker_cwl_entrypoint_modular.sh /usr/share/cwl/docker_cwl_entrypoint_modular.sh
+
 WORKDIR /usr/share/cwl
 ENTRYPOINT ["/usr/share/cwl/docker_cwl_entrypoint.sh"]

airflow/plugins/unity_sps_utils.py

@@ -2,14 +2,17 @@
 Module containing common utilities for the Unity Science Processing System.
 """
 
+from datetime import datetime
+
 from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator
 from kubernetes.client import models as k8s
 
 # Shared constants
 POD_NAMESPACE = "sps"  # The Kubernetes namespace within which the Pod is run (it must already exist)
-POD_LABEL = "cwl_task"
+POD_LABEL = "cwl_task" + datetime.now().strftime(
+    "%Y%m%d_%H%M%S_%f"
+)  # unique pod label to assure each job runs on its own pod
 SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.4.0"
-SPS_DOCKER_CWL_IMAGE_MODULAR = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl-mod:2.4.0"
 
 NODE_POOL_DEFAULT = "airflow-kubernetes-pod-operator"
 NODE_POOL_HIGH_WORKLOAD = "airflow-kubernetes-pod-operator-high-workload"