Added example specification files

examples/bps_config_template.yaml

@@ -0,0 +1,9 @@
+includeConfigs:
+- ${CTRL_BPS_PANDA_DIR}/config/bps_usdf.yaml
+- ${CM_PROD_DIR}/src/lsst/cm/prod/configs/HSC/test/requestMemory.yaml
+
+executionButler:
+  requestMemory: 64000
+  queue: "SLAC_Rubin_Merge"
+
+numberOfRetries: 3

examples/bps_script_template.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env -S -i CM_PROD_DIR="${CM_PROD_DIR}" HOME="${HOME}" bash
+
+# The shebang lines above are needed b/c setup lsst_distrib in putting
+# the lsst python _after_ the virtual env python in the PATH, which
+# is causing errors
+
+# setup LSST env.
+export WEEKLY='{lsst_version}'
+source /cvmfs/sw.lsst.eu/linux-x86_64/lsst_distrib/${WEEKLY}/loadLSST.bash
+setup lsst_distrib
+
+# setup PanDA env.
+latest_panda=$(ls -td /cvmfs/sw.lsst.eu/linux-x86_64/panda_env/v* | head -1)
+setupScript=${latest_panda}/setup_panda_s3df.sh
+source $setupScript ${WEEKLY}
+
+env | grep PANDA
+
+# let's drop a panda_auth status here for kicks
+panda_auth status

examples/config_dc2_steps.yaml

@@ -0,0 +1,121 @@
+# This is an example of the template specification file for Steps used in DC2 campaigns
+
+# Import the common element templates
+- import: "${CM_CONFIGS}/config_standard_elements.yaml"
+# Now define the steps
+- SpecBlock:
+      name: dc2_step1
+      includes: ['step']
+      data:
+          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/LSSTCam-imSim/DRP-test-med-1.yaml#step1"
+      child_config:
+          spec_block: group
+          base_query: "instrument='LSSTCam-imSim' and skymap='DC2'"
+          split_method: split_by_query
+          split_dataset: raw
+          split_field: exposure
+- SpecBlock:
+      name: dc2_step2
+      includes: ['step']
+      data:
+          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/LSSTCam-imSim/DRP-test-med-1.yaml#step2"
+      child_config:
+          spec_block: group
+          base_query: "instrument='LSSTCam-imSim' and skymap='DC2'"
+          split_method: no_split
+- SpecBlock:
+      name: dc2_step3
+      includes: ['step']
+      data:
+          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/LSSTCam-imSim/DRP-test-med-1.yaml#step3"
+      child_config:
+          spec_block: group
+          base_query: "instrument='LSSTCam-imSim' and skymap='DC2'"
+          split_method: split_by_vals
+          split_field: tract
+- SpecBlock:
+      name: dc2_step4
+      includes: ['step']
+      data:
+          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/LSSTCam-imSim/DRP-test-med-1.yaml#step4"
+      child_config:
+          spec_block: group
+          base_query: "instrument='LSSTCam-imSim' and skymap='DC2'"
+          split_method: split_by_query
+          split_dataset: calexp
+          split_field: visit
+- SpecBlock:
+      name: dc2_step5
+      includes: ['step']
+      data:
+          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/LSSTCam-imSim/DRP-test-med-1.yaml#step5"
+      child_config:
+          spec_block: group
+          base_query: "instrument='LSSTCam-imSim' and skymap='DC2'"
+          split_method: split_by_vals
+          split_field: tract
+- SpecBlock:
+      name: dc2_step6
+      includes: ['step']
+      data:
+          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/LSSTCam-imSim/DRP-test-med-1.yaml#step6"
+      child_config:
+          spec_block: group
+          base_query: "instrument='LSSTCam-imSim' and skymap='DC2'"
+          split_method: split_by_query
+          split_dataset: calexp
+          split_field: visit
+- SpecBlock:
+      name: dc2_step7
+      includes: ['step']
+      data:
+          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/LSSTCam-imSim/DRP-test-med-1.yaml#step7"
+      child_config:
+          spec_block: group
+          base_query: "instrument='LSSTCam-imSim' and skymap='DC2'"
+          split_method: no_split
+- SpecBlock:
+      name: dc2_step8
+      includes: ['step']
+      data:
+          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/LSSTCam-imSim/DRP-test-med-1.yaml#step8"
+      child_config:
+          spec_block: group
+          base_query: "instrument='LSSTCam-imSim' and skymap='DC2'"
+          split_method: no_split
+- SpecBlock:
+      name: dc2_faro_visit
+      includes: ['step']
+      data:
+          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/LSSTCam-imSim/DRP-test-med-1.yaml#faro_visit"
+      child_config:
+          spec_block: group
+          base_query: "instrument='LSSTCam-imSim' and skymap='DC2'"
+          split_method: no_split
+- SpecBlock:
+      name: dc2_faro_matched
+      includes: ['step']
+      data:
+          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/LSSTCam-imSim/DRP-test-med-1.yaml#faro_matched"
+      child_config:
+          spec_block: group
+          base_query: "instrument='LSSTCam-imSim' and skymap='DC2'"
+          split_method: no_split
+- SpecBlock:
+      name: dc2_faro_tract
+      includes: ['step']
+      data:
+          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/LSSTCam-imSim/DRP-test-med-1.yaml#faro_tract"
+      child_config:
+          spec_block: group
+          base_query: "instrument='LSSTCam-imSim' and skymap='DC2'"
+          split_method: no_split
+- SpecBlock:
+      name: dc2_plots
+      includes: ['step']
+      data:
+          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/LSSTCam-imSim/DRP-test-med-1.yaml#analysis_coadd_plots"
+      child_config:
+          spec_block: group
+          base_query: "instrument='LSSTCam-imSim' and skymap='DC2'"
+          split_method: no_split

examples/config_dc2_test_med.yaml

@@ -0,0 +1,85 @@
+# This is an example of the campaign level template specification file for dc2-test-med type campaigns
+
+# Import the Step templates for DC2
+- import: "${CM_CONFIGS}/config_dc2_steps.yaml"
+# Now build the Campaign Template for DC2 test med
+- SpecBlock:
+      name: d2c_campaign
+      # Define collections associated with the campaign
+      collections:
+          root: 'cm/dc2_test_med'
+      # Define the steps, their connections and override parameters as needed
+      child_config:
+          step1:
+              spec_block: dc2_step1
+              child_config:
+                  base_query: "instrument='LSSTCam-imSim' and skymap='DC2' and tract in (3828, 3829)"
+                  split_min_groups: 3
+          step2:
+              spec_block: dc2_step2
+              prerequisites: ['step1']
+              child_config:
+                  base_query: "instrument='LSSTCam-imSim' and skymap='DC2' and tract in (3828, 3829)"
+          step3:
+              spec_block: dc2_step3
+              prerequisites: ['step2']
+              child_config:
+                  base_query: "instrument='LSSTCam-imSim' and skymap='DC2'"
+                  split_vals:
+                      - 3828
+                      - 3829
+          step4:
+              spec_block: dc2_step4
+              prerequisites: ['step3']
+              child_config:
+                  base_query: "instrument='LSSTCam-imSim' and skymap='DC2' and tract in (3828, 3829)"
+                  split_min_groups: 4
+          step5:
+              spec_block: dc2_step5
+              prerequisites: ['step4']
+              child_config:
+                  base_query: "instrument='LSSTCam-imSim' and skymap='DC2' and tract in (3828, 3829)"
+                  split_vals:
+                      - 3828
+                      - 3829
+          step6:
+              spec_block: dc2_step6
+              prerequisites: ['step4']
+              child_config:
+                  base_query: "instrument='LSSTCam-imSim' and skymap='DC2' and tract in (3828, 3829)"
+                  split_min_groups: 4
+          step7:
+              spec_block: dc2_step7
+              prerequisites: ['step3']
+              child_config:
+                  base_query: "instrument='LSSTCam-imSim' and skymap='DC2' and tract in (3828, 3829)"
+          step8:
+              spec_block: dc2_step8
+              prerequisites: ['step3']
+              child_config:
+                  base_query: "instrument='LSSTCam-imSim' and skymap='DC2' and tract in (3828, 3829)"
+          faro_visit:
+              spec_block: dc2_faro_visit
+              prerequisites: ['step6']
+              child_config:
+                  base_query: "instrument='LSSTCam-imSim' and skymap='DC2' and tract in (3828, 3829)"
+          faro_matched:
+              spec_block: dc2_faro_matched
+              prerequisites: ['step6']
+              child_config:
+                  base_query: "instrument='LSSTCam-imSim' and skymap='DC2' and tract in (3828, 3829)"
+          faro_tract:
+              spec_block: dc2_faro_tract
+              prerequisites: ['step3']
+              child_config:
+                  base_query: "instrument='LSSTCam-imSim' and skymap='DC2' and tract in (3828, 3829)"
+          plots:
+              spec_block: dc2_plots
+              prerequisites: ['step3']
+              child_config:
+                  base_query: "instrument='LSSTCam-imSim' and skymap='DC2' and tract in (3828, 3829)"
+      # Campaign level paramters
+      data:
+          butler_repo: '/repo/dc2'
+          data_query: "instrument='LSSTCam-imSim' and skymap='DC2' and tract in (3828, 3829)"
+          lsst_version: "${WEEKLY}"

examples/config_standard_elements.yaml

@@ -0,0 +1,137 @@
+# This is an example of the template specification file for the standard cm-service processing setup
+# for 'Elements', i.e., Jobs, Group, Steps, Campaign
+
+
+# Import the common script templates
+- import: "${CM_CONFIGS}/config_standard_scripts.yaml"
+# Now define the templates for the standard elements
+- SpecBlock:
+      # Job Template, runs a single workflow
+      name: job
+      handler: lsst.cmservice.handlers.job_handler.JobHandler
+      collections:
+          # This is the output RUN collection for the Job
+          job_run: "{root}/{campaign}/{step}/{group}/{job}"
+      scripts:
+          # The bps script and configuration
+          - Script:
+                name: bps
+                spec_block: panda_script
+                collections:
+                    run: "{job_run}"
+                    inputs: ["{step_input}", "{campaign_input}", "{campaign_ancillary}"]
+          # Uses bps report to collect information on the bps job and know when it is done
+          - Script:
+                name: bps_report
+                spec_block: panda_report_script
+                prerequisites: ['bps']
+                collections:
+                    run: "{job_run}"
+                    inputs: ["{step_input}", "{campaign_input}", "{campaign_ancillary}"]
+          # Uses manifest checker to gather information about the workflow
+          - Script:
+                name: manifest_report
+                spec_block: manifest_report_script
+                prerequisites: ['bps_report']
+                collections:
+                    run: "{job_run}"
+      data:
+          rescue: false
+- SpecBlock:
+      # Group Template, runs a group, hopefully using a single job to do so
+      name: group
+      handler: lsst.cmservice.handlers.element_handler.ElementHandler
+      collections:
+          group_output: "{root}/{campaign}/{step}/{group}"
+          group_validation: "{root}/{campaign}/{step}/{group}/validate"
+      scripts:
+          - Script:
+                name: run
+                spec_block: run_jobs
+      child_config:
+          spec_block: job
+- SpecBlock:
+      name: step
+      handler: lsst.cmservice.handlers.element_handler.ElementHandler
+      # Define collections assocated to the step
+      collections:
+          step_input: "{root}/{campaign}/{step}/input"
+          step_output: "{root}/{campaign}/{step}_ouput"
+          step_public_output: "{root}/{campaign}/{step}"
+          step_validation: "{root}/{campaign}/{step}/validate"
+      scripts:
+          # Prepare the step by building input collection
+          - Script:
+                name: prepare
+                spec_block: prepare_step_script
+                collections:
+                    output: "{step_input}"
+                    inputs: ["{campaign_input}", "{campaign_ancillary}"]
+          # Prepare the groups for this step
+          - Script:
+                name: run
+                prerequisites: ['prepare']
+                spec_block: run_groups
+          # Build the step output collection by chaining the job RUN collections
+          - Script:
+                name: collect_groups
+                prerequisites: ['run']
+                spec_block: chain_collect_jobs_script
+                collections:
+                    inputs: []
+                    output: "{step_output}"
+          # Build the 'public' step output collection by also chaining in the campagin inputs
+          - Script:
+                name: make_step_public_output
+                prerequisites: ['collect_groups']
+                spec_block: chain_create_script
+                collections:
+                    inputs: ["{step_output}", "{campaign_input}", "{campaign_ancillary}"]
+                    output: "{step_public_output}"
+# Now build the generic Campaign Template
+- SpecBlock:
+      name: campaign
+      handler: lsst.cmservice.handlers.element_handler.ElementHandler
+      # Define collections associated with the campaign
+      collections:
+          campaign_source: /prod/raw/all
+          campaign_input: "{root}/{campaign}/input"
+          campaign_output: "{root}/{campaign}"
+          campaign_ancillary: "{root}/{campaign}/ancillary"
+          campaign_validation: "{root}/{campaign}/validate"
+      # Define scripts associate with the campaign
+      scripts:
+          # Tag the inputs for the campaign
+          - Script:
+                name: tag_inputs
+                spec_block: tag_inputs_script
+                collections:
+                    input: "{campaign_source}"
+                    output: "{campaign_input}"
+          # Create the ancillary chain for the campaign
+          - Script:
+                name: ancillary
+                spec_block: chain_create_script
+                collections:
+                    inputs:
+                        - calib_input
+                        - other_calib_input
+                    output: "{campaign_ancillary}"
+          # Run the steps of the campaign
+          - Script:
+                name: run
+                spec_block: run_steps
+                prerequisites: ['tag_inputs', 'ancillary']
+          # Build the chained collection for the campaign
+          - Script:
+                name: collect_steps
+                prerequisites: ['run']
+                spec_block: chain_collect_steps_script
+                collections:
+                    inputs: []
+                    output: "{campaign_output}"
+      data:
+          prod_area: 'output/archive'
+          bps_yaml_template: "${CM_CONFIGS}/bps_config_template.yaml"
+          bps_script_template: "${CM_CONFIGS}/bps_script_template.sh"
+          manifest_script_template: "${CM_CONFIGS}/manifest_script_template.sh"