Newer yaml stuff

examples/example_config.yaml

@@ -1,169 +1,198 @@
-chain_create_script:
-    handler: lsst.cmservice.handlers.scripts.ChainCreateScriptHandler
-chain_prepend_script:
-    handler: lsst.cmservice.handlers.scripts.ChainPrependScriptHandler
-chain_collect_script:
-    handler: lsst.cmservice.handlers.scripts.ChainCollectScriptHandler
-tag_inputs_script:
-    handler: lsst.cmservice.handlers.scripts.TagInputsScriptHandler
-tag_create_script:
-    handler: lsst.cmservice.handlers.scripts.TagCreateScriptHandler
-tag_associate_script:
-    handler: lsst.cmservice.handlers.scripts.TagAssociateScriptHandler
-prepare_step_script:
-    handler: lsst.cmservice.handlers.scripts.PrepareStepScriptHandler
-validate_script:
-    handler: lsst.cmservice.handlers.scripts.ValidateScriptHandler
-run_jobs:
-    handler: lsst.cmservice.handlers.elements.RunJobsScriptHandler
-    child_config:
-        spec_block: panda_job
-run_groups:
-    handler: lsst.cmservice.handlers.elements.RunGroupsScriptHandler
-run_steps:
-    handler: lsst.cmservice.handlers.elements.RunStepsScriptHandler
-panda_job:
-    handler: lsst.cmservice.handlers.jobs.PandaJobHandler
-    collections:
-        run: "{root}/{campaign}/{step}/{group}/{script}/{job}"
-        output: "{group_output}"
-    data:
-        rescue: false
-group:
-    collections:
-        group_output: "{root}/{campaign}/{step}/{group}"
-        group_validation: "{root}/{campaign}/{step}/{group}/validate"
-    scripts:
-        - run:
-              spec_block: run_jobs
-        - validate:
-              prerequisites: ['run']
-              spec_block: validate_script
-              collections:
-                  input: "{group_output}"
-                  output: "{group_validation}"
-        - add_to_output:
-              prerequisites: ['validate']
-              spec_block: tag_associate_script
-              collections:
-                  input: "{group_output}"
-                  output: "{campaign_tagged_output}"
-step:
-    collections:
-        step_input: "{root}/{campaign}/{step}/input"
-        step_output: "{root}/{campaign}/{step}_ouput"
-        step_public_output: "{root}/{campaign}/{step}"
-        step_validation: "{root}/{campaign}/{step}/validate"
-    scripts:
-        - prepare:
-              spec_block: prepare_step_script
-              collections:
-                  output: "{step_input}"
-                  inputs: ["{campaign_input}", "{campaign_ancilllary}"]
-        - run:
-              prerequisites: ['prepare']
-              spec_block: run_groups
-        - collect_groups:
-              prerequisites: ['run']
-              spec_block: chain_collect_script
-              collections:
-                  inputs: []
-                  output: "{step_output}"
-        - make_step_public_output:
-              prerequisites: ['collect_groups']
-              spec_block: chain_create_script
-              collections:
-                  inputs: ["{step_output}", "{campaign_input}", "{campaign_ancilllary}"]
-                  output: "{step_public_output}"
-        - validate:
-              prerequisites: ['make_step_public_output']
-              spec_block: validate_script
-              collections:
-                  input: "{step_public_output}"
-                  output: "{step_validation}"
-campaign:
-    collections:
-        root: 'cm/p1'
-        campaign_source: /prod/raw/all
-        campaign_input: "{root}/{campaign}/input"
-        campaign_output: "{root}/{campaign}"
-        campaign_tagged_output: "{root}/{campaign}/_output"
-        campaign_ancillary: "{root}/{campaign}/ancillary"
-        campaign_validation: "{root}/{campaign}/validate"
-    scripts:
-        - tag_inputs:
-              spec_block: tag_inputs_script
-              collections:
-                  input: "{campaign_source}"
-                  output: "{campaign_input}"
-        - ancillary:
-              spec_block: chain_create_script
-              collections:
-                  inputs:
-                      - calib_input
-                      - other_calib_input
-                  output:
-                      - "{campaign_ancillary}"
-        - create_empty_output:
-              spec_block: tag_create_script
-              collections:
-                  output: "{campaign_tagged_output}"
-        - create_chained_output:
-              spec_block: chain_create_script
-              prerequisites: ['tag_inputs', 'ancillary', 'create_empty_output']
-              collections:
-                  inputs:
-                      - "{campaign_ancillary}"
-                      - "{campaign_input}"
-                      - "{campaign_tagged_output}"
-                  output:
-                      - "{campaign_output}"
-        - run:
-              spec_block: run_steps
-              prerequisites: ['create_chained_output']
-        - validate:
-              spec_block: validate_script
-              prerequisites: ['run']
-              collections:
-                  input: "{campaign_output}"
-                  output: "{campaign_validation}"
-    child_config:
-        step1:
-            spec_block: step
-            data:
-                pipeline_yaml: "${OBS_LSST_DIR}/pipelines/imsim/DRP.yaml#step1"
-            child_config:
-                spec_block: group
-                split_methd: by_field_value
-                base_query: "a == 1"
-                split_field: "b"
-                split_vals: ["1", "2", "3"]
-        step2:
-            spec_block: step
-            prerequisites: ['step1']
-            data:
-                pipeline_yaml: "${OBS_LSST_DIR}/pipelines/imsim/DRP.yaml#step2"
-            child_config:
-                spec_block: group
-                split_methd: by_field_value
-                base_query: "a == 1"
-                split_field: "b"
-                split_vals: ["1", "2", "3"]
-        step3:
-            spec_block: step
-            prerequisites: ['step2']
-            data:
-                pipeline_yaml: "${OBS_LSST_DIR}/pipelines/imsim/DRP.yaml#step3"
-            child_config:
-                spec_block: group
-                split_methd: by_field_value
-                base_query: "a == 1"
-                split_field: "b"
-                split_vals: ["1", "2", "3"]
-    data:
-        butler_repo: 'dummy'
-        prod_area: 'output/archive'
-        data_query: "instrument = 'HSC' and exposure < 500"
-        bps_yaml_template: "${CM_CONFIGS}/example_template.yaml"
-        bps_script_template: "${CM_CONFIGS}/example_bps_template.sh"
-        lsst_version: "${WEEKLY}"
+- SpecBlock:
+      name: chain_create_script
+      handler: lsst.cmservice.handlers.scripts.ChainCreateScriptHandler
+- SpecBlock:
+      name: chain_prepend_script
+      handler: lsst.cmservice.handlers.scripts.ChainPrependScriptHandler
+- SpecBlock:
+      name: chain_collect_script
+      handler: lsst.cmservice.handlers.scripts.ChainCollectScriptHandler
+- SpecBlock:
+      name: tag_inputs_script
+      handler: lsst.cmservice.handlers.scripts.TagInputsScriptHandler
+- SpecBlock:
+      name: tag_create_script
+      handler: lsst.cmservice.handlers.scripts.TagCreateScriptHandler
+- SpecBlock:
+      name: tag_associate_script
+      handler: lsst.cmservice.handlers.scripts.TagAssociateScriptHandler
+- SpecBlock:
+      name: prepare_step_script
+      handler: lsst.cmservice.handlers.scripts.PrepareStepScriptHandler
+- SpecBlock:
+      name: validate_script
+      handler: lsst.cmservice.handlers.scripts.ValidateScriptHandler
+- SpecBlock:
+      name: run_jobs
+      handler: lsst.cmservice.handlers.elements.RunJobsScriptHandler
+      child_config:
+          spec_block: panda_job
+- SpecBlock:
+      name: run_groups
+      handler: lsst.cmservice.handlers.elements.RunGroupsScriptHandler
+- SpecBlock:
+      name: run_steps
+      handler: lsst.cmservice.handlers.elements.RunStepsScriptHandler
+- SpecBlock:
+      name: panda_job
+      handler: lsst.cmservice.handlers.jobs.PandaJobHandler
+      collections:
+          run: "{root}/{campaign}/{step}/{group}/{script}/{job}"
+          output: "{group_output}"
+      data:
+          rescue: false
+- SpecBlock:
+      name: group
+      collections:
+          group_output: "{root}/{campaign}/{step}/{group}"
+          group_validation: "{root}/{campaign}/{step}/{group}/validate"
+      scripts:
+          - Script:
+                name: run
+                spec_block: run_jobs
+          - Script:
+                name: validate
+                prerequisites: ['run']
+                spec_block: validate_script
+                collections:
+                    input: "{group_output}"
+                    output: "{group_validation}"
+          - Script:
+                add_to_output:
+                prerequisites: ['validate']
+                spec_block: tag_associate_script
+                collections:
+                    input: "{group_output}"
+                    output: "{campaign_tagged_output}"
+- SpecBlock:
+      name: step
+      collections:
+          step_input: "{root}/{campaign}/{step}/input"
+          step_output: "{root}/{campaign}/{step}_ouput"
+          step_public_output: "{root}/{campaign}/{step}"
+          step_validation: "{root}/{campaign}/{step}/validate"
+      scripts:
+          - Script:
+                name: prepare
+                spec_block: prepare_step_script
+                collections:
+                    output: "{step_input}"
+                    inputs: ["{campaign_input}", "{campaign_ancilllary}"]
+          - Script:
+                name: run
+                prerequisites: ['prepare']
+                spec_block: run_groups
+          - Script:
+                name: collect_groups
+                prerequisites: ['run']
+                spec_block: chain_collect_script
+                collections:
+                    inputs: []
+                    output: "{step_output}"
+          - Script:
+                name: make_step_public_output
+                prerequisites: ['collect_groups']
+                spec_block: chain_create_script
+                collections:
+                    inputs: ["{step_output}", "{campaign_input}", "{campaign_ancilllary}"]
+                    output: "{step_public_output}"
+          - Script:
+                name: validate
+                prerequisites: ['make_step_public_output']
+                spec_block: validate_script
+                collections:
+                    input: "{step_public_output}"
+                    output: "{step_validation}"
+- SpecBlock:
+      name: campaign
+      collections:
+          root: 'cm/p1'
+          campaign_source: /prod/raw/all
+          campaign_input: "{root}/{campaign}/input"
+          campaign_output: "{root}/{campaign}"
+          campaign_tagged_output: "{root}/{campaign}/_output"
+          campaign_ancillary: "{root}/{campaign}/ancillary"
+          campaign_validation: "{root}/{campaign}/validate"
+      scripts:
+          - Script:
+                name: tag_inputs
+                spec_block: tag_inputs_script
+                collections:
+                    input: "{campaign_source}"
+                    output: "{campaign_input}"
+          - Script:
+                name: ancillary
+                spec_block: chain_create_script
+                collections:
+                    inputs:
+                        - calib_input
+                        - other_calib_input
+                    output:
+                        - "{campaign_ancillary}"
+          - Script:
+                name: create_empty_output
+                spec_block: tag_create_script
+                collections:
+                    output: "{campaign_tagged_output}"
+          - Script:
+                name: create_chained_output
+                spec_block: chain_create_script
+                prerequisites: ['tag_inputs', 'ancillary', 'create_empty_output']
+                collections:
+                    inputs:
+                        - "{campaign_ancillary}"
+                        - "{campaign_input}"
+                        - "{campaign_tagged_output}"
+                    output:
+                        - "{campaign_output}"
+          - Script:
+                name: run
+                spec_block: run_steps
+                prerequisites: ['create_chained_output']
+          - Script:
+                name: validate
+                spec_block: validate_script
+                prerequisites: ['run']
+                collections:
+                    input: "{campaign_output}"
+                    output: "{campaign_validation}"
+      child_config:
+          step1:
+              spec_block: step
+              data:
+                  pipeline_yaml: "${OBS_LSST_DIR}/pipelines/imsim/DRP.yaml#step1"
+              child_config:
+                  spec_block: group
+                  split_methd: by_field_value
+                  base_query: "a == 1"
+                  split_field: "b"
+                  split_vals: ["1", "2", "3"]
+          step2:
+              spec_block: step
+              prerequisites: ['step1']
+              data:
+                  pipeline_yaml: "${OBS_LSST_DIR}/pipelines/imsim/DRP.yaml#step2"
+              child_config:
+                  spec_block: group
+                  split_methd: by_field_value
+                  base_query: "a == 1"
+                  split_field: "b"
+                  split_vals: ["1", "2", "3"]
+          step3:
+              spec_block: step
+              prerequisites: ['step2']
+              data:
+                  pipeline_yaml: "${OBS_LSST_DIR}/pipelines/imsim/DRP.yaml#step3"
+              child_config:
+                  spec_block: group
+                  split_methd: by_field_value
+                  base_query: "a == 1"
+                  split_field: "b"
+                  split_vals: ["1", "2", "3"]
+      data:
+          butler_repo: 'dummy'
+          prod_area: 'output/archive'
+          data_query: "instrument = 'HSC' and exposure < 500"
+          bps_yaml_template: "${CM_CONFIGS}/example_template.yaml"
+          bps_script_template: "${CM_CONFIGS}/example_bps_template.sh"
+          lsst_version: "${WEEKLY}"