Eric's Big Adventure

This commit addresses the greater part of Eric's v1 feature-parity work,
formerly the `eac-dev` branch.

This commit has not yet been DM reviewed, but does pass linters, is mypy
clean, and has a functional test suite -- in light of this, the team has
chosen to conduct further review prep work incrementally on `main`.

.gitignore

@@ -8,6 +8,10 @@ __pycache__/
 
 # Unit test / coverage reports
 .coverage
+htmlcov/
+
+# Backup filies
+*~
 
 # Environments
 .env
@@ -21,6 +25,13 @@ venv.bak/
 # mypy
 .mypy_cache/
 
+# ruff
+.ruff_cache/
+
+#pytest
+.pytest_cache/
+output/
+
 # macOS
 .DS_Store/
 

Makefile

@@ -31,7 +31,7 @@ test:
 	export CM_DATABASE_SCHEMA=cm_service_test && \
 	export CM_ARQ_REDIS_URL=redis://localhost:$${CM_ARQ_REDIS_PORT}/1 && \
 	export CM_ARQ_REDIS_PASSWORD=INSECURE-PASSWORD && \
-	pytest -vvv --cov=lsst.cmservice --cov-branch
+	pytest -vvv --cov=lsst.cmservice --cov-branch --cov-report=term --cov-report=html
 
 .PHONY: run
 run:
@@ -44,7 +44,7 @@ run:
 	export CM_ARQ_REDIS_URL=redis://localhost:$${CM_ARQ_REDIS_PORT}/1 && \
 	export CM_ARQ_REDIS_PASSWORD=INSECURE-PASSWORD && \
 	cm-service init && \
-	cm-service run --port=0
+	cm-service run
 
 .PHONY: lint
 lint:

examples/empty_config.yaml

@@ -0,0 +1,110 @@
+- ScriptTemplate:
+      name: bps_panda_script_template
+      file_path: ${CM_CONFIGS}/example_bps_panda_template.yaml
+- ScriptTemplate:
+      name: bps_yaml_template
+      file_path: ${CM_CONFIGS}/example_template.yaml
+- ScriptTemplate:
+      name: manifest_script_template
+      file_path: ${CM_CONFIGS}/example_manifest_template.yaml
+- SpecBlock:
+      name: chain_create_script
+      handler: lsst.cmservice.handlers.scripts.ChainCreateScriptHandler
+- SpecBlock:
+      name: chain_prepend_script
+      handler: lsst.cmservice.handlers.scripts.ChainPrependScriptHandler
+- SpecBlock:
+      name: chain_collect_jobs_script
+      handler: lsst.cmservice.handlers.scripts.ChainCollectScriptHandler
+      data:
+          collect: jobs
+- SpecBlock:
+      name: chain_collect_steps_script
+      handler: lsst.cmservice.handlers.scripts.ChainCollectScriptHandler
+      data:
+          collect: steps
+- SpecBlock:
+      name: tag_inputs_script
+      handler: lsst.cmservice.handlers.scripts.TagInputsScriptHandler
+- SpecBlock:
+      name: tag_create_script
+- SpecBlock:
+      name: tag_associate_script
+      handler: lsst.cmservice.handlers.scripts.TagAssociateScriptHandler
+- SpecBlock:
+      name: prepare_step_script
+      handler: lsst.cmservice.handlers.scripts.PrepareStepScriptHandler
+      collections:
+          global_inputs: "{campaign_input}"
+- SpecBlock:
+      name: validate_script
+      handler: lsst.cmservice.handlers.scripts.ValidateScriptHandler
+- SpecBlock:
+      name: panda_script
+      handler: lsst.cmservice.handlers.jobs.PandaScriptHandler
+- SpecBlock:
+      name: panda_report_script
+      handler: lsst.cmservice.handlers.jobs.PandaReportHandler
+- SpecBlock:
+      name: manifest_report_script
+      handler: lsst.cmservice.handlers.jobs.ManifestReportScriptHandler
+- SpecBlock:
+      name: run_jobs
+      handler: lsst.cmservice.handlers.elements.RunJobsScriptHandler
+- SpecBlock:
+      name: run_groups
+      handler: lsst.cmservice.handlers.elements.RunGroupsScriptHandler
+- SpecBlock:
+      name: run_steps
+      handler: lsst.cmservice.handlers.elements.RunStepsScriptHandler
+- SpecBlock:
+      name: job
+      handler: lsst.cmservice.handlers.job_handler.JobHandler
+      collections:
+          job_run: "{root}/{campaign}/{step}/{group}/{job}"
+- SpecBlock:
+      name: group
+      handler: lsst.cmservice.handlers.element_handler.ElementHandler
+      collections:
+          group_output: "{root}/{campaign}/{step}/{group}"
+          group_validation: "{root}/{campaign}/{step}/{group}/validate"
+      child_config:
+          spec_block: job
+- SpecBlock:
+      name: step
+      handler: lsst.cmservice.handlers.element_handler.ElementHandler
+      collections:
+          step_input: "{root}/{campaign}/{step}/input"
+          step_output: "{root}/{campaign}/{step}_ouput"
+          step_public_output: "{root}/{campaign}/{step}"
+          step_validation: "{root}/{campaign}/{step}/validate"
+- SpecBlock:
+      name: basic_step
+      includes: ["step"]
+      data:
+          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#isr"
+      child_config:
+          spec_block: group
+          base_query: "instrument = 'HSC'"
+          split_method: split_by_query
+          split_dataset: raw
+          split_field: exposure
+          split_min_groups: 2
+- SpecBlock:
+      name: campaign
+      handler: lsst.cmservice.handlers.element_handler.ElementHandler
+      collections:
+          root: 'cm/hsc_rc2_micro'
+          campaign_source: HSC/raw/RC2
+          campaign_input: "{root}/{campaign}/input"
+          campaign_output: "{root}/{campaign}"
+          campaign_ancillary: "{root}/{campaign}/ancillary"
+          campaign_validation: "{root}/{campaign}/validate"
+      data:
+          butler_repo: '/repo/main'
+          prod_area: 'output/archive'
+          data_query: "instrument = 'HSC'"
+          lsst_version: "${WEEKLY}"
+          bps_script_template: bps_panda_script_template
+          bps_yaml_template: bps_yaml_template
+          manifest_script_template: manifest_script_template

examples/error_types.yaml

@@ -0,0 +1,8 @@
+- PipetaskErrorType:
+      source: manifest
+      flavor: configuration
+      action: review
+      task_name: skyObjectMean
+      diagnostic_message: 'Execution of task ''skyObjectMean'' on quantum.*failed. Exception ValueError: Failure
+          from formatter ''lsst.daf.butler.formatters.parquet.ParquetFormatter'' for dataset.*:
+          Column d_pixelFlags_edge specified in parameters not available in parquet file.'

examples/example_bps_panda_template.yaml

@@ -0,0 +1,21 @@
+text: "#!/usr/bin/env -S -i CM_PROD_DIR=\"${CM_PROD_DIR}\" HOME=\"${HOME}\" bash\n
+
+# The shebang lines above are needed b/c setup lsst_distrib is putting\n
+# the lsst python _after_ the virtual env python in the PATH, which\n
+# is causing errors\n
+
+# setup LSST env.\n
+export WEEKLY='{lsst_version}'\n
+source /cvmfs/sw.lsst.eu/linux-x86_64/lsst_distrib/${WEEKLY}/loadLSST.bash\n
+setup lsst_distrib\n
+
+# setup PanDA env.\n
+latest_panda=$(ls -td /cvmfs/sw.lsst.eu/linux-x86_64/panda_env/v* | head -1)\n
+setupScript=${latest_panda}/setup_panda_s3df.sh\n
+source $setupScript ${WEEKLY}\n
+
+env | grep PANDA\n
+
+# let's drop a panda_auth status here for kicks\n
+panda_auth status\n
+"