diff --git a/src/lsst/cmservice/models/campaign.py b/src/lsst/cmservice/models/campaign.py index 0e39599a..ebe77686 100644 --- a/src/lsst/cmservice/models/campaign.py +++ b/src/lsst/cmservice/models/campaign.py @@ -1,10 +1,20 @@ +"""Pydantic model for the Campaign tables + +These tables don't have anything beyond +standard Element columns +""" + from .element import ElementCreateMixin, ElementMixin class CampaignCreate(ElementCreateMixin): + """Parameters that are used to create new rows but not in DB tables""" + pass class Campaign(ElementMixin): + """Parameters that are in DB tables and not used to create new rows""" + class Config: orm_mode = True diff --git a/src/lsst/cmservice/models/dependency.py b/src/lsst/cmservice/models/dependency.py index 9676e501..904663cd 100644 --- a/src/lsst/cmservice/models/dependency.py +++ b/src/lsst/cmservice/models/dependency.py @@ -1,16 +1,34 @@ +"""Pydantic model for the StepDependency and ScriptDependency +database classes. + +These are classes that track processing dependencies +between Steps and Scripts, respectively. + +In each case the 'prerequisite' Node must run before the 'dependent' Node +""" + from pydantic import BaseModel class DependencyBase(BaseModel): + """Parameters that are in DB tables and also used to create new rows""" + + # ForeignKey for the prerequiste prereq_id: int + # ForignKey for the dependency depend_id: int class DependencyCreate(DependencyBase): + """Parameters that are used to create new rows but not in DB tables""" + pass class Dependency(DependencyBase): + """Parameters that are in DB tables and not used to create new rows""" + + # primary key id: int class Config: diff --git a/src/lsst/cmservice/models/element.py b/src/lsst/cmservice/models/element.py index 45e2b780..80b122aa 100644 --- a/src/lsst/cmservice/models/element.py +++ b/src/lsst/cmservice/models/element.py @@ -1,28 +1,65 @@ +"""Pydantic model for the Processing Elements + +These are the things that are shared between +'Campaign', 'Step', 'Group', 'Job' and 'Script' +""" + from pydantic import BaseModel from ..common.enums import StatusEnum class ElementBase(BaseModel): + """Parameters that are in DB tables and also used to create new rows""" + + # Local name for this element, unique relative to parent element name: str + + # Parameter Overrides data: dict | str | None = None + + # Overrides for configuring child nodes child_config: dict | str | None = None + + # Overrides for making collection names collections: dict | str | None = None + + # Overrides for which SpecBlocks to use in constructing child Nodes spec_aliases: dict | str | None = None + + # Override for Callback handler class handler: str | None = None class ElementCreateMixin(ElementBase): + """Parameters that are used to create new rows but not in DB tables""" + + # Name of the SpecBlock to use as a template spec_block_name: str + + # Fullname of the parent Node parent_name: str class ElementMixin(ElementBase): + """Parameters that are in DB tables and not used to create new rows""" + + # primary key id: int + + # ForeignKey for SpecBlock spec_block_id: int + + # ForeignKey for Parent Node parent_id: int + + # Full unique name for this Node fullname: str + + # Processing Status status: StatusEnum = StatusEnum.waiting + + # Flag to set if this Node is superseded superseded: bool = False class Config: diff --git a/src/lsst/cmservice/models/group.py b/src/lsst/cmservice/models/group.py index e0b35049..e6a99267 100644 --- a/src/lsst/cmservice/models/group.py +++ b/src/lsst/cmservice/models/group.py @@ -1,9 +1,19 @@ +"""Pydantic model for the Group tables + +These tables don't have anything beyond +standard Element columns +""" + from .element import ElementCreateMixin, ElementMixin class GroupCreate(ElementCreateMixin): + """Parameters that are used to create new rows but not in DB tables""" + pass class Group(ElementMixin): + """Parameters that are in DB tables and not used to create new rows""" + pass diff --git a/src/lsst/cmservice/models/job.py b/src/lsst/cmservice/models/job.py index 8608f5c5..9c0eee49 100644 --- a/src/lsst/cmservice/models/job.py +++ b/src/lsst/cmservice/models/job.py @@ -1,15 +1,26 @@ +"""Pydantic model for the Job tables + +These tables have a few columns beyond the +standard Element columns +""" from .element import ElementBase, ElementCreateMixin, ElementMixin class JobBase(ElementBase): + """Parameters that are in DB tables and also used to create new rows""" + attempt: int = 0 wms_job_id: str | None = None stamp_url: str | None = None class JobCreate(JobBase, ElementCreateMixin): + """Parameters that are used to create new rows but not in DB tables""" + pass class Job(JobBase, ElementMixin): + """Parameters that are in DB tables and not used to create new rows""" + pass diff --git a/src/lsst/cmservice/models/pipetask_error.py b/src/lsst/cmservice/models/pipetask_error.py index d0fecb20..a7d83e45 100644 --- a/src/lsst/cmservice/models/pipetask_error.py +++ b/src/lsst/cmservice/models/pipetask_error.py @@ -1,19 +1,45 @@ +"""Pydantic model for the PipetaskError tables + +These tables represent errors reported +by pipetask report. + +These are errors associated to processing +specific quanta. +""" + from pydantic import BaseModel class PipetaskErrorBase(BaseModel): + """Parameters that are in DB tables and also used to create new rows""" + + # ForeignKey into PipetaskErrorType table + # None means that the error in not yet identified error_type_id: int | None = None + + # ForiegnKey into TaskSet table task_id: int + + # UUID for the quanta that had the error quanta: str + + # Diagnostic message produced by the error diagnostic_message: str + + # Data ID for the quanta that had the error data_id: dict class PipetaskErrorCreate(PipetaskErrorBase): + """Parameters that are used to create new rows but not in DB tables""" + pass class PipetaskError(PipetaskErrorBase): + """Parameters that are in DB tables and not used to create new rows""" + + # Primary Key id: int class Config: diff --git a/src/lsst/cmservice/models/pipetask_error_type.py b/src/lsst/cmservice/models/pipetask_error_type.py index bd1ea99a..2953f2a0 100644 --- a/src/lsst/cmservice/models/pipetask_error_type.py +++ b/src/lsst/cmservice/models/pipetask_error_type.py @@ -1,21 +1,47 @@ +"""Pydantic model for the PipetaskErrorType tables + +These are used to classify the errors +reported by pipetask report. + +A PipetaskError is considered to match a PipetaskErrorType +if the diagnostic_message matches the regexp defined +in the PipetaskErrorType AND the task_name also matches +""" + from pydantic import BaseModel from ..common.enums import ErrorActionEnum, ErrorFlavorEnum, ErrorSourceEnum class PipetaskErrorTypeBase(BaseModel): + """Parameters that are in DB tables and also used to create new rows""" + + # Who reported this error source: ErrorSourceEnum + + # What sort of error is this flavor: ErrorFlavorEnum + + # What action should we take action: ErrorActionEnum + + # What Pipetask is this error tpye associated to task_name: str + + # A regexp to define this error type diagnostic_message: str class PipetaskErrorTypeCreate(PipetaskErrorTypeBase): + """Parameters that are used to create new rows but not in DB tables""" + pass class PipetaskErrorType(PipetaskErrorTypeBase): + """Parameters that are in DB tables and not used to create new rows""" + + # PrimaryKey id: int class Config: diff --git a/src/lsst/cmservice/models/product_set.py b/src/lsst/cmservice/models/product_set.py index 6eb07750..5a14a80c 100644 --- a/src/lsst/cmservice/models/product_set.py +++ b/src/lsst/cmservice/models/product_set.py @@ -1,24 +1,49 @@ +"""Pydantic model for the ProductSet tables + +These are used to keep track of files +produced by workflows and stored in the butler. + +These tables are populated with the output +of pipetask report +""" + from pydantic import BaseModel class ProductSetBase(BaseModel): + """Parameters that are in DB tables and also used to create new rows""" + + # Type of data product name: str + # ForeignKey to the associated Job job_id: int + # ForeignKey to the associated TaskSet task_id: int + # Number of files of this type expected for this task n_expected: int class ProductSetCreate(ProductSetBase): + """Parameters that are used to create new rows but not in DB tables""" + pass class ProductSet(ProductSetBase): + """Parameters that are in DB tables and not used to create new rows""" + + # Primary Key id: int + # Unique Name for the combination of Job/Task/File fullname: str + # Number of files produced n_done: int = 0 + # Number of files not produced because the task failed n_failed: int = 0 + # Number of files not produced because of upstream failures n_failed_upstream: int = 0 + # Number of files missing n_missing: int = 0 class Config: diff --git a/src/lsst/cmservice/models/production.py b/src/lsst/cmservice/models/production.py index bf019a6a..6c99c76f 100644 --- a/src/lsst/cmservice/models/production.py +++ b/src/lsst/cmservice/models/production.py @@ -1,15 +1,25 @@ +"""Pydantic model for the Production tables +""" from pydantic import BaseModel class ProductionBase(BaseModel): + """Parameters that are in DB tables and also used to create new rows""" + + # Unique name for this production name: str class ProductionCreate(ProductionBase): + """Parameters that are used to create new rows but not in DB tables""" + pass class Production(ProductionBase): + """Parameters that are in DB tables and not used to create new rows""" + + # PrimaryKey id: int class Config: diff --git a/src/lsst/cmservice/models/queue.py b/src/lsst/cmservice/models/queue.py index 1ea6773e..57416569 100644 --- a/src/lsst/cmservice/models/queue.py +++ b/src/lsst/cmservice/models/queue.py @@ -1,25 +1,44 @@ +"""Pydantic model for the Queue tables + +This Table keeps track of Elements that are +being processed by daemons. +""" + from datetime import datetime from pydantic import BaseModel class QueueBase(BaseModel): + """Parameters that are in DB tables and also used to create new rows""" + + # Interval between calls to process interval: float = 300.0 + # Options based to process options: dict | str | None = None class QueueCreate(QueueBase): + """Parameters that are used to create new rows but not in DB tables""" + + # Fullname of associated Element fullname: str class Queue(QueueBase): + """Parameters that are in DB tables and not used to create new rows""" + + # PrimaryKey id: int - # element_level: int + # element_level: int #FIXME element_id: int + # When this was added to Queue time_created: datetime + # Time last call to process finished time_updated: datetime + # When processing of this element completed time_finished: datetime | None class Config: diff --git a/src/lsst/cmservice/models/script.py b/src/lsst/cmservice/models/script.py index 0d319336..3669a0b6 100644 --- a/src/lsst/cmservice/models/script.py +++ b/src/lsst/cmservice/models/script.py @@ -4,33 +4,63 @@ class ScriptBase(BaseModel): + """Parameters that are in DB tables and also used to create new rows""" + + # Name for this script name: str + # Attempt number from this script attempt: int = 0 + # Method used to process this script method: ScriptMethodEnum = ScriptMethodEnum.slurm - parent_level: LevelEnum + # Override for Callback handler class handler: str | None = None + # Parameter Overrides data: dict | None = None + # Overrides for configuring child nodes child_config: dict | None = None + # Overrides for making collection names collections: dict | None = None + # URL for script file script_url: str | None = None + # URL used to check script processing stamp_url: str | None = None + # URL for processing log file log_url: str | None = None class ScriptCreate(ScriptBase): + """Parameters that are used to create new rows but not in DB tables""" + + # Name of associated SpecBlock spec_block_name: str + # Name of Parent Node parent_name: str class Script(ScriptBase): + """Parameters that are in DB tables and not used to create new rows""" + + # Primary Key id: int + + # ForeignKey giving associated SpecBlock spec_block_id: int + # Id of parent Node parent_id: int + # Level of parent Node + parent_level: LevelEnum + + # ForeignKey giving associated Campaign c_id: int | None = None + # ForeignKey giving associated Step s_id: int | None = None + # ForeignKey giving associated Gropu g_id: int | None = None + # Unique name for this script fullname: str + # Status of processing status: StatusEnum = StatusEnum.waiting + # True is Script is superseded superseded: bool = False class Config: diff --git a/src/lsst/cmservice/models/script_error.py b/src/lsst/cmservice/models/script_error.py index 72e22863..fc692df7 100644 --- a/src/lsst/cmservice/models/script_error.py +++ b/src/lsst/cmservice/models/script_error.py @@ -1,17 +1,34 @@ +"""Pydantic model for the ScriptError tables + +These tables represent errors reported from +Scripts, i.e., stuff like butler commands or +quantum graph generation failing. +""" + from pydantic import BaseModel class ScriptErrorBase(BaseModel): + """Parameters that are in DB tables and also used to create new rows""" + + # ForeignKey identifying the associated script script_id: int + # Who reported This error source: int + # Message associated to this error diagnostic_message: str class ScriptErrorCreate(ScriptErrorBase): + """Parameters that are used to create new rows but not in DB tables""" + pass class ScriptError(ScriptErrorBase): + """Parameters that are in DB tables and not used to create new rows""" + + # Primary Key id: int class Config: diff --git a/src/lsst/cmservice/models/script_template.py b/src/lsst/cmservice/models/script_template.py index c0debd19..46f34c59 100644 --- a/src/lsst/cmservice/models/script_template.py +++ b/src/lsst/cmservice/models/script_template.py @@ -1,19 +1,39 @@ +"""Pydantic model for the ScriptTemplate tables + +These represent files that provide templates +for bash scripts or pipetask configurations that +have been uploaded to the database +""" from pydantic import BaseModel class ScriptTemplateBase(BaseModel): - spec_id: int + """Parameters that are in DB tables and also used to create new rows""" + + # Name for this name: str + + # Corresponding data data: dict | list | None class ScriptTemplateCreate(ScriptTemplateBase): + """Parameters that are used to create new rows but not in DB tables""" + + # Name of associated Specification spec_name: str class ScriptTemplate(ScriptTemplateBase): + """Parameters that are in DB tables and not used to create new rows""" + + # PrimaryKey id: int + + # ForeignKey giving assocated Specification spec_id: int + + # Unique name for this fullname: str class Config: diff --git a/src/lsst/cmservice/models/specification.py b/src/lsst/cmservice/models/specification.py index 1478480c..292fd47a 100644 --- a/src/lsst/cmservice/models/specification.py +++ b/src/lsst/cmservice/models/specification.py @@ -1,24 +1,46 @@ +"""Pydantic models for the Specification and SpecBlock tables + +These tables provide templates for build processing Nodes +""" from pydantic import BaseModel class SpecBlockBase(BaseModel): - spec_id: int + """Parameters that are in DB tables and also used to create new rows""" + + # Name for this name: str + # Class of associated Handler handler: str | None = None + # General Parameters data: dict | list | None + # Parameters defining associated collection names collections: dict | list | None + # Configuration of child nodes associated to this node child_config: dict | list | None + # Used to override Spec Block Configuration spec_aliases: dict | list | None + # Configuraiton of scripts associated to this Node scripts: dict | list | None class SpecBlockCreate(SpecBlockBase): + """Parameters that are used to create new rows but not in DB tables""" + + # Name of associated Specification spec_name: str class SpecBlock(SpecBlockBase): + """Parameters that are in DB tables and not used to create new rows""" + + # PrimaryKey id: int + + # ForeignKey giving the associated Specification spec_id: int + + # Unique name for this specification fullname: str class Config: @@ -26,14 +48,22 @@ class Config: class SpecificationBase(BaseModel): + """Parameters that are in DB tables and also used to create new rows""" + + # Unique name for this name: str class SpecificationCreate(SpecificationBase): + """Parameters that are used to create new rows but not in DB tables""" + pass class Specification(SpecificationBase): + """Parameters that are in DB tables and not used to create new rows""" + + # Primary Key id: int class Config: @@ -41,5 +71,10 @@ class Config: class SpecificationLoad(BaseModel): + """Parameters need to specifiy loading a Specification file""" + + # Name of the Specficiation to create spec_name: str = "example" + + # Name of the file to load yaml_file: str = "examples/example_config.yaml" diff --git a/src/lsst/cmservice/models/step.py b/src/lsst/cmservice/models/step.py index a7610d71..520e9d44 100644 --- a/src/lsst/cmservice/models/step.py +++ b/src/lsst/cmservice/models/step.py @@ -1,9 +1,20 @@ +"""Pydantic model for the Step tables + +These tables don't have anything beyond +standard Element columns +""" + from .element import ElementCreateMixin, ElementMixin class StepCreate(ElementCreateMixin): + """Parameters that are used to create new rows but not in DB tables""" + pass class Step(ElementMixin): - pass + """Parameters that are in DB tables and not used to create new rows""" + + class Config: + orm_mode = True diff --git a/src/lsst/cmservice/models/task_set.py b/src/lsst/cmservice/models/task_set.py index 48680602..b0ae37d3 100644 --- a/src/lsst/cmservice/models/task_set.py +++ b/src/lsst/cmservice/models/task_set.py @@ -1,22 +1,48 @@ +"""Pydantic model for the ProductSet tables + +These are used to keep track of pipetasks associated +to workflows. + +These tables are populated with the output +of pipetask report +""" + from pydantic import BaseModel class TaskSetBase(BaseModel): + """Parameters that are in DB tables and also used to create new rows""" + + # Name of the Pipetask name: str + + # ForiegnKey giving associated Job job_id: int + + # Number of expected quanta run in the workflow n_expected: int class TaskSetCreate(TaskSetBase): + """Parameters that are used to create new rows but not in DB tables""" + pass class TaskSet(TaskSetBase): + """Parameters that are in DB tables and not used to create new rows""" + + # Primary Key id: int + # Unique Name for the combination of Job/Task fullname: str + + # Number of quanta run sucessfully n_done: int = 0 + # Number of quanta that failed n_failed: int = 0 + # Number of quanta did not run b/c of upstream failures n_failed_upstream: int = 0 class Config: diff --git a/src/lsst/cmservice/models/wms_task_report.py b/src/lsst/cmservice/models/wms_task_report.py index d5707524..637ab019 100644 --- a/src/lsst/cmservice/models/wms_task_report.py +++ b/src/lsst/cmservice/models/wms_task_report.py @@ -20,6 +20,8 @@ class WmsTaskReportBase(BaseModel): class WmsTaskReportCreate(WmsTaskReportBase): + """Parameters that are used to create new rows but not in DB tables""" + pass