Skip to content

Commit

Permalink
look at all the pretty comments
Browse files Browse the repository at this point in the history
  • Loading branch information
eacharles authored and fritzm committed Dec 18, 2023
1 parent 91d67b0 commit d32e87d
Show file tree
Hide file tree
Showing 17 changed files with 338 additions and 5 deletions.
10 changes: 10 additions & 0 deletions src/lsst/cmservice/models/campaign.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,20 @@
"""Pydantic model for the Campaign tables
These tables don't have anything beyond
standard Element columns
"""

from .element import ElementCreateMixin, ElementMixin


class CampaignCreate(ElementCreateMixin):
"""Parameters that are used to create new rows but not in DB tables"""

pass


class Campaign(ElementMixin):
"""Parameters that are in DB tables and not used to create new rows"""

class Config:
orm_mode = True
18 changes: 18 additions & 0 deletions src/lsst/cmservice/models/dependency.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,34 @@
"""Pydantic model for the StepDependency and ScriptDependency
database classes.
These are classes that track processing dependencies
between Steps and Scripts, respectively.
In each case the 'prerequisite' Node must run before the 'dependent' Node
"""

from pydantic import BaseModel


class DependencyBase(BaseModel):
"""Parameters that are in DB tables and also used to create new rows"""

# ForeignKey for the prerequiste
prereq_id: int
# ForignKey for the dependency
depend_id: int


class DependencyCreate(DependencyBase):
"""Parameters that are used to create new rows but not in DB tables"""

pass


class Dependency(DependencyBase):
"""Parameters that are in DB tables and not used to create new rows"""

# primary key
id: int

class Config:
Expand Down
37 changes: 37 additions & 0 deletions src/lsst/cmservice/models/element.py
Original file line number Diff line number Diff line change
@@ -1,28 +1,65 @@
"""Pydantic model for the Processing Elements
These are the things that are shared between
'Campaign', 'Step', 'Group', 'Job' and 'Script'
"""

from pydantic import BaseModel

from ..common.enums import StatusEnum


class ElementBase(BaseModel):
"""Parameters that are in DB tables and also used to create new rows"""

# Local name for this element, unique relative to parent element
name: str

# Parameter Overrides
data: dict | str | None = None

# Overrides for configuring child nodes
child_config: dict | str | None = None

# Overrides for making collection names
collections: dict | str | None = None

# Overrides for which SpecBlocks to use in constructing child Nodes
spec_aliases: dict | str | None = None

# Override for Callback handler class
handler: str | None = None


class ElementCreateMixin(ElementBase):
"""Parameters that are used to create new rows but not in DB tables"""

# Name of the SpecBlock to use as a template
spec_block_name: str

# Fullname of the parent Node
parent_name: str


class ElementMixin(ElementBase):
"""Parameters that are in DB tables and not used to create new rows"""

# primary key
id: int

# ForeignKey for SpecBlock
spec_block_id: int

# ForeignKey for Parent Node
parent_id: int

# Full unique name for this Node
fullname: str

# Processing Status
status: StatusEnum = StatusEnum.waiting

# Flag to set if this Node is superseded
superseded: bool = False

class Config:
Expand Down
10 changes: 10 additions & 0 deletions src/lsst/cmservice/models/group.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,19 @@
"""Pydantic model for the Group tables
These tables don't have anything beyond
standard Element columns
"""

from .element import ElementCreateMixin, ElementMixin


class GroupCreate(ElementCreateMixin):
"""Parameters that are used to create new rows but not in DB tables"""

pass


class Group(ElementMixin):
"""Parameters that are in DB tables and not used to create new rows"""

pass
11 changes: 11 additions & 0 deletions src/lsst/cmservice/models/job.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,26 @@
"""Pydantic model for the Job tables
These tables have a few columns beyond the
standard Element columns
"""
from .element import ElementBase, ElementCreateMixin, ElementMixin


class JobBase(ElementBase):
"""Parameters that are in DB tables and also used to create new rows"""

attempt: int = 0
wms_job_id: str | None = None
stamp_url: str | None = None


class JobCreate(JobBase, ElementCreateMixin):
"""Parameters that are used to create new rows but not in DB tables"""

pass


class Job(JobBase, ElementMixin):
"""Parameters that are in DB tables and not used to create new rows"""

pass
26 changes: 26 additions & 0 deletions src/lsst/cmservice/models/pipetask_error.py
Original file line number Diff line number Diff line change
@@ -1,19 +1,45 @@
"""Pydantic model for the PipetaskError tables
These tables represent errors reported
by pipetask report.
These are errors associated to processing
specific quanta.
"""

from pydantic import BaseModel


class PipetaskErrorBase(BaseModel):
"""Parameters that are in DB tables and also used to create new rows"""

# ForeignKey into PipetaskErrorType table
# None means that the error in not yet identified
error_type_id: int | None = None

# ForiegnKey into TaskSet table
task_id: int

# UUID for the quanta that had the error
quanta: str

# Diagnostic message produced by the error
diagnostic_message: str

# Data ID for the quanta that had the error
data_id: dict


class PipetaskErrorCreate(PipetaskErrorBase):
"""Parameters that are used to create new rows but not in DB tables"""

pass


class PipetaskError(PipetaskErrorBase):
"""Parameters that are in DB tables and not used to create new rows"""

# Primary Key
id: int

class Config:
Expand Down
26 changes: 26 additions & 0 deletions src/lsst/cmservice/models/pipetask_error_type.py
Original file line number Diff line number Diff line change
@@ -1,21 +1,47 @@
"""Pydantic model for the PipetaskErrorType tables
These are used to classify the errors
reported by pipetask report.
A PipetaskError is considered to match a PipetaskErrorType
if the diagnostic_message matches the regexp defined
in the PipetaskErrorType AND the task_name also matches
"""

from pydantic import BaseModel

from ..common.enums import ErrorActionEnum, ErrorFlavorEnum, ErrorSourceEnum


class PipetaskErrorTypeBase(BaseModel):
"""Parameters that are in DB tables and also used to create new rows"""

# Who reported this error
source: ErrorSourceEnum

# What sort of error is this
flavor: ErrorFlavorEnum

# What action should we take
action: ErrorActionEnum

# What Pipetask is this error tpye associated to
task_name: str

# A regexp to define this error type
diagnostic_message: str


class PipetaskErrorTypeCreate(PipetaskErrorTypeBase):
"""Parameters that are used to create new rows but not in DB tables"""

pass


class PipetaskErrorType(PipetaskErrorTypeBase):
"""Parameters that are in DB tables and not used to create new rows"""

# PrimaryKey
id: int

class Config:
Expand Down
25 changes: 25 additions & 0 deletions src/lsst/cmservice/models/product_set.py
Original file line number Diff line number Diff line change
@@ -1,24 +1,49 @@
"""Pydantic model for the ProductSet tables
These are used to keep track of files
produced by workflows and stored in the butler.
These tables are populated with the output
of pipetask report
"""

from pydantic import BaseModel


class ProductSetBase(BaseModel):
"""Parameters that are in DB tables and also used to create new rows"""

# Type of data product
name: str
# ForeignKey to the associated Job
job_id: int
# ForeignKey to the associated TaskSet
task_id: int
# Number of files of this type expected for this task
n_expected: int


class ProductSetCreate(ProductSetBase):
"""Parameters that are used to create new rows but not in DB tables"""

pass


class ProductSet(ProductSetBase):
"""Parameters that are in DB tables and not used to create new rows"""

# Primary Key
id: int
# Unique Name for the combination of Job/Task/File
fullname: str

# Number of files produced
n_done: int = 0
# Number of files not produced because the task failed
n_failed: int = 0
# Number of files not produced because of upstream failures
n_failed_upstream: int = 0
# Number of files missing
n_missing: int = 0

class Config:
Expand Down
10 changes: 10 additions & 0 deletions src/lsst/cmservice/models/production.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,25 @@
"""Pydantic model for the Production tables
"""
from pydantic import BaseModel


class ProductionBase(BaseModel):
"""Parameters that are in DB tables and also used to create new rows"""

# Unique name for this production
name: str


class ProductionCreate(ProductionBase):
"""Parameters that are used to create new rows but not in DB tables"""

pass


class Production(ProductionBase):
"""Parameters that are in DB tables and not used to create new rows"""

# PrimaryKey
id: int

class Config:
Expand Down
21 changes: 20 additions & 1 deletion src/lsst/cmservice/models/queue.py
Original file line number Diff line number Diff line change
@@ -1,25 +1,44 @@
"""Pydantic model for the Queue tables
This Table keeps track of Elements that are
being processed by daemons.
"""

from datetime import datetime

from pydantic import BaseModel


class QueueBase(BaseModel):
"""Parameters that are in DB tables and also used to create new rows"""

# Interval between calls to process
interval: float = 300.0
# Options based to process
options: dict | str | None = None


class QueueCreate(QueueBase):
"""Parameters that are used to create new rows but not in DB tables"""

# Fullname of associated Element
fullname: str


class Queue(QueueBase):
"""Parameters that are in DB tables and not used to create new rows"""

# PrimaryKey
id: int

# element_level: int
# element_level: int #FIXME
element_id: int

# When this was added to Queue
time_created: datetime
# Time last call to process finished
time_updated: datetime
# When processing of this element completed
time_finished: datetime | None

class Config:
Expand Down
Loading

0 comments on commit d32e87d

Please sign in to comment.