From b7bae8b6afcfc2291c19d942c1ded0e125bf21c3 Mon Sep 17 00:00:00 2001 From: Fritz Mueller Date: Sat, 4 Nov 2023 12:22:50 -0700 Subject: [PATCH] Ruff autofixes + enable additional checks in pre-commit --- pyproject.toml | 3 + src/lsst/cmservice/cli/commands.py | 3 +- src/lsst/cmservice/cli/options.py | 2 +- src/lsst/cmservice/client.py | 2 + src/lsst/cmservice/common/__init__.py | 0 src/lsst/cmservice/common/bash.py | 7 +- src/lsst/cmservice/common/slurm.py | 12 ++-- src/lsst/cmservice/common/utils.py | 2 +- src/lsst/cmservice/db/campaign.py | 21 +++--- src/lsst/cmservice/db/element.py | 14 ++-- src/lsst/cmservice/db/group.py | 15 +++-- src/lsst/cmservice/db/handler.py | 2 +- src/lsst/cmservice/db/job.py | 42 ++++++------ src/lsst/cmservice/db/node.py | 17 ++--- src/lsst/cmservice/db/pipetask_error.py | 4 +- src/lsst/cmservice/db/pipetask_error_type.py | 4 +- src/lsst/cmservice/db/production.py | 5 +- src/lsst/cmservice/db/queue.py | 15 ++--- src/lsst/cmservice/db/row.py | 3 +- src/lsst/cmservice/db/script.py | 25 +++---- src/lsst/cmservice/db/script_dependency.py | 4 +- src/lsst/cmservice/db/script_template.py | 14 ++-- src/lsst/cmservice/db/specification.py | 24 +++---- src/lsst/cmservice/db/step.py | 27 ++++---- src/lsst/cmservice/db/step_dependency.py | 4 +- src/lsst/cmservice/db/task_set.py | 4 +- .../cmservice/handlers/element_handler.py | 4 +- src/lsst/cmservice/handlers/elements.py | 5 +- src/lsst/cmservice/handlers/functions.py | 17 ++--- src/lsst/cmservice/handlers/interface.py | 43 ++++++------ src/lsst/cmservice/handlers/jobs.py | 9 ++- src/lsst/cmservice/handlers/script_handler.py | 4 +- src/lsst/cmservice/handlers/scripts.py | 5 +- src/lsst/cmservice/models/script_template.py | 4 +- src/lsst/cmservice/models/specification.py | 12 ++-- src/lsst/cmservice/routers/actions.py | 30 +++------ src/lsst/cmservice/routers/adders.py | 9 +-- src/lsst/cmservice/routers/campaigns.py | 6 +- .../cmservice/routers/expert_campaigns.py | 8 +-- src/lsst/cmservice/routers/expert_groups.py | 8 +-- src/lsst/cmservice/routers/expert_jobs.py | 8 +-- .../routers/expert_pipetask_error_types.py | 8 +-- .../routers/expert_pipetask_errors.py | 8 +-- .../cmservice/routers/expert_product_sets.py | 11 ++- .../cmservice/routers/expert_productions.py | 9 +-- src/lsst/cmservice/routers/expert_queues.py | 11 ++- .../routers/expert_script_dependencies.py | 8 +-- .../cmservice/routers/expert_script_errors.py | 8 +-- .../routers/expert_script_templates.py | 8 +-- src/lsst/cmservice/routers/expert_scripts.py | 11 ++- .../cmservice/routers/expert_spec_blocks.py | 8 +-- .../routers/expert_specifications.py | 8 +-- .../routers/expert_step_dependencies.py | 8 +-- src/lsst/cmservice/routers/expert_steps.py | 8 +-- .../cmservice/routers/expert_task_sets.py | 11 ++- src/lsst/cmservice/routers/groups.py | 6 +- src/lsst/cmservice/routers/jobs.py | 8 +-- src/lsst/cmservice/routers/loaders.py | 10 +-- .../cmservice/routers/pipetask_error_types.py | 8 +-- src/lsst/cmservice/routers/productions.py | 8 +-- src/lsst/cmservice/routers/queries.py | 67 +++++++------------ .../cmservice/routers/script_templates.py | 8 +-- src/lsst/cmservice/routers/scripts.py | 8 +-- src/lsst/cmservice/routers/spec_blocks.py | 8 +-- src/lsst/cmservice/routers/steps.py | 6 +- src/lsst/cmservice/routers/updates.py | 15 ++--- tests/db/test_campaign.py | 15 ++++- tests/db/test_group.py | 20 ++++-- tests/db/test_step.py | 15 ++++- 69 files changed, 346 insertions(+), 428 deletions(-) create mode 100644 src/lsst/cmservice/common/__init__.py diff --git a/pyproject.toml b/pyproject.toml index b19f21ca..9feee3e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,6 +94,9 @@ select = [ "F", # pyflakes "N", # pep8-naming "W", # pycodestyle + "COM", # pyflakes-commas + "FBT", # flake8-boolean-trap + "UP", # pyupgrade ] target-version = "py311" extend-select = [ diff --git a/src/lsst/cmservice/cli/commands.py b/src/lsst/cmservice/cli/commands.py index 52f1fd6b..80599b01 100644 --- a/src/lsst/cmservice/cli/commands.py +++ b/src/lsst/cmservice/cli/commands.py @@ -1,5 +1,6 @@ import json -from typing import Any, Sequence, TypeVar +from collections.abc import Sequence +from typing import Any, TypeVar import click import structlog diff --git a/src/lsst/cmservice/cli/options.py b/src/lsst/cmservice/cli/options.py index 76afce5a..03fa1505 100644 --- a/src/lsst/cmservice/cli/options.py +++ b/src/lsst/cmservice/cli/options.py @@ -82,7 +82,7 @@ def convert( # pylint: disable=inconsistent-return-statements keyvalue_pairs = value.rstrip(";").split(";") result_dict = {} for pair in keyvalue_pairs: - key, values = [item.strip() for item in pair.split("=")] + key, values = (item.strip() for item in pair.split("=")) converted_values = [] for value_ in values.split(","): value_ = value_.strip() diff --git a/src/lsst/cmservice/client.py b/src/lsst/cmservice/client.py index dd0a9fce..47efaa63 100644 --- a/src/lsst/cmservice/client.py +++ b/src/lsst/cmservice/client.py @@ -165,6 +165,7 @@ def get_scripts( self, fullname: str, script_name: str, + *, remaining_only: bool = False, skip_superseded: bool = True, ) -> list[models.Script]: @@ -184,6 +185,7 @@ def get_scripts( def get_jobs( self, fullname: str, + *, remaining_only: bool = False, skip_superseded: bool = True, ) -> list[models.Job]: diff --git a/src/lsst/cmservice/common/__init__.py b/src/lsst/cmservice/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/lsst/cmservice/common/bash.py b/src/lsst/cmservice/common/bash.py index 1306c51d..8733efc2 100644 --- a/src/lsst/cmservice/common/bash.py +++ b/src/lsst/cmservice/common/bash.py @@ -43,10 +43,9 @@ async def check_stamp_file( """ if not os.path.exists(stamp_file): return None - with open(stamp_file, "rt", encoding="utf-8") as fin: + with open(stamp_file, encoding="utf-8") as fin: fields = yaml.safe_load(fin) - status = StatusEnum[fields["status"]] - return status + return StatusEnum[fields["status"]] async def write_bash_script( @@ -102,7 +101,7 @@ async def write_bash_script( except OSError: pass - with open(script_url, "wt", encoding="utf-8") as fout: + with open(script_url, "w", encoding="utf-8") as fout: if prepend: fout.write(f"{prepend}\n") if fake: diff --git a/src/lsst/cmservice/common/slurm.py b/src/lsst/cmservice/common/slurm.py index fb2b1e4c..dc2cc0ca 100644 --- a/src/lsst/cmservice/common/slurm.py +++ b/src/lsst/cmservice/common/slurm.py @@ -58,8 +58,7 @@ async def submit_slurm_job( ) as sbatch: assert sbatch.stdout line = sbatch.stdout.read().decode().strip() - job_id = line.split("|")[0] - return job_id + return line.split("|")[0] except TypeError as msg: raise TypeError(f"Bad slurm submit from {script_url}") from msg @@ -85,12 +84,9 @@ async def check_slurm_job( assert sacct.stdout lines = sacct.stdout.read().decode().split("\n") if len(lines) < 2: - status = slurm_status_map["PENDING"] - return status + return slurm_status_map["PENDING"] tokens = lines[1].split("|") if len(tokens) < 2: - status = slurm_status_map["PENDING"] - return status + return slurm_status_map["PENDING"] slurm_status = tokens[1] - status = slurm_status_map[slurm_status] - return status + return slurm_status_map[slurm_status] diff --git a/src/lsst/cmservice/common/utils.py b/src/lsst/cmservice/common/utils.py index 20822efa..9a20adca 100644 --- a/src/lsst/cmservice/common/utils.py +++ b/src/lsst/cmservice/common/utils.py @@ -3,7 +3,7 @@ import contextlib import os import sys -from typing import Iterator +from collections.abc import Iterator @contextlib.contextmanager diff --git a/src/lsst/cmservice/db/campaign.py b/src/lsst/cmservice/db/campaign.py index 1435a739..f0676758 100644 --- a/src/lsst/cmservice/db/campaign.py +++ b/src/lsst/cmservice/db/campaign.py @@ -1,6 +1,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Iterable, List, Optional +from collections.abc import Iterable +from typing import TYPE_CHECKING, Any from sqlalchemy import JSON from sqlalchemy.ext.asyncio import async_scoped_session @@ -45,15 +46,15 @@ class Campaign(Base, ElementMixin): status: Mapped[StatusEnum] = mapped_column(default=StatusEnum.waiting) superseded: Mapped[bool] = mapped_column(default=False) handler: Mapped[str | None] = mapped_column() - data: Mapped[Optional[dict | list]] = mapped_column(type_=JSON) - child_config: Mapped[Optional[dict | list]] = mapped_column(type_=JSON) - collections: Mapped[Optional[dict | list]] = mapped_column(type_=JSON) - spec_aliases: Mapped[Optional[dict | list]] = mapped_column(type_=JSON) - - spec_block_: Mapped["SpecBlock"] = relationship("SpecBlock", viewonly=True) - parent_: Mapped["Production"] = relationship("Production", viewonly=True) - s_: Mapped[List["Step"]] = relationship("Step", viewonly=True) - scripts_: Mapped[List["Script"]] = relationship("Script", viewonly=True) + data: Mapped[dict | list | None] = mapped_column(type_=JSON) + child_config: Mapped[dict | list | None] = mapped_column(type_=JSON) + collections: Mapped[dict | list | None] = mapped_column(type_=JSON) + spec_aliases: Mapped[dict | list | None] = mapped_column(type_=JSON) + + spec_block_: Mapped[SpecBlock] = relationship("SpecBlock", viewonly=True) + parent_: Mapped[Production] = relationship("Production", viewonly=True) + s_: Mapped[list[Step]] = relationship("Step", viewonly=True) + scripts_: Mapped[list[Script]] = relationship("Script", viewonly=True) @hybrid_property def db_id(self) -> DbId: diff --git a/src/lsst/cmservice/db/element.py b/src/lsst/cmservice/db/element.py index 898d5e93..142f8c6c 100644 --- a/src/lsst/cmservice/db/element.py +++ b/src/lsst/cmservice/db/element.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, List +from typing import TYPE_CHECKING, Any from sqlalchemy.ext.asyncio import async_scoped_session @@ -31,9 +31,10 @@ async def get_scripts( self, session: async_scoped_session, script_name: str | None = None, + *, remaining_only: bool = False, skip_superseded: bool = True, - ) -> List["Script"]: + ) -> list[Script]: """Return the `Script`s associated to an element Parameters @@ -71,9 +72,10 @@ async def get_scripts( async def get_jobs( self, session: async_scoped_session, + *, remaining_only: bool = False, skip_superseded: bool = True, - ) -> List["Job"]: + ) -> list[Job]: """Return the `Job`s associated to an element Parameters @@ -128,12 +130,12 @@ async def retry_script( scripts = await self.get_scripts(session, script_name) if len(scripts) != 1: raise ValueError( - f"Expected one active script matching {script_name} for {self.fullname}, got {len(scripts)}" + f"Expected one active script matching {script_name} for {self.fullname}, got {len(scripts)}", ) the_script = scripts[0] if the_script.status.value > StatusEnum.rejected.value: raise ValueError( - f"Can only retry failed/rejected scripts, {the_script.fullname} is {the_script.status.value}" + f"Can only retry failed/rejected scripts, {the_script.fullname} is {the_script.status.value}", ) new_script = await the_script.copy_script(session) await the_script.update_values(session, superseded=True) @@ -175,7 +177,7 @@ async def rescue_job( async def mark_job_rescued( self, session: async_scoped_session, - ) -> List["Job"]: + ) -> list[Job]: """Mark jobs as `rescued` once one of their siblings is `accepted` Parameters diff --git a/src/lsst/cmservice/db/group.py b/src/lsst/cmservice/db/group.py index fabd2b93..7d8d1a5e 100644 --- a/src/lsst/cmservice/db/group.py +++ b/src/lsst/cmservice/db/group.py @@ -1,4 +1,5 @@ -from typing import TYPE_CHECKING, Any, Iterable, List, Optional +from collections.abc import Iterable +from typing import TYPE_CHECKING, Any from sqlalchemy import JSON from sqlalchemy.ext.asyncio import async_scoped_session @@ -41,10 +42,10 @@ class Group(Base, ElementMixin): status: Mapped[StatusEnum] = mapped_column(default=StatusEnum.waiting) # Status flag superseded: Mapped[bool] = mapped_column(default=False) # Has this been supersede handler: Mapped[str | None] = mapped_column() - data: Mapped[Optional[dict | list]] = mapped_column(type_=JSON) - child_config: Mapped[Optional[dict | list]] = mapped_column(type_=JSON) - collections: Mapped[Optional[dict | list]] = mapped_column(type_=JSON) - spec_aliases: Mapped[Optional[dict | list]] = mapped_column(type_=JSON) + data: Mapped[dict | list | None] = mapped_column(type_=JSON) + child_config: Mapped[dict | list | None] = mapped_column(type_=JSON) + collections: Mapped[dict | list | None] = mapped_column(type_=JSON) + spec_aliases: Mapped[dict | list | None] = mapped_column(type_=JSON) spec_block_: Mapped["SpecBlock"] = relationship("SpecBlock", viewonly=True) c_: Mapped["Campaign"] = relationship( @@ -63,8 +64,8 @@ class Group(Base, ElementMixin): ) parent_: Mapped["Step"] = relationship("Step", viewonly=True) - scripts_: Mapped[List["Script"]] = relationship("Script", viewonly=True) - jobs_: Mapped[List["Job"]] = relationship("Job", viewonly=True) + scripts_: Mapped[list["Script"]] = relationship("Script", viewonly=True) + jobs_: Mapped[list["Job"]] = relationship("Job", viewonly=True) @hybrid_property def db_id(self) -> DbId: diff --git a/src/lsst/cmservice/db/handler.py b/src/lsst/cmservice/db/handler.py index 41a1cd9f..b706ca25 100644 --- a/src/lsst/cmservice/db/handler.py +++ b/src/lsst/cmservice/db/handler.py @@ -65,7 +65,7 @@ def get_handler( with add_sys_path(Handler.plugin_dir): handler_class = doImport(class_name) if isinstance(handler_class, types.ModuleType): - raise TypeError() + raise TypeError cached_handler = handler_class(spec_block_id, **kwargs) Handler.handler_cache[spec_block_id] = cached_handler return cached_handler diff --git a/src/lsst/cmservice/db/job.py b/src/lsst/cmservice/db/job.py index 9881afe0..0d409fa1 100644 --- a/src/lsst/cmservice/db/job.py +++ b/src/lsst/cmservice/db/job.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, List, Optional +from typing import TYPE_CHECKING, Any from sqlalchemy import JSON from sqlalchemy.ext.asyncio import async_scoped_session @@ -47,29 +47,29 @@ class Job(Base, ElementMixin): status: Mapped[StatusEnum] = mapped_column(default=StatusEnum.waiting) superseded: Mapped[bool] = mapped_column(default=False) handler: Mapped[str | None] = mapped_column() - data: Mapped[Optional[dict | list]] = mapped_column(type_=JSON) - child_config: Mapped[Optional[dict | list]] = mapped_column(type_=JSON) - collections: Mapped[Optional[dict | list]] = mapped_column(type_=JSON) - spec_aliases: Mapped[Optional[dict | list]] = mapped_column(type_=JSON) - wms_job_id: Mapped[Optional[int]] = mapped_column() - stamp_url: Mapped[Optional[str]] = mapped_column() - - spec_block_: Mapped["SpecBlock"] = relationship("SpecBlock", viewonly=True) - s_: Mapped["Step"] = relationship( + data: Mapped[dict | list | None] = mapped_column(type_=JSON) + child_config: Mapped[dict | list | None] = mapped_column(type_=JSON) + collections: Mapped[dict | list | None] = mapped_column(type_=JSON) + spec_aliases: Mapped[dict | list | None] = mapped_column(type_=JSON) + wms_job_id: Mapped[int | None] = mapped_column() + stamp_url: Mapped[str | None] = mapped_column() + + spec_block_: Mapped[SpecBlock] = relationship("SpecBlock", viewonly=True) + s_: Mapped[Step] = relationship( "Step", primaryjoin="Job.parent_id==Group.id", secondary="join(Group, Step)", secondaryjoin="Group.parent_id==Step.id", viewonly=True, ) - c_: Mapped["Campaign"] = relationship( + c_: Mapped[Campaign] = relationship( "Campaign", primaryjoin="Job.parent_id==Group.id", secondary="join(Group, Step).join(Campaign)", secondaryjoin="and_(Group.parent_id==Step.id, Step.parent_id==Campaign.id) ", viewonly=True, ) - p_: Mapped["Production"] = relationship( + p_: Mapped[Production] = relationship( "Production", primaryjoin="Job.parent_id==Group.id", secondary="join(Group, Step).join(Campaign).join(Production)", @@ -80,18 +80,18 @@ class Job(Base, ElementMixin): ") ", viewonly=True, ) - parent_: Mapped["Group"] = relationship("Group", viewonly=True) - scripts_: Mapped[List["Script"]] = relationship("Script", viewonly=True) - tasks_: Mapped[List["TaskSet"]] = relationship("TaskSet", viewonly=True) - products_: Mapped[List["ProductSet"]] = relationship("ProductSet", viewonly=True) - errors_: Mapped[List["PipetaskError"]] = relationship( + parent_: Mapped[Group] = relationship("Group", viewonly=True) + scripts_: Mapped[list[Script]] = relationship("Script", viewonly=True) + tasks_: Mapped[list[TaskSet]] = relationship("TaskSet", viewonly=True) + products_: Mapped[list[ProductSet]] = relationship("ProductSet", viewonly=True) + errors_: Mapped[list[PipetaskError]] = relationship( "PipetaskError", primaryjoin="Job.id==TaskSet.job_id", secondary="join(TaskSet, PipetaskError)", secondaryjoin="PipetaskError.task_id==TaskSet.id", viewonly=True, ) - wms_reports_: Mapped[List["WmsTaskReport"]] = relationship("WmsTaskReport", viewonly=True) + wms_reports_: Mapped[list[WmsTaskReport]] = relationship("WmsTaskReport", viewonly=True) @hybrid_property def db_id(self) -> DbId: @@ -114,7 +114,7 @@ async def get_create_kwargs( spec_block = await SpecBlock.get_row_by_fullname(session, spec_block_name) parent = await Group.get_row_by_fullname(session, parent_name) - ret_dict = { + return { "spec_block_id": spec_block.id, "parent_id": parent.id, "name": name, @@ -126,8 +126,6 @@ async def get_create_kwargs( "spec_aliases": kwargs.get("spec_aliases", {}), } - return ret_dict - async def copy_job( self, session: async_scoped_session, @@ -148,4 +146,4 @@ async def copy_job( new_job: Job Newly created Job """ - raise NotImplementedError() + raise NotImplementedError diff --git a/src/lsst/cmservice/db/node.py b/src/lsst/cmservice/db/node.py index 0c3498d2..3dbfc7fe 100644 --- a/src/lsst/cmservice/db/node.py +++ b/src/lsst/cmservice/db/node.py @@ -124,11 +124,10 @@ async def get_handler( else: spec_block = await self.get_spec_block(session) handler_class = spec_block.handler - handler = Handler.get_handler( + return Handler.get_handler( self.spec_block_id, handler_class, ) - return handler def _split_fullname(self, fullname: str) -> dict: """Split a fullname into named fields @@ -203,7 +202,7 @@ async def resolve_collections( resolved_collections[name_].append(f1.format(**name_dict)) except KeyError as msg: raise KeyError( - f"Failed to resolve collection {name_} {f1} using: {str(name_dict)}", + f"Failed to resolve collection {name_} {f1} using: {name_dict!s}", ) from msg else: try: @@ -214,7 +213,7 @@ async def resolve_collections( resolved_collections[name_] = f1.format(**name_dict) except KeyError as msg: raise KeyError( - f"Failed to resolve collection {name_}, {f1} using: {str(name_dict)}", + f"Failed to resolve collection {name_}, {f1} using: {name_dict!s}", ) from msg return resolved_collections @@ -357,7 +356,7 @@ async def get_spec_aliases( ret_dict = {} async with session.begin_nested(): if self.level == LevelEnum.script: - raise NotImplementedError() + raise NotImplementedError if self.level.value > LevelEnum.campaign.value: await session.refresh(self, attribute_names=["parent_"]) parent_data = await self.parent_.get_spec_aliases(session) @@ -632,7 +631,7 @@ async def _clean_up_node( node: NodeMixin Node being cleaned """ - raise NotImplementedError() + raise NotImplementedError async def process( self, @@ -655,8 +654,7 @@ async def process( The status of the processing """ handler = await self.get_handler(session) - status = await handler.process(session, self, **kwargs) - return status + return await handler.process(session, self, **kwargs) async def run_check( self, @@ -679,5 +677,4 @@ async def run_check( The status of the processing """ handler = await self.get_handler(session) - status = await handler.run_check(session, self, **kwargs) - return status + return await handler.run_check(session, self, **kwargs) diff --git a/src/lsst/cmservice/db/pipetask_error.py b/src/lsst/cmservice/db/pipetask_error.py index 5198ce1c..688ad522 100644 --- a/src/lsst/cmservice/db/pipetask_error.py +++ b/src/lsst/cmservice/db/pipetask_error.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from sqlalchemy import JSON from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -26,7 +26,7 @@ class PipetaskError(Base, RowMixin): task_id: Mapped[int] = mapped_column(ForeignKey("task_set.id", ondelete="CASCADE"), index=True) quanta: Mapped[str] = mapped_column() diagnostic_message: Mapped[str] = mapped_column() - data_id: Mapped[Optional[dict | list]] = mapped_column(type_=JSON) + data_id: Mapped[dict | list | None] = mapped_column(type_=JSON) job_: Mapped["Job"] = relationship( "Job", diff --git a/src/lsst/cmservice/db/pipetask_error_type.py b/src/lsst/cmservice/db/pipetask_error_type.py index f99f7bf7..63474632 100644 --- a/src/lsst/cmservice/db/pipetask_error_type.py +++ b/src/lsst/cmservice/db/pipetask_error_type.py @@ -1,5 +1,5 @@ import re -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -23,7 +23,7 @@ class PipetaskErrorType(Base, RowMixin): task_name: Mapped[str] = mapped_column() diagnostic_message: Mapped[str] = mapped_column(unique=True) - errors_: Mapped[List["PipetaskError"]] = relationship("PipetaskError", viewonly=True) + errors_: Mapped[list["PipetaskError"]] = relationship("PipetaskError", viewonly=True) def __repr__(self) -> str: s = f"Id={self.id}\n" diff --git a/src/lsst/cmservice/db/production.py b/src/lsst/cmservice/db/production.py index 2ba3de4f..c174ea6f 100644 --- a/src/lsst/cmservice/db/production.py +++ b/src/lsst/cmservice/db/production.py @@ -1,4 +1,5 @@ -from typing import TYPE_CHECKING, Iterable, List +from collections.abc import Iterable +from typing import TYPE_CHECKING from sqlalchemy.ext.asyncio import async_scoped_session from sqlalchemy.ext.hybrid import hybrid_property @@ -21,7 +22,7 @@ class Production(Base, RowMixin): id: Mapped[int] = mapped_column(primary_key=True) name: Mapped[str] = mapped_column(index=True, unique=True) - c_: Mapped[List["Campaign"]] = relationship("Campaign", viewonly=True) + c_: Mapped[list["Campaign"]] = relationship("Campaign", viewonly=True) @hybrid_property def db_id(self) -> DbId: diff --git a/src/lsst/cmservice/db/queue.py b/src/lsst/cmservice/db/queue.py index b5a4806d..b723560f 100644 --- a/src/lsst/cmservice/db/queue.py +++ b/src/lsst/cmservice/db/queue.py @@ -1,7 +1,7 @@ from __future__ import annotations from datetime import datetime, timedelta -from typing import TYPE_CHECKING, Any, Optional +from typing import Any import pause from sqlalchemy import JSON, DateTime @@ -20,9 +20,6 @@ from .node import NodeMixin from .step import Step -if TYPE_CHECKING: - pass - class Queue(Base, NodeMixin): """Database table to implement processing queue""" @@ -34,7 +31,7 @@ class Queue(Base, NodeMixin): time_updated: Mapped[datetime] = mapped_column(type_=DateTime) time_finished: Mapped[datetime | None] = mapped_column(type_=DateTime, default=None) interval: Mapped[float] = mapped_column(default=300.0) - options: Mapped[Optional[dict | list]] = mapped_column(type_=JSON) + options: Mapped[dict | list | None] = mapped_column(type_=JSON) element_level: Mapped[LevelEnum] = mapped_column() element_id: Mapped[int] = mapped_column() @@ -43,10 +40,10 @@ class Queue(Base, NodeMixin): g_id: Mapped[int | None] = mapped_column(ForeignKey("group.id", ondelete="CASCADE"), index=True) j_id: Mapped[int | None] = mapped_column(ForeignKey("job.id", ondelete="CASCADE"), index=True) - c_: Mapped["Campaign"] = relationship("Campaign", viewonly=True) - s_: Mapped["Step"] = relationship("Step", viewonly=True) - g_: Mapped["Group"] = relationship("Group", viewonly=True) - j_: Mapped["Job"] = relationship("Job", viewonly=True) + c_: Mapped[Campaign] = relationship("Campaign", viewonly=True) + s_: Mapped[Step] = relationship("Step", viewonly=True) + g_: Mapped[Group] = relationship("Group", viewonly=True) + j_: Mapped[Job] = relationship("Job", viewonly=True) @hybrid_property def element_db_id(self) -> DbId: diff --git a/src/lsst/cmservice/db/row.py b/src/lsst/cmservice/db/row.py index 378b3044..a1301eb6 100644 --- a/src/lsst/cmservice/db/row.py +++ b/src/lsst/cmservice/db/row.py @@ -1,6 +1,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Sequence, TypeVar +from collections.abc import Sequence +from typing import TYPE_CHECKING, Any, TypeVar from fastapi import HTTPException from sqlalchemy import select diff --git a/src/lsst/cmservice/db/script.py b/src/lsst/cmservice/db/script.py index 1a03e08e..a025dbbf 100644 --- a/src/lsst/cmservice/db/script.py +++ b/src/lsst/cmservice/db/script.py @@ -1,6 +1,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, List, Optional, Sequence +from collections.abc import Sequence +from typing import TYPE_CHECKING, Any from sqlalchemy import JSON, and_, select from sqlalchemy.ext.asyncio import async_scoped_session @@ -50,20 +51,20 @@ class Script(Base, NodeMixin): method: Mapped[ScriptMethod] = mapped_column(default=ScriptMethod.default) superseded: Mapped[bool] = mapped_column(default=False) # Has this been supersede handler: Mapped[str | None] = mapped_column() - data: Mapped[Optional[dict | list]] = mapped_column(type_=JSON) - child_config: Mapped[Optional[dict | list]] = mapped_column(type_=JSON) - collections: Mapped[Optional[dict | list]] = mapped_column(type_=JSON) + data: Mapped[dict | list | None] = mapped_column(type_=JSON) + child_config: Mapped[dict | list | None] = mapped_column(type_=JSON) + collections: Mapped[dict | list | None] = mapped_column(type_=JSON) script_url: Mapped[str | None] = mapped_column() stamp_url: Mapped[str | None] = mapped_column() log_url: Mapped[str | None] = mapped_column() - spec_block_: Mapped["SpecBlock"] = relationship("SpecBlock", viewonly=True) - c_: Mapped["Campaign"] = relationship("Campaign", viewonly=True) - s_: Mapped["Step"] = relationship("Step", viewonly=True) - g_: Mapped["Group"] = relationship("Group", viewonly=True) - j_: Mapped["Job"] = relationship("Job", viewonly=True) - errors_: Mapped[List["ScriptError"]] = relationship("ScriptError", viewonly=True) - prereqs_: Mapped[List["ScriptDependency"]] = relationship( + spec_block_: Mapped[SpecBlock] = relationship("SpecBlock", viewonly=True) + c_: Mapped[Campaign] = relationship("Campaign", viewonly=True) + s_: Mapped[Step] = relationship("Step", viewonly=True) + g_: Mapped[Group] = relationship("Group", viewonly=True) + j_: Mapped[Job] = relationship("Job", viewonly=True) + errors_: Mapped[list[ScriptError]] = relationship("ScriptError", viewonly=True) + prereqs_: Mapped[list[ScriptDependency]] = relationship( "ScriptDependency", foreign_keys="ScriptDependency.depend_id", viewonly=True, @@ -144,7 +145,7 @@ async def get_siblings( Script.parent_level == self.parent_level, Script.name == self.name, Script.id != self.id, - ) + ), ) async with session.begin_nested(): rows = await session.scalars(q) diff --git a/src/lsst/cmservice/db/script_dependency.py b/src/lsst/cmservice/db/script_dependency.py index a49bb322..19ded5f9 100644 --- a/src/lsst/cmservice/db/script_dependency.py +++ b/src/lsst/cmservice/db/script_dependency.py @@ -27,8 +27,8 @@ class ScriptDependency(Base, RowMixin): prereq_id: Mapped[int] = mapped_column(ForeignKey("script.id", ondelete="CASCADE"), index=True) depend_id: Mapped[int] = mapped_column(ForeignKey("script.id", ondelete="CASCADE"), index=True) - prereq_: Mapped["Script"] = relationship("Script", viewonly=True, foreign_keys=[prereq_id]) - depend_: Mapped["Script"] = relationship("Script", back_populates="prereqs_", foreign_keys=[depend_id]) + prereq_: Mapped[Script] = relationship("Script", viewonly=True, foreign_keys=[prereq_id]) + depend_: Mapped[Script] = relationship("Script", back_populates="prereqs_", foreign_keys=[depend_id]) def __repr__(self) -> str: return f"ScriptDependency {self.prereq_id}: {self.depend_id}" diff --git a/src/lsst/cmservice/db/script_template.py b/src/lsst/cmservice/db/script_template.py index c16d70cc..92804e38 100644 --- a/src/lsst/cmservice/db/script_template.py +++ b/src/lsst/cmservice/db/script_template.py @@ -1,7 +1,7 @@ from __future__ import annotations import os -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any import yaml from sqlalchemy import JSON @@ -28,9 +28,9 @@ class ScriptTemplate(Base, RowMixin): spec_id: Mapped[int] = mapped_column(ForeignKey("specification.id", ondelete="CASCADE"), index=True) name: Mapped[str] = mapped_column(index=True) fullname: Mapped[str] = mapped_column(unique=True) - data: Mapped[Optional[dict | list]] = mapped_column(type_=JSON) + data: Mapped[dict | list | None] = mapped_column(type_=JSON) - spec_: Mapped["Specification"] = relationship("Specification", viewonly=True) + spec_: Mapped[Specification] = relationship("Specification", viewonly=True) def __repr__(self) -> str: return f"ScriptTemplate {self.id}: {self.fullname} {self.data}" @@ -45,13 +45,12 @@ async def get_create_kwargs( spec_name = kwargs["spec_name"] name = kwargs["name"] - ret_dict = { + return { "spec_id": spec_id, "name": name, "fullname": f"{spec_name}#{name}", "data": kwargs.get("data", None), } - return ret_dict @classmethod async def load( # pylint: disable=too-many-arguments @@ -84,8 +83,7 @@ async def load( # pylint: disable=too-many-arguments Newly created `ScriptTemplate` """ full_file_path = os.path.abspath(os.path.expandvars(file_path)) - with open(full_file_path, "r", encoding="utf-8") as fin: + with open(full_file_path, encoding="utf-8") as fin: data = yaml.safe_load(fin) - new_row = await cls.create_row(session, name=name, spec_id=spec_id, spec_name=spec_name, data=data) - return new_row + return await cls.create_row(session, name=name, spec_id=spec_id, spec_name=spec_name, data=data) diff --git a/src/lsst/cmservice/db/specification.py b/src/lsst/cmservice/db/specification.py index a66a2ea4..df3c830f 100644 --- a/src/lsst/cmservice/db/specification.py +++ b/src/lsst/cmservice/db/specification.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, List, Optional +from typing import Any from sqlalchemy import JSON from sqlalchemy.ext.asyncio import async_scoped_session @@ -12,9 +12,6 @@ from .row import RowMixin from .script_template import ScriptTemplate -if TYPE_CHECKING: - pass - class SpecBlock(Base, RowMixin): """Database table to manage blocks that are used to build campaigns @@ -30,13 +27,13 @@ class SpecBlock(Base, RowMixin): name: Mapped[str] = mapped_column(index=True) fullname: Mapped[str] = mapped_column(unique=True) handler: Mapped[str | None] = mapped_column() - data: Mapped[Optional[dict | list]] = mapped_column(type_=JSON) - collections: Mapped[Optional[dict | list]] = mapped_column(type_=JSON) - child_config: Mapped[Optional[dict | list]] = mapped_column(type_=JSON) - scripts: Mapped[Optional[dict | list]] = mapped_column(type_=JSON) - spec_aliases: Mapped[Optional[dict | list]] = mapped_column(type_=JSON) + data: Mapped[dict | list | None] = mapped_column(type_=JSON) + collections: Mapped[dict | list | None] = mapped_column(type_=JSON) + child_config: Mapped[dict | list | None] = mapped_column(type_=JSON) + scripts: Mapped[dict | list | None] = mapped_column(type_=JSON) + spec_aliases: Mapped[dict | list | None] = mapped_column(type_=JSON) - spec_: Mapped["Specification"] = relationship("Specification", viewonly=True) + spec_: Mapped[Specification] = relationship("Specification", viewonly=True) def __repr__(self) -> str: return f"SpecBlock {self.id}: {self.fullname} {self.data}" @@ -51,7 +48,7 @@ async def get_create_kwargs( spec = await Specification.get_row_by_fullname(session, spec_name) handler = kwargs["handler"] name = kwargs["name"] - ret_dict = { + return { "spec_id": spec.id, "name": name, "handler": handler, @@ -62,7 +59,6 @@ async def get_create_kwargs( "scripts": kwargs.get("scripts", {}), "spec_aliases": kwargs.get("spec_aliases", {}), } - return ret_dict class Specification(Base, RowMixin): @@ -71,8 +67,8 @@ class Specification(Base, RowMixin): id: Mapped[int] = mapped_column(primary_key=True) name: Mapped[str] = mapped_column(index=True) - blocks_: Mapped[List["SpecBlock"]] = relationship("SpecBlock", viewonly=True) - script_templates_: Mapped[List["ScriptTemplate"]] = relationship("ScriptTemplate", viewonly=True) + blocks_: Mapped[list[SpecBlock]] = relationship("SpecBlock", viewonly=True) + script_templates_: Mapped[list[ScriptTemplate]] = relationship("ScriptTemplate", viewonly=True) @hybrid_property def fullname(self) -> str: diff --git a/src/lsst/cmservice/db/step.py b/src/lsst/cmservice/db/step.py index 8cc42aff..bb4a24e7 100644 --- a/src/lsst/cmservice/db/step.py +++ b/src/lsst/cmservice/db/step.py @@ -1,6 +1,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Iterable, List, Optional +from collections.abc import Iterable +from typing import TYPE_CHECKING, Any from sqlalchemy import JSON from sqlalchemy.ext.asyncio import async_scoped_session @@ -45,28 +46,28 @@ class Step(Base, ElementMixin): status: Mapped[StatusEnum] = mapped_column(default=StatusEnum.waiting) # Status flag superseded: Mapped[bool] = mapped_column(default=False) # Has this been supersede handler: Mapped[str | None] = mapped_column() - data: Mapped[Optional[dict | list]] = mapped_column(type_=JSON) - child_config: Mapped[Optional[dict | list]] = mapped_column(type_=JSON) - collections: Mapped[Optional[dict | list]] = mapped_column(type_=JSON) - spec_aliases: Mapped[Optional[dict | list]] = mapped_column(type_=JSON) - - spec_block_: Mapped["SpecBlock"] = relationship("SpecBlock", viewonly=True) - parent_: Mapped["Campaign"] = relationship("Campaign", back_populates="s_") - p_: Mapped["Production"] = relationship( + data: Mapped[dict | list | None] = mapped_column(type_=JSON) + child_config: Mapped[dict | list | None] = mapped_column(type_=JSON) + collections: Mapped[dict | list | None] = mapped_column(type_=JSON) + spec_aliases: Mapped[dict | list | None] = mapped_column(type_=JSON) + + spec_block_: Mapped[SpecBlock] = relationship("SpecBlock", viewonly=True) + parent_: Mapped[Campaign] = relationship("Campaign", back_populates="s_") + p_: Mapped[Production] = relationship( "Production", primaryjoin="Step.parent_id==Campaign.id", secondary="join(Campaign, Production)", secondaryjoin="Campaign.parent_id==Production.id", viewonly=True, ) - g_: Mapped[List["Group"]] = relationship("Group", viewonly=True) - scripts_: Mapped[List["Script"]] = relationship("Script", viewonly=True) - prereqs_: Mapped[List["StepDependency"]] = relationship( + g_: Mapped[list[Group]] = relationship("Group", viewonly=True) + scripts_: Mapped[list[Script]] = relationship("Script", viewonly=True) + prereqs_: Mapped[list[StepDependency]] = relationship( "StepDependency", foreign_keys="StepDependency.depend_id", viewonly=True, ) - jobs_: Mapped[List["Job"]] = relationship( + jobs_: Mapped[list[Job]] = relationship( "Job", primaryjoin="Group.parent_id==Step.id", secondary="join(Group, Job)", diff --git a/src/lsst/cmservice/db/step_dependency.py b/src/lsst/cmservice/db/step_dependency.py index 1cb26475..3eac3833 100644 --- a/src/lsst/cmservice/db/step_dependency.py +++ b/src/lsst/cmservice/db/step_dependency.py @@ -29,8 +29,8 @@ class StepDependency(Base, RowMixin): prereq_id: Mapped[int] = mapped_column(ForeignKey("step.id", ondelete="CASCADE"), index=True) depend_id: Mapped[int] = mapped_column(ForeignKey("step.id", ondelete="CASCADE"), index=True) - prereq_: Mapped["Step"] = relationship("Step", viewonly=True, foreign_keys=[prereq_id]) - depend_: Mapped["Step"] = relationship("Step", viewonly=True, foreign_keys=[depend_id]) + prereq_: Mapped[Step] = relationship("Step", viewonly=True, foreign_keys=[prereq_id]) + depend_: Mapped[Step] = relationship("Step", viewonly=True, foreign_keys=[depend_id]) @hybrid_property def prereq_db_id(self) -> DbId: diff --git a/src/lsst/cmservice/db/task_set.py b/src/lsst/cmservice/db/task_set.py index c3aa56e4..1dd3b5df 100644 --- a/src/lsst/cmservice/db/task_set.py +++ b/src/lsst/cmservice/db/task_set.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING from sqlalchemy.orm import Mapped, mapped_column, relationship from sqlalchemy.schema import ForeignKey @@ -27,4 +27,4 @@ class TaskSet(Base, RowMixin): n_failed_upstream: Mapped[int] = mapped_column(default=0) job_: Mapped["Job"] = relationship("Job", viewonly=True) - products_: Mapped[List["ProductSet"]] = relationship("ProductSet", viewonly=True) + products_: Mapped[list["ProductSet"]] = relationship("ProductSet", viewonly=True) diff --git a/src/lsst/cmservice/handlers/element_handler.py b/src/lsst/cmservice/handlers/element_handler.py index e6d1f8bf..bbd28569 100644 --- a/src/lsst/cmservice/handlers/element_handler.py +++ b/src/lsst/cmservice/handlers/element_handler.py @@ -102,12 +102,10 @@ async def run_check( node: NodeMixin, **kwargs: Any, ) -> StatusEnum: - status = node.status # Need this so mypy doesn't think we are passing in Script if TYPE_CHECKING: assert isinstance(node, ElementMixin) - status = await self.check(session, node, **kwargs) - return status + return await self.check(session, node, **kwargs) async def prepare( self, diff --git a/src/lsst/cmservice/handlers/elements.py b/src/lsst/cmservice/handlers/elements.py index 6e81ab3d..3f0838f2 100644 --- a/src/lsst/cmservice/handlers/elements.py +++ b/src/lsst/cmservice/handlers/elements.py @@ -1,6 +1,7 @@ from __future__ import annotations -from typing import Any, AsyncGenerator +from collections.abc import AsyncGenerator +from typing import Any import numpy as np from sqlalchemy.ext.asyncio import async_scoped_session @@ -21,7 +22,7 @@ def parse_bps_stdout(url: str) -> dict[str, str]: """Parse the std from a bps submit job""" out_dict = {} - with open(url, "r", encoding="utf8") as fin: + with open(url, encoding="utf8") as fin: line = fin.readline() while line: tokens = line.split(":") diff --git a/src/lsst/cmservice/handlers/functions.py b/src/lsst/cmservice/handlers/functions.py index 1d9aced3..36287148 100644 --- a/src/lsst/cmservice/handlers/functions.py +++ b/src/lsst/cmservice/handlers/functions.py @@ -55,7 +55,7 @@ async def load_spec_block( block_data[include_key] = include_val handler = block_data.pop("handler", None) - new_spec_block = await SpecBlock.create_row( + return await SpecBlock.create_row( session, spec_name=specification.name, name=key, @@ -65,7 +65,6 @@ async def load_spec_block( child_config=block_data.get("child_config"), scripts=block_data.get("scripts"), ) - return new_spec_block async def load_script_template( @@ -81,14 +80,13 @@ async def load_script_template( if script_template: print(f"ScriptTemplate {key} already defined, skipping it") return None - new_script_template = await ScriptTemplate.load( + return await ScriptTemplate.load( session, spec_name=specification.name, spec_id=specification.id, name=key, file_path=config_values["file_path"], ) - return new_script_template async def load_specification( @@ -96,7 +94,7 @@ async def load_specification( spec_name: str, yaml_file: str, ) -> Specification: - with open(yaml_file, "rt", encoding="utf-8") as fin: + with open(yaml_file, encoding="utf-8") as fin: spec_data = yaml.safe_load(fin) loaded_specs: dict = {} @@ -133,12 +131,11 @@ async def add_step_prerequisite( script_id: int, prereq_id: int, ) -> StepDependency: - new_depend = await StepDependency.create_row( + return await StepDependency.create_row( session, prereq_id=prereq_id, depend_id=script_id, ) - return new_depend async def add_steps( @@ -157,7 +154,7 @@ async def add_steps( spec_block_name = child_config_.pop("spec_block") if spec_block_name is None: raise AttributeError( - f"child_config_ {child_name_} of {campaign.fullname} does contain 'spec_block'" + f"child_config_ {child_name_} of {campaign.fullname} does contain 'spec_block'", ) spec_block_name = spec_aliases.get(spec_block_name, spec_block_name) spec_block = await specification.get_block(session, spec_block_name) @@ -233,7 +230,7 @@ async def load_manifest_report( job_name: str, yaml_file: str, ) -> Job: - with open(yaml_file, "rt", encoding="utf-8") as fin: + with open(yaml_file, encoding="utf-8") as fin: manifest_data = yaml.safe_load(fin) job = await Job.get_row_by_fullname(session, job_name) @@ -319,7 +316,7 @@ async def load_error_types( session: async_scoped_session, yaml_file: str, ) -> list[PipetaskErrorType]: - with open(yaml_file, "rt", encoding="utf-8") as fin: + with open(yaml_file, encoding="utf-8") as fin: error_types = yaml.safe_load(fin) ret_list: list[PipetaskErrorType] = [] diff --git a/src/lsst/cmservice/handlers/interface.py b/src/lsst/cmservice/handlers/interface.py index c37687bd..1e0ec060 100644 --- a/src/lsst/cmservice/handlers/interface.py +++ b/src/lsst/cmservice/handlers/interface.py @@ -1,5 +1,5 @@ # pylint: disable=too-many-lines -from typing import TYPE_CHECKING, Any, Dict, List +from typing import Any from fastapi import HTTPException from sqlalchemy import select @@ -9,11 +9,7 @@ from ..common.enums import LevelEnum, NodeTypeEnum, StatusEnum, TableEnum from . import functions -if TYPE_CHECKING: - pass - - -TABLE_DICT: Dict[TableEnum, type[db.RowMixin]] = { +TABLE_DICT: dict[TableEnum, type[db.RowMixin]] = { TableEnum.production: db.Production, TableEnum.campaign: db.Campaign, TableEnum.step: db.Step, @@ -32,7 +28,7 @@ } -LEVEL_DICT: Dict[LevelEnum, type[db.NodeMixin]] = { +LEVEL_DICT: dict[LevelEnum, type[db.NodeMixin]] = { LevelEnum.campaign: db.Campaign, LevelEnum.step: db.Step, LevelEnum.group: db.Group, @@ -56,8 +52,7 @@ def get_table( table_class : type[db.RowMixin] The class that defines the table """ - table_class = TABLE_DICT[table_enum] - return table_class + return TABLE_DICT[table_enum] async def get_row_by_table_and_id( @@ -659,9 +654,10 @@ async def get_scripts( session: async_scoped_session, fullname: str, script_name: str, + *, remaining_only: bool = False, skip_superseded: bool = True, -) -> List[db.Script]: +) -> list[db.Script]: """Get the scripts associated to an `Element` Parameters @@ -693,15 +689,21 @@ async def get_scripts( HTTPException : Code 404, Could not find Element """ element = await get_element_by_fullname(session, fullname) - return await element.get_scripts(session, script_name, remaining_only, skip_superseded) + return await element.get_scripts( + session, + script_name, + remaining_only=remaining_only, + skip_superseded=skip_superseded, + ) async def get_jobs( session: async_scoped_session, fullname: str, + *, remaining_only: bool = False, skip_superseded: bool = True, -) -> List[db.Job]: +) -> list[db.Job]: """Get the jobs associated to an `Element` Parameters @@ -730,7 +732,7 @@ async def get_jobs( HTTPException : Code 404, Could not find Element """ element = await get_element_by_fullname(session, fullname) - return await element.get_jobs(session, remaining_only, skip_superseded) + return await element.get_jobs(session, remaining_only=remaining_only, skip_superseded=skip_superseded) async def process_script( @@ -956,7 +958,7 @@ async def rescue_job( async def mark_job_rescued( session: async_scoped_session, fullname: str, -) -> List[db.Job]: +) -> list[db.Job]: """Mark a `Job` as rescued Notes @@ -995,7 +997,7 @@ async def mark_job_rescued( async def get_task_sets_for_job( session: async_scoped_session, fullname: str, -) -> List[db.TaskSet]: +) -> list[db.TaskSet]: """Get `TaskSet`s associated to a `Job` Parameters @@ -1020,7 +1022,7 @@ async def get_task_sets_for_job( async def get_wms_reports_for_job( session: async_scoped_session, fullname: str, -) -> List[db.WmsTaskReport]: +) -> list[db.WmsTaskReport]: """Get `WmsTaskReport`s associated to a `Job` Parameters @@ -1045,7 +1047,7 @@ async def get_wms_reports_for_job( async def get_product_sets_for_job( session: async_scoped_session, fullname: str, -) -> List[db.ProductSet]: +) -> list[db.ProductSet]: """Get `ProductSet`s associated to a `Job` Parameters @@ -1070,7 +1072,7 @@ async def get_product_sets_for_job( async def get_errors_for_job( session: async_scoped_session, fullname: str, -) -> List[db.PipetaskError]: +) -> list[db.PipetaskError]: """Get `PipetaskError`s associated to a `Job` Parameters @@ -1279,7 +1281,7 @@ async def load_and_create_campaign( # pylint: disable=too-many-arguments async def load_error_types( session: async_scoped_session, yaml_file: str, -) -> List[db.PipetaskErrorType]: +) -> list[db.PipetaskErrorType]: """Load a set of `PipetaskErrorType`s from a yaml file Parameters @@ -1330,8 +1332,9 @@ async def load_manifest_report( async def match_pipetask_errors( # pylint: disable=unused-argument session: async_scoped_session, + *, rematch: bool = False, -) -> List[db.PipetaskError]: +) -> list[db.PipetaskError]: """Match PipetaskErrors to PipetaskErrorTypes Parameters diff --git a/src/lsst/cmservice/handlers/jobs.py b/src/lsst/cmservice/handlers/jobs.py index 290e4fc1..b5c7e9ec 100644 --- a/src/lsst/cmservice/handlers/jobs.py +++ b/src/lsst/cmservice/handlers/jobs.py @@ -51,7 +51,7 @@ def parse_bps_stdout(url: str) -> dict[str, str]: """Parse the std from a bps submit job""" out_dict = {} - with open(url, "r", encoding="utf8") as fin: + with open(url, encoding="utf8") as fin: line = fin.readline() while line: tokens = line.split(":") @@ -115,7 +115,7 @@ async def _write_script( data_query = data_dict.get("data_query", None) workflow_config["submitPath"] = os.path.abspath( - os.path.expandvars(f"{prod_area}/{parent.fullname}/submit") + os.path.expandvars(f"{prod_area}/{parent.fullname}/submit"), ) workflow_config["LSST_VERSION"] = os.path.expandvars(data_dict["lsst_version"]) @@ -140,7 +140,7 @@ async def _write_script( except OSError: pass - with open(config_url, "wt", encoding="utf-8") as fout: + with open(config_url, "w", encoding="utf-8") as fout: yaml.dump(workflow_config, fout) return StatusEnum.prepared @@ -214,8 +214,7 @@ def _get_wms_report( Report for requested job """ wms_svc = self._get_wms_svc() - wms_run_report = wms_svc.report(wms_workflow_id=wms_workflow_id)[0][0] - return wms_run_report + return wms_svc.report(wms_workflow_id=wms_workflow_id)[0][0] async def _load_wms_reports( self, diff --git a/src/lsst/cmservice/handlers/script_handler.py b/src/lsst/cmservice/handlers/script_handler.py index d688aa81..2bd1b967 100644 --- a/src/lsst/cmservice/handlers/script_handler.py +++ b/src/lsst/cmservice/handlers/script_handler.py @@ -80,10 +80,8 @@ async def run_check( # Need this so mypy doesn't think we are passing in Element if TYPE_CHECKING: assert isinstance(node, Script) - status = node.status parent = await node.get_parent(session) - status = await self.check(session, node, parent, **kwargs) - return status + return await self.check(session, node, parent, **kwargs) async def prepare( self, diff --git a/src/lsst/cmservice/handlers/scripts.py b/src/lsst/cmservice/handlers/scripts.py index 255cb817..6f60d160 100644 --- a/src/lsst/cmservice/handlers/scripts.py +++ b/src/lsst/cmservice/handlers/scripts.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import Any from sqlalchemy.ext.asyncio import async_scoped_session @@ -12,9 +12,6 @@ from ..db.step import Step from .script_handler import ScriptHandler -if TYPE_CHECKING: - pass - class ChainCreateScriptHandler(ScriptHandler): """Write a script to chain together collections diff --git a/src/lsst/cmservice/models/script_template.py b/src/lsst/cmservice/models/script_template.py index 703f03ac..c0debd19 100644 --- a/src/lsst/cmservice/models/script_template.py +++ b/src/lsst/cmservice/models/script_template.py @@ -1,12 +1,10 @@ -from typing import Optional - from pydantic import BaseModel class ScriptTemplateBase(BaseModel): spec_id: int name: str - data: Optional[dict | list] + data: dict | list | None class ScriptTemplateCreate(ScriptTemplateBase): diff --git a/src/lsst/cmservice/models/specification.py b/src/lsst/cmservice/models/specification.py index 8aff40e6..1478480c 100644 --- a/src/lsst/cmservice/models/specification.py +++ b/src/lsst/cmservice/models/specification.py @@ -1,5 +1,3 @@ -from typing import Optional - from pydantic import BaseModel @@ -7,11 +5,11 @@ class SpecBlockBase(BaseModel): spec_id: int name: str handler: str | None = None - data: Optional[dict | list] - collections: Optional[dict | list] - child_config: Optional[dict | list] - spec_aliases: Optional[dict | list] - scripts: Optional[dict | list] + data: dict | list | None + collections: dict | list | None + child_config: dict | list | None + spec_aliases: dict | list | None + scripts: dict | list | None class SpecBlockCreate(SpecBlockBase): diff --git a/src/lsst/cmservice/routers/actions.py b/src/lsst/cmservice/routers/actions.py index c8e0bfa9..af3b8234 100644 --- a/src/lsst/cmservice/routers/actions.py +++ b/src/lsst/cmservice/routers/actions.py @@ -1,5 +1,3 @@ -from typing import List - from fastapi import APIRouter, Depends from safir.dependencies.db_session import db_session_dependency from sqlalchemy.ext.asyncio import async_scoped_session @@ -27,8 +25,7 @@ async def process_script( params = query.dict() if params.get("fake_status"): params["fake_status"] = StatusEnum(params["fake_status"]) - result = await interface.process_script(session, **params) - return result + return await interface.process_script(session, **params) @router.post( @@ -44,8 +41,7 @@ async def process_job( params = query.dict() if params.get("fake_status"): params["fake_status"] = StatusEnum(params["fake_status"]) - result = await interface.process_job(session, **params) - return result + return await interface.process_job(session, **params) @router.post( @@ -61,8 +57,7 @@ async def process_element( params = query.dict() if params.get("fake_status"): params["fake_status"] = StatusEnum(params["fake_status"]) - result = await interface.process_element(session, **params) - return result + return await interface.process_element(session, **params) @router.post( @@ -78,8 +73,7 @@ async def process( params = query.dict() if params.get("fake_status"): params["fake_status"] = StatusEnum(params["fake_status"]) - result = await interface.process(session, **params) - return result + return await interface.process(session, **params) @router.post( @@ -93,8 +87,7 @@ async def retry_script( session: async_scoped_session = Depends(db_session_dependency), ) -> db.Script: params = query.dict() - result = await interface.retry_script(session, **params) - return result + return await interface.retry_script(session, **params) @router.post( @@ -108,8 +101,7 @@ async def rescue_job( session: async_scoped_session = Depends(db_session_dependency), ) -> db.Job: params = query.dict() - result = await interface.rescue_job(session, **params) - return result + return await interface.rescue_job(session, **params) @router.post( @@ -121,10 +113,9 @@ async def rescue_job( async def mark_job_rescued( query: models.NodeQuery, session: async_scoped_session = Depends(db_session_dependency), -) -> List[db.Job]: +) -> list[db.Job]: params = query.dict() - result = await interface.mark_job_rescued(session, **params) - return result + return await interface.mark_job_rescued(session, **params) @router.post( @@ -136,7 +127,6 @@ async def mark_job_rescued( async def rematch_pipetask_errors( query: models.RematchQuery, session: async_scoped_session = Depends(db_session_dependency), -) -> List[db.PipetaskError]: +) -> list[db.PipetaskError]: params = query.dict() - result = await interface.match_pipetask_errors(session, **params) - return result + return await interface.match_pipetask_errors(session, **params) diff --git a/src/lsst/cmservice/routers/adders.py b/src/lsst/cmservice/routers/adders.py index 3eae68b5..3a420e15 100644 --- a/src/lsst/cmservice/routers/adders.py +++ b/src/lsst/cmservice/routers/adders.py @@ -21,8 +21,7 @@ async def add_groups( query: models.AddGroups, session: async_scoped_session = Depends(db_session_dependency), ) -> db.Step: - result = await interface.add_groups(session, **query.dict()) - return result + return await interface.add_groups(session, **query.dict()) @router.post( @@ -35,8 +34,7 @@ async def add_steps( query: models.AddGroups, session: async_scoped_session = Depends(db_session_dependency), ) -> db.Campaign: - result = await interface.add_steps(session, **query.dict()) - return result + return await interface.add_steps(session, **query.dict()) @router.post( @@ -49,5 +47,4 @@ async def add_campaign( query: models.CampaignCreate, session: async_scoped_session = Depends(db_session_dependency), ) -> db.Campaign: - result = await interface.create_campaign(session, **query.dict()) - return result + return await interface.create_campaign(session, **query.dict()) diff --git a/src/lsst/cmservice/routers/campaigns.py b/src/lsst/cmservice/routers/campaigns.py index 4606a0b5..2ff6f935 100644 --- a/src/lsst/cmservice/routers/campaigns.py +++ b/src/lsst/cmservice/routers/campaigns.py @@ -30,7 +30,7 @@ async def get_rows( limit: int = 100, session: async_scoped_session = Depends(db_session_dependency), ) -> Sequence[db_class]: - result = await db_class.get_rows( + return await db_class.get_rows( session, parent_id=parent_id, skip=skip, @@ -38,7 +38,6 @@ async def get_rows( parent_name=parent_name, parent_class=db.Production, ) - return result @router.get( @@ -50,5 +49,4 @@ async def get_row( row_id: int, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.get_row(session, row_id) - return result + return await db_class.get_row(session, row_id) diff --git a/src/lsst/cmservice/routers/expert_campaigns.py b/src/lsst/cmservice/routers/expert_campaigns.py index 06ea9d20..bdd87230 100644 --- a/src/lsst/cmservice/routers/expert_campaigns.py +++ b/src/lsst/cmservice/routers/expert_campaigns.py @@ -1,4 +1,4 @@ -from typing import Sequence +from collections.abc import Sequence from fastapi import APIRouter, Depends, HTTPException from safir.dependencies.db_session import db_session_dependency @@ -30,14 +30,13 @@ async def get_rows( limit: int = 100, session: async_scoped_session = Depends(db_session_dependency), ) -> Sequence[db_class]: - result = await db_class.get_rows( + return await db_class.get_rows( session, parent_id=parent_id, skip=skip, limit=limit, parent_class=db.Production, ) - return result @router.get( @@ -49,8 +48,7 @@ async def get_row( row_id: int, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.get_row(session, row_id) - return result + return await db_class.get_row(session, row_id) @router.post( diff --git a/src/lsst/cmservice/routers/expert_groups.py b/src/lsst/cmservice/routers/expert_groups.py index 7bd70e6f..47157dc6 100644 --- a/src/lsst/cmservice/routers/expert_groups.py +++ b/src/lsst/cmservice/routers/expert_groups.py @@ -1,4 +1,4 @@ -from typing import Sequence +from collections.abc import Sequence from fastapi import APIRouter, Depends from safir.dependencies.db_session import db_session_dependency @@ -29,8 +29,7 @@ async def get_rows( limit: int = 100, session: async_scoped_session = Depends(db_session_dependency), ) -> Sequence[db_class]: - result = await db_class.get_rows(session, skip=skip, limit=limit) - return result + return await db_class.get_rows(session, skip=skip, limit=limit) @router.get( @@ -42,8 +41,7 @@ async def get_row( row_id: int, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.get_row(session, row_id) - return result + return await db_class.get_row(session, row_id) @router.post( diff --git a/src/lsst/cmservice/routers/expert_jobs.py b/src/lsst/cmservice/routers/expert_jobs.py index d0776531..0b33a2d2 100644 --- a/src/lsst/cmservice/routers/expert_jobs.py +++ b/src/lsst/cmservice/routers/expert_jobs.py @@ -1,4 +1,4 @@ -from typing import Sequence +from collections.abc import Sequence from fastapi import APIRouter, Depends from safir.dependencies.db_session import db_session_dependency @@ -29,8 +29,7 @@ async def get_rows( limit: int = 100, session: async_scoped_session = Depends(db_session_dependency), ) -> Sequence[db_class]: - result = await db_class.get_rows(session, skip=skip, limit=limit) - return result + return await db_class.get_rows(session, skip=skip, limit=limit) @router.get( @@ -42,8 +41,7 @@ async def get_row( row_id: int, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.get_row(session, row_id) - return result + return await db_class.get_row(session, row_id) @router.post( diff --git a/src/lsst/cmservice/routers/expert_pipetask_error_types.py b/src/lsst/cmservice/routers/expert_pipetask_error_types.py index d489bd3b..11197d40 100644 --- a/src/lsst/cmservice/routers/expert_pipetask_error_types.py +++ b/src/lsst/cmservice/routers/expert_pipetask_error_types.py @@ -1,4 +1,4 @@ -from typing import Sequence +from collections.abc import Sequence from fastapi import APIRouter, Depends from safir.dependencies.db_session import db_session_dependency @@ -29,8 +29,7 @@ async def get_rows( limit: int = 100, session: async_scoped_session = Depends(db_session_dependency), ) -> Sequence[db_class]: - result = await db_class.get_rows(session, skip=skip, limit=limit) - return result + return await db_class.get_rows(session, skip=skip, limit=limit) @router.get( @@ -42,8 +41,7 @@ async def get_row( row_id: int, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.get_row(session, row_id) - return result + return await db_class.get_row(session, row_id) @router.post( diff --git a/src/lsst/cmservice/routers/expert_pipetask_errors.py b/src/lsst/cmservice/routers/expert_pipetask_errors.py index 80215c80..1eff9d62 100644 --- a/src/lsst/cmservice/routers/expert_pipetask_errors.py +++ b/src/lsst/cmservice/routers/expert_pipetask_errors.py @@ -1,4 +1,4 @@ -from typing import Sequence +from collections.abc import Sequence from fastapi import APIRouter, Depends from safir.dependencies.db_session import db_session_dependency @@ -28,8 +28,7 @@ async def get_rows( limit: int = 100, session: async_scoped_session = Depends(db_session_dependency), ) -> Sequence[db_class]: - result = await db_class.get_rows(session, skip=skip, limit=limit) - return result + return await db_class.get_rows(session, skip=skip, limit=limit) @router.get( @@ -41,8 +40,7 @@ async def get_row( row_id: int, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.get_row(session, row_id) - return result + return await db_class.get_row(session, row_id) @router.post( diff --git a/src/lsst/cmservice/routers/expert_product_sets.py b/src/lsst/cmservice/routers/expert_product_sets.py index a7612c94..26125893 100644 --- a/src/lsst/cmservice/routers/expert_product_sets.py +++ b/src/lsst/cmservice/routers/expert_product_sets.py @@ -1,4 +1,4 @@ -from typing import Sequence +from collections.abc import Sequence from fastapi import APIRouter, Depends from safir.dependencies.db_session import db_session_dependency @@ -29,8 +29,7 @@ async def get_rows( limit: int = 100, session: async_scoped_session = Depends(db_session_dependency), ) -> Sequence[db_class]: - result = await db_class.get_rows(session, skip=skip, limit=limit) - return result + return await db_class.get_rows(session, skip=skip, limit=limit) @router.get( @@ -42,8 +41,7 @@ async def get_row( row_id: int, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.get_row(session, row_id) - return result + return await db_class.get_row(session, row_id) @router.post( @@ -83,5 +81,4 @@ async def update_row( row_update: response_model_class, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.update_row(session, row_id, **row_update.dict()) - return result + return await db_class.update_row(session, row_id, **row_update.dict()) diff --git a/src/lsst/cmservice/routers/expert_productions.py b/src/lsst/cmservice/routers/expert_productions.py index 3ed561f5..da2286a7 100644 --- a/src/lsst/cmservice/routers/expert_productions.py +++ b/src/lsst/cmservice/routers/expert_productions.py @@ -29,8 +29,7 @@ async def get_rows( limit: int = 100, session: async_scoped_session = Depends(db_session_dependency), ) -> Sequence[db_class]: - result = await db_class.get_rows(session, skip=skip, limit=limit) - return result + return await db_class.get_rows(session, skip=skip, limit=limit) @router.get( @@ -42,8 +41,7 @@ async def get_row( row_id: int, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.get_row(session, row_id) - return result + return await db_class.get_row(session, row_id) @router.post( @@ -83,5 +81,4 @@ async def update_row( row_update: response_model_class, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.update_row(session, row_id, **row_update.dict()) - return result + return await db_class.update_row(session, row_id, **row_update.dict()) diff --git a/src/lsst/cmservice/routers/expert_queues.py b/src/lsst/cmservice/routers/expert_queues.py index a0270687..eafdb558 100644 --- a/src/lsst/cmservice/routers/expert_queues.py +++ b/src/lsst/cmservice/routers/expert_queues.py @@ -1,4 +1,4 @@ -from typing import Sequence +from collections.abc import Sequence from fastapi import APIRouter, Depends from safir.dependencies.db_session import db_session_dependency @@ -28,8 +28,7 @@ async def get_rows( limit: int = 100, session: async_scoped_session = Depends(db_session_dependency), ) -> Sequence[db_class]: - result = await db_class.get_rows(session, skip=skip, limit=limit) - return result + return await db_class.get_rows(session, skip=skip, limit=limit) @router.get( @@ -41,8 +40,7 @@ async def get_row( row_id: int, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.get_row(session, row_id) - return result + return await db_class.get_row(session, row_id) @router.post( @@ -82,5 +80,4 @@ async def update_row( row_update: response_model_class, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.update_row(session, row_id, **row_update.dict()) - return result + return await db_class.update_row(session, row_id, **row_update.dict()) diff --git a/src/lsst/cmservice/routers/expert_script_dependencies.py b/src/lsst/cmservice/routers/expert_script_dependencies.py index 1f2560f5..a571b2c9 100644 --- a/src/lsst/cmservice/routers/expert_script_dependencies.py +++ b/src/lsst/cmservice/routers/expert_script_dependencies.py @@ -1,4 +1,4 @@ -from typing import Sequence +from collections.abc import Sequence from fastapi import APIRouter, Depends from safir.dependencies.db_session import db_session_dependency @@ -29,8 +29,7 @@ async def get_rows( limit: int = 100, session: async_scoped_session = Depends(db_session_dependency), ) -> Sequence[db_class]: - result = await db_class.get_rows(session, skip=skip, limit=limit) - return result + return await db_class.get_rows(session, skip=skip, limit=limit) @router.get( @@ -42,8 +41,7 @@ async def get_row( row_id: int, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.get_row(session, row_id) - return result + return await db_class.get_row(session, row_id) @router.post( diff --git a/src/lsst/cmservice/routers/expert_script_errors.py b/src/lsst/cmservice/routers/expert_script_errors.py index 354757e2..5279f78f 100644 --- a/src/lsst/cmservice/routers/expert_script_errors.py +++ b/src/lsst/cmservice/routers/expert_script_errors.py @@ -1,4 +1,4 @@ -from typing import Sequence +from collections.abc import Sequence from fastapi import APIRouter, Depends from safir.dependencies.db_session import db_session_dependency @@ -28,8 +28,7 @@ async def get_rows( limit: int = 100, session: async_scoped_session = Depends(db_session_dependency), ) -> Sequence[db_class]: - result = await db_class.get_rows(session, skip=skip, limit=limit) - return result + return await db_class.get_rows(session, skip=skip, limit=limit) @router.get( @@ -41,8 +40,7 @@ async def get_row( row_id: int, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.get_row(session, row_id) - return result + return await db_class.get_row(session, row_id) @router.post( diff --git a/src/lsst/cmservice/routers/expert_script_templates.py b/src/lsst/cmservice/routers/expert_script_templates.py index 1c6134cf..51e26b7b 100644 --- a/src/lsst/cmservice/routers/expert_script_templates.py +++ b/src/lsst/cmservice/routers/expert_script_templates.py @@ -1,4 +1,4 @@ -from typing import Sequence +from collections.abc import Sequence from fastapi import APIRouter, Depends from safir.dependencies.db_session import db_session_dependency @@ -29,8 +29,7 @@ async def get_rows( limit: int = 100, session: async_scoped_session = Depends(db_session_dependency), ) -> Sequence[db_class]: - result = await db_class.get_rows(session, skip=skip, limit=limit) - return result + return await db_class.get_rows(session, skip=skip, limit=limit) @router.get( @@ -42,8 +41,7 @@ async def get_row( row_id: int, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.get_row(session, row_id) - return result + return await db_class.get_row(session, row_id) @router.post( diff --git a/src/lsst/cmservice/routers/expert_scripts.py b/src/lsst/cmservice/routers/expert_scripts.py index 1313f69a..5c186404 100644 --- a/src/lsst/cmservice/routers/expert_scripts.py +++ b/src/lsst/cmservice/routers/expert_scripts.py @@ -1,4 +1,4 @@ -from typing import Sequence +from collections.abc import Sequence from fastapi import APIRouter, Depends from safir.dependencies.db_session import db_session_dependency @@ -30,8 +30,7 @@ async def get_rows( limit: int = 100, session: async_scoped_session = Depends(db_session_dependency), ) -> Sequence[db_class]: - result = await db_class.get_rows(session, skip=skip, limit=limit) - return result + return await db_class.get_rows(session, skip=skip, limit=limit) @router.get( @@ -43,8 +42,7 @@ async def get_row( row_id: int, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.get_row(session, row_id) - return result + return await db_class.get_row(session, row_id) @router.post( @@ -84,8 +82,7 @@ async def update_row( row_update: response_model_class, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.update_row(session, row_id, **row_update.dict()) - return result + return await db_class.update_row(session, row_id, **row_update.dict()) @router.put( diff --git a/src/lsst/cmservice/routers/expert_spec_blocks.py b/src/lsst/cmservice/routers/expert_spec_blocks.py index fcc54d60..65c33947 100644 --- a/src/lsst/cmservice/routers/expert_spec_blocks.py +++ b/src/lsst/cmservice/routers/expert_spec_blocks.py @@ -1,4 +1,4 @@ -from typing import Sequence +from collections.abc import Sequence from fastapi import APIRouter, Depends from safir.dependencies.db_session import db_session_dependency @@ -29,8 +29,7 @@ async def get_rows( limit: int = 100, session: async_scoped_session = Depends(db_session_dependency), ) -> Sequence[db_class]: - result = await db_class.get_rows(session, skip=skip, limit=limit) - return result + return await db_class.get_rows(session, skip=skip, limit=limit) @router.get( @@ -42,8 +41,7 @@ async def get_row( row_id: int, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.get_row(session, row_id) - return result + return await db_class.get_row(session, row_id) @router.post( diff --git a/src/lsst/cmservice/routers/expert_specifications.py b/src/lsst/cmservice/routers/expert_specifications.py index 0edf154a..b047f5d3 100644 --- a/src/lsst/cmservice/routers/expert_specifications.py +++ b/src/lsst/cmservice/routers/expert_specifications.py @@ -1,4 +1,4 @@ -from typing import Sequence +from collections.abc import Sequence from fastapi import APIRouter, Depends from safir.dependencies.db_session import db_session_dependency @@ -29,8 +29,7 @@ async def get_rows( limit: int = 100, session: async_scoped_session = Depends(db_session_dependency), ) -> Sequence[db_class]: - result = await db_class.get_rows(session, skip=skip, limit=limit) - return result + return await db_class.get_rows(session, skip=skip, limit=limit) @router.get( @@ -42,8 +41,7 @@ async def get_row( row_id: int, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.get_row(session, row_id) - return result + return await db_class.get_row(session, row_id) @router.post( diff --git a/src/lsst/cmservice/routers/expert_step_dependencies.py b/src/lsst/cmservice/routers/expert_step_dependencies.py index 0f0efc33..4829765c 100644 --- a/src/lsst/cmservice/routers/expert_step_dependencies.py +++ b/src/lsst/cmservice/routers/expert_step_dependencies.py @@ -1,4 +1,4 @@ -from typing import Sequence +from collections.abc import Sequence from fastapi import APIRouter, Depends from safir.dependencies.db_session import db_session_dependency @@ -29,8 +29,7 @@ async def get_rows( limit: int = 100, session: async_scoped_session = Depends(db_session_dependency), ) -> Sequence[db_class]: - result = await db_class.get_rows(session, skip=skip, limit=limit) - return result + return await db_class.get_rows(session, skip=skip, limit=limit) @router.get( @@ -42,8 +41,7 @@ async def get_row( row_id: int, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.get_row(session, row_id) - return result + return await db_class.get_row(session, row_id) @router.post( diff --git a/src/lsst/cmservice/routers/expert_steps.py b/src/lsst/cmservice/routers/expert_steps.py index 769a976d..3dc81c63 100644 --- a/src/lsst/cmservice/routers/expert_steps.py +++ b/src/lsst/cmservice/routers/expert_steps.py @@ -1,4 +1,4 @@ -from typing import Sequence +from collections.abc import Sequence from fastapi import APIRouter, Depends from safir.dependencies.db_session import db_session_dependency @@ -29,8 +29,7 @@ async def get_rows( limit: int = 100, session: async_scoped_session = Depends(db_session_dependency), ) -> Sequence[db_class]: - result = await db_class.get_rows(session, skip=skip, limit=limit) - return result + return await db_class.get_rows(session, skip=skip, limit=limit) @router.get( @@ -42,8 +41,7 @@ async def get_row( row_id: int, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.get_row(session, row_id) - return result + return await db_class.get_row(session, row_id) @router.post( diff --git a/src/lsst/cmservice/routers/expert_task_sets.py b/src/lsst/cmservice/routers/expert_task_sets.py index 48996541..5f925c0f 100644 --- a/src/lsst/cmservice/routers/expert_task_sets.py +++ b/src/lsst/cmservice/routers/expert_task_sets.py @@ -1,4 +1,4 @@ -from typing import Sequence +from collections.abc import Sequence from fastapi import APIRouter, Depends from safir.dependencies.db_session import db_session_dependency @@ -28,8 +28,7 @@ async def get_rows( limit: int = 100, session: async_scoped_session = Depends(db_session_dependency), ) -> Sequence[db_class]: - result = await db_class.get_rows(session, skip=skip, limit=limit) - return result + return await db_class.get_rows(session, skip=skip, limit=limit) @router.get( @@ -41,8 +40,7 @@ async def get_row( row_id: int, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.get_row(session, row_id) - return result + return await db_class.get_row(session, row_id) @router.post( @@ -82,5 +80,4 @@ async def update_row( row_update: response_model_class, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.update_row(session, row_id, **row_update.dict()) - return result + return await db_class.update_row(session, row_id, **row_update.dict()) diff --git a/src/lsst/cmservice/routers/groups.py b/src/lsst/cmservice/routers/groups.py index 5710bc81..bb064d8b 100644 --- a/src/lsst/cmservice/routers/groups.py +++ b/src/lsst/cmservice/routers/groups.py @@ -31,7 +31,7 @@ async def get_rows( limit: int = 100, session: async_scoped_session = Depends(db_session_dependency), ) -> Sequence[db_class]: - result = await db_class.get_rows( + return await db_class.get_rows( session, parent_id=parent_id, skip=skip, @@ -39,7 +39,6 @@ async def get_rows( parent_name=parent_name, parent_class=db.Step, ) - return result @router.get( @@ -51,5 +50,4 @@ async def get_row( row_id: int, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.get_row(session, row_id) - return result + return await db_class.get_row(session, row_id) diff --git a/src/lsst/cmservice/routers/jobs.py b/src/lsst/cmservice/routers/jobs.py index c8f7647a..4a026905 100644 --- a/src/lsst/cmservice/routers/jobs.py +++ b/src/lsst/cmservice/routers/jobs.py @@ -1,4 +1,4 @@ -from typing import Sequence +from collections.abc import Sequence from fastapi import APIRouter, Depends from safir.dependencies.db_session import db_session_dependency @@ -29,8 +29,7 @@ async def get_rows( limit: int = 100, session: async_scoped_session = Depends(db_session_dependency), ) -> Sequence[db_class]: - result = await db_class.get_rows(session, skip=skip, limit=limit) - return result + return await db_class.get_rows(session, skip=skip, limit=limit) @router.get( @@ -42,5 +41,4 @@ async def get_row( row_id: int, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.get_row(session, row_id) - return result + return await db_class.get_row(session, row_id) diff --git a/src/lsst/cmservice/routers/loaders.py b/src/lsst/cmservice/routers/loaders.py index 648af793..5b24b180 100644 --- a/src/lsst/cmservice/routers/loaders.py +++ b/src/lsst/cmservice/routers/loaders.py @@ -1,5 +1,3 @@ -from typing import List - from fastapi import APIRouter, Depends from safir.dependencies.db_session import db_session_dependency from sqlalchemy.ext.asyncio import async_scoped_session @@ -52,9 +50,8 @@ async def load_and_create_campaign( async def load_error_types( query: models.YamlFileQuery, session: async_scoped_session = Depends(db_session_dependency), -) -> List[db.PipetaskErrorType]: - result = await interface.load_error_types(session, **query.dict()) - return result +) -> list[db.PipetaskErrorType]: + return await interface.load_error_types(session, **query.dict()) @router.post( @@ -67,5 +64,4 @@ async def load_manifest_report( query: models.LoadManifestReport, session: async_scoped_session = Depends(db_session_dependency), ) -> db.Job: - result = await interface.load_manifest_report(session, **query.dict()) - return result + return await interface.load_manifest_report(session, **query.dict()) diff --git a/src/lsst/cmservice/routers/pipetask_error_types.py b/src/lsst/cmservice/routers/pipetask_error_types.py index d489bd3b..11197d40 100644 --- a/src/lsst/cmservice/routers/pipetask_error_types.py +++ b/src/lsst/cmservice/routers/pipetask_error_types.py @@ -1,4 +1,4 @@ -from typing import Sequence +from collections.abc import Sequence from fastapi import APIRouter, Depends from safir.dependencies.db_session import db_session_dependency @@ -29,8 +29,7 @@ async def get_rows( limit: int = 100, session: async_scoped_session = Depends(db_session_dependency), ) -> Sequence[db_class]: - result = await db_class.get_rows(session, skip=skip, limit=limit) - return result + return await db_class.get_rows(session, skip=skip, limit=limit) @router.get( @@ -42,8 +41,7 @@ async def get_row( row_id: int, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.get_row(session, row_id) - return result + return await db_class.get_row(session, row_id) @router.post( diff --git a/src/lsst/cmservice/routers/productions.py b/src/lsst/cmservice/routers/productions.py index 3855f9bb..d91cc3d2 100644 --- a/src/lsst/cmservice/routers/productions.py +++ b/src/lsst/cmservice/routers/productions.py @@ -1,4 +1,4 @@ -from typing import Sequence +from collections.abc import Sequence from fastapi import APIRouter, Depends from safir.dependencies.db_session import db_session_dependency @@ -29,8 +29,7 @@ async def get_rows( limit: int = 100, session: async_scoped_session = Depends(db_session_dependency), ) -> Sequence[db_class]: - result = await db_class.get_rows(session, skip=skip, limit=limit) - return result + return await db_class.get_rows(session, skip=skip, limit=limit) @router.get( @@ -42,5 +41,4 @@ async def get_row( row_id: int, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.get_row(session, row_id) - return result + return await db_class.get_row(session, row_id) diff --git a/src/lsst/cmservice/routers/queries.py b/src/lsst/cmservice/routers/queries.py index 9b8f39c7..ce417f35 100644 --- a/src/lsst/cmservice/routers/queries.py +++ b/src/lsst/cmservice/routers/queries.py @@ -1,5 +1,3 @@ -from typing import List - from fastapi import APIRouter, Depends from safir.dependencies.db_session import db_session_dependency from sqlalchemy.ext.asyncio import async_scoped_session @@ -22,11 +20,10 @@ async def get_element( fullname: str, session: async_scoped_session = Depends(db_session_dependency), ) -> db.ElementMixin: - result = await interface.get_element_by_fullname( + return await interface.get_element_by_fullname( session, fullname, ) - return result @router.get( @@ -38,11 +35,10 @@ async def get_script( fullname: str, session: async_scoped_session = Depends(db_session_dependency), ) -> db.Script: - result = await db.Script.get_row_by_fullname( + return await db.Script.get_row_by_fullname( session, fullname, ) - return result @router.get( @@ -54,11 +50,10 @@ async def get_job( fullname: str, session: async_scoped_session = Depends(db_session_dependency), ) -> db.Job: - result = await db.Job.get_row_by_fullname( + return await db.Job.get_row_by_fullname( session, fullname, ) - return result @router.get( @@ -70,11 +65,10 @@ async def get_spec_block( fullname: str, session: async_scoped_session = Depends(db_session_dependency), ) -> db.SpecBlock: - result = await interface.get_spec_block( + return await interface.get_spec_block( session, fullname, ) - return result @router.get( @@ -86,11 +80,10 @@ async def get_specification( fullname: str, session: async_scoped_session = Depends(db_session_dependency), ) -> db.Specification: - result = await interface.get_specification( + return await interface.get_specification( session, fullname, ) - return result @router.get( @@ -102,11 +95,10 @@ async def get_resolved_collections( fullname: str, session: async_scoped_session = Depends(db_session_dependency), ) -> dict: - result = await interface.get_resolved_collections( + return await interface.get_resolved_collections( session, fullname, ) - return result @router.get( @@ -118,11 +110,10 @@ async def get_collections( fullname: str, session: async_scoped_session = Depends(db_session_dependency), ) -> dict: - result = await interface.get_collections( + return await interface.get_collections( session, fullname, ) - return result @router.get( @@ -134,11 +125,10 @@ async def get_child_config( fullname: str, session: async_scoped_session = Depends(db_session_dependency), ) -> dict: - result = await interface.get_child_config( + return await interface.get_child_config( session, fullname, ) - return result @router.get( @@ -150,11 +140,10 @@ async def get_data_dict( fullname: str, session: async_scoped_session = Depends(db_session_dependency), ) -> dict: - result = await interface.get_data_dict( + return await interface.get_data_dict( session, fullname, ) - return result @router.get( @@ -166,11 +155,10 @@ async def get_spec_aliases( fullname: str, session: async_scoped_session = Depends(db_session_dependency), ) -> dict: - result = await interface.get_spec_aliases( + return await interface.get_spec_aliases( session, fullname, ) - return result @router.get( @@ -182,11 +170,10 @@ async def get_prerequisites( fullname: str, session: async_scoped_session = Depends(db_session_dependency), ) -> bool: - result = await interface.check_prerequisites( + return await interface.check_prerequisites( session, fullname=fullname, ) - return result @router.get( @@ -197,18 +184,18 @@ async def get_prerequisites( async def get_scripts( fullname: str, script_name: str, + *, remaining_only: bool = False, skip_superseded: bool = True, session: async_scoped_session = Depends(db_session_dependency), -) -> List[db.Script]: - result = await interface.get_scripts( +) -> list[db.Script]: + return await interface.get_scripts( session, fullname=fullname, script_name=script_name, remaining_only=remaining_only, skip_superseded=skip_superseded, ) - return result @router.get( @@ -218,17 +205,17 @@ async def get_scripts( ) async def get_jobs( fullname: str, + *, remaining_only: bool = False, skip_superseded: bool = True, session: async_scoped_session = Depends(db_session_dependency), -) -> List[db.Job]: - result = await interface.get_jobs( +) -> list[db.Job]: + return await interface.get_jobs( session, fullname=fullname, remaining_only=remaining_only, skip_superseded=skip_superseded, ) - return result @router.get( @@ -239,12 +226,11 @@ async def get_jobs( async def get_job_task_sets( fullname: str, session: async_scoped_session = Depends(db_session_dependency), -) -> List[db.TaskSet]: - result = await interface.get_task_sets_for_job( +) -> list[db.TaskSet]: + return await interface.get_task_sets_for_job( session, fullname=fullname, ) - return result @router.get( @@ -255,12 +241,11 @@ async def get_job_task_sets( async def get_job_wms_reports( fullname: str, session: async_scoped_session = Depends(db_session_dependency), -) -> List[db.WmsTaskReport]: - result = await interface.get_wms_reports_for_job( +) -> list[db.WmsTaskReport]: + return await interface.get_wms_reports_for_job( session, fullname=fullname, ) - return result @router.get( @@ -271,12 +256,11 @@ async def get_job_wms_reports( async def get_job_product_sets( fullname: str, session: async_scoped_session = Depends(db_session_dependency), -) -> List[db.ProductSet]: - result = await interface.get_product_sets_for_job( +) -> list[db.ProductSet]: + return await interface.get_product_sets_for_job( session, fullname=fullname, ) - return result @router.get( @@ -287,9 +271,8 @@ async def get_job_product_sets( async def get_job_errors( fullname: str, session: async_scoped_session = Depends(db_session_dependency), -) -> List[db.PipetaskError]: - result = await interface.get_errors_for_job( +) -> list[db.PipetaskError]: + return await interface.get_errors_for_job( session, fullname=fullname, ) - return result diff --git a/src/lsst/cmservice/routers/script_templates.py b/src/lsst/cmservice/routers/script_templates.py index 821a640e..75fb6e10 100644 --- a/src/lsst/cmservice/routers/script_templates.py +++ b/src/lsst/cmservice/routers/script_templates.py @@ -1,4 +1,4 @@ -from typing import Sequence +from collections.abc import Sequence from fastapi import APIRouter, Depends from safir.dependencies.db_session import db_session_dependency @@ -31,7 +31,7 @@ async def get_rows( limit: int = 100, session: async_scoped_session = Depends(db_session_dependency), ) -> Sequence[db_class]: - result = await db_class.get_rows( + return await db_class.get_rows( session, parent_id=parent_id, skip=skip, @@ -39,7 +39,6 @@ async def get_rows( parent_name=parent_name, parent_class=db.Specification, ) - return result @router.get( @@ -51,5 +50,4 @@ async def get_row( row_id: int, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.get_row(session, row_id) - return result + return await db_class.get_row(session, row_id) diff --git a/src/lsst/cmservice/routers/scripts.py b/src/lsst/cmservice/routers/scripts.py index 3096fbaf..fc0e6c4c 100644 --- a/src/lsst/cmservice/routers/scripts.py +++ b/src/lsst/cmservice/routers/scripts.py @@ -1,4 +1,4 @@ -from typing import Sequence +from collections.abc import Sequence from fastapi import APIRouter, Depends from safir.dependencies.db_session import db_session_dependency @@ -29,8 +29,7 @@ async def get_rows( limit: int = 100, session: async_scoped_session = Depends(db_session_dependency), ) -> Sequence[db_class]: - result = await db_class.get_rows(session, skip=skip, limit=limit) - return result + return await db_class.get_rows(session, skip=skip, limit=limit) @router.get( @@ -42,5 +41,4 @@ async def get_row( row_id: int, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.get_row(session, row_id) - return result + return await db_class.get_row(session, row_id) diff --git a/src/lsst/cmservice/routers/spec_blocks.py b/src/lsst/cmservice/routers/spec_blocks.py index 22a9a0d5..b08d2d52 100644 --- a/src/lsst/cmservice/routers/spec_blocks.py +++ b/src/lsst/cmservice/routers/spec_blocks.py @@ -1,4 +1,4 @@ -from typing import Sequence +from collections.abc import Sequence from fastapi import APIRouter, Depends from safir.dependencies.db_session import db_session_dependency @@ -31,7 +31,7 @@ async def get_rows( limit: int = 100, session: async_scoped_session = Depends(db_session_dependency), ) -> Sequence[db_class]: - result = await db_class.get_rows( + return await db_class.get_rows( session, parent_id=parent_id, skip=skip, @@ -39,7 +39,6 @@ async def get_rows( parent_name=parent_name, parent_class=db.Specification, ) - return result @router.get( @@ -51,5 +50,4 @@ async def get_row( row_id: int, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.get_row(session, row_id) - return result + return await db_class.get_row(session, row_id) diff --git a/src/lsst/cmservice/routers/steps.py b/src/lsst/cmservice/routers/steps.py index 31315856..3a672ba2 100644 --- a/src/lsst/cmservice/routers/steps.py +++ b/src/lsst/cmservice/routers/steps.py @@ -31,7 +31,7 @@ async def get_rows( limit: int = 100, session: async_scoped_session = Depends(db_session_dependency), ) -> Sequence[db_class]: - result = await db_class.get_rows( + return await db_class.get_rows( session, parent_id=parent_id, skip=skip, @@ -39,7 +39,6 @@ async def get_rows( parent_name=parent_name, parent_class=db.Campaign, ) - return result @router.get( @@ -51,5 +50,4 @@ async def get_row( row_id: int, session: async_scoped_session = Depends(db_session_dependency), ) -> db_class: - result = await db_class.get_row(session, row_id) - return result + return await db_class.get_row(session, row_id) diff --git a/src/lsst/cmservice/routers/updates.py b/src/lsst/cmservice/routers/updates.py index a7f5cd41..40402385 100644 --- a/src/lsst/cmservice/routers/updates.py +++ b/src/lsst/cmservice/routers/updates.py @@ -21,12 +21,11 @@ async def update_status( query: models.UpdateStatusQuery, session: async_scoped_session = Depends(db_session_dependency), ) -> db.NodeMixin: - result = await interface.update_status( + return await interface.update_status( session, query.fullname, query.status, ) - return result @router.post( @@ -39,12 +38,11 @@ async def update_collections( query: models.UpdateNodeQuery, session: async_scoped_session = Depends(db_session_dependency), ) -> db.NodeMixin: - result = await interface.update_collections( + return await interface.update_collections( session, query.fullname, **query.update_dict, ) - return result @router.post( @@ -57,12 +55,11 @@ async def update_child_config( query: models.UpdateNodeQuery, session: async_scoped_session = Depends(db_session_dependency), ) -> db.NodeMixin: - result = await interface.update_child_config( + return await interface.update_child_config( session, query.fullname, **query.update_dict, ) - return result @router.post( @@ -75,12 +72,11 @@ async def update_data_dict( query: models.UpdateNodeQuery, session: async_scoped_session = Depends(db_session_dependency), ) -> db.NodeMixin: - result = await interface.update_data_dict( + return await interface.update_data_dict( session, query.fullname, **query.update_dict, ) - return result @router.post( @@ -93,9 +89,8 @@ async def update_spec_aliases( query: models.UpdateNodeQuery, session: async_scoped_session = Depends(db_session_dependency), ) -> db.NodeMixin: - result = await interface.update_spec_aliases( + return await interface.update_spec_aliases( session, query.fullname, **query.update_dict, ) - return result diff --git a/tests/db/test_campaign.py b/tests/db/test_campaign.py index aaabb0c1..d37c56d7 100644 --- a/tests/db/test_campaign.py +++ b/tests/db/test_campaign.py @@ -17,7 +17,10 @@ async def test_campaign_db(session: async_scoped_session) -> None: camps0 = [ await db.Campaign.create_row( - session, name=cname_, spec_block_name="base#campaign", parent_name=pnames[0] + session, + name=cname_, + spec_block_name="base#campaign", + parent_name=pnames[0], ) for cname_ in cnames ] @@ -25,7 +28,10 @@ async def test_campaign_db(session: async_scoped_session) -> None: camps1 = [ await db.Campaign.create_row( - session, name=cname_, spec_block_name="base#campaign", parent_name=pnames[1] + session, + name=cname_, + spec_block_name="base#campaign", + parent_name=pnames[1], ) for cname_ in cnames ] @@ -33,7 +39,10 @@ async def test_campaign_db(session: async_scoped_session) -> None: with pytest.raises(IntegrityError): await db.Campaign.create_row( - session, name=cnames[0], parent_name=pnames[0], spec_block_name="base#campaign" + session, + name=cnames[0], + parent_name=pnames[0], + spec_block_name="base#campaign", ) await db.Production.delete_row(session, prods[0].id) diff --git a/tests/db/test_group.py b/tests/db/test_group.py index d38e7da9..9484748f 100644 --- a/tests/db/test_group.py +++ b/tests/db/test_group.py @@ -13,7 +13,10 @@ async def test_group_db(session: async_scoped_session) -> None: prod = await db.Production.create_row(session, name=pname) cname = str(uuid1()) camp = await db.Campaign.create_row( - session, name=cname, spec_block_name="base#campaign", parent_name=pname + session, + name=cname, + spec_block_name="base#campaign", + parent_name=pname, ) snames = [str(uuid1()) for n in range(2)] @@ -31,7 +34,10 @@ async def test_group_db(session: async_scoped_session) -> None: groups0 = [ await db.Group.create_row( - session, name=gname_, spec_block_name="base#group", parent_name=steps[0].fullname + session, + name=gname_, + spec_block_name="base#group", + parent_name=steps[0].fullname, ) for gname_ in gnames ] @@ -39,7 +45,10 @@ async def test_group_db(session: async_scoped_session) -> None: groups1 = [ await db.Group.create_row( - session, name=gname_, spec_block_name="base#group", parent_name=steps[1].fullname + session, + name=gname_, + spec_block_name="base#group", + parent_name=steps[1].fullname, ) for gname_ in gnames ] @@ -47,7 +56,10 @@ async def test_group_db(session: async_scoped_session) -> None: with pytest.raises(IntegrityError): await db.Group.create_row( - session, name=gnames[0], parent_name=steps[0].fullname, spec_block_name="base#group" + session, + name=gnames[0], + parent_name=steps[0].fullname, + spec_block_name="base#group", ) # Finish clean up diff --git a/tests/db/test_step.py b/tests/db/test_step.py index 3bb41384..13c08b18 100644 --- a/tests/db/test_step.py +++ b/tests/db/test_step.py @@ -22,7 +22,10 @@ async def test_step_db(session: async_scoped_session) -> None: steps0 = [ await db.Step.create_row( - session, name=sname_, spec_block_name="base#basic_step", parent_name=camps[0].fullname + session, + name=sname_, + spec_block_name="base#basic_step", + parent_name=camps[0].fullname, ) for sname_ in snames ] @@ -30,7 +33,10 @@ async def test_step_db(session: async_scoped_session) -> None: steps1 = [ await db.Step.create_row( - session, name=sname_, spec_block_name="base#basic_step", parent_name=camps[1].fullname + session, + name=sname_, + spec_block_name="base#basic_step", + parent_name=camps[1].fullname, ) for sname_ in snames ] @@ -38,7 +44,10 @@ async def test_step_db(session: async_scoped_session) -> None: with pytest.raises(IntegrityError): await db.Step.create_row( - session, name=snames[0], parent_name=camps[0].fullname, spec_block_name="base#basic_step" + session, + name=snames[0], + parent_name=camps[0].fullname, + spec_block_name="base#basic_step", ) await db.Campaign.delete_row(session, camps[0].id)