Ruff autofixes + enable ruff UP,COM in pre-commit

pyproject.toml

@@ -94,6 +94,8 @@ select = [
     "F",  # pyflakes
     "N",  # pep8-naming
     "W",  # pycodestyle
+    "UP", # pyupgrade
+    "COM", # pyflakes-commas
 ]
 target-version = "py311"
 extend-select = [

src/lsst/cmservice/cli/commands.py

@@ -1,5 +1,6 @@
 import json
-from typing import Any, Sequence, TypeVar
+from collections.abc import Sequence
+from typing import Any, TypeVar
 
 import click
 import structlog

src/lsst/cmservice/cli/options.py

@@ -82,7 +82,7 @@ def convert(  # pylint:  disable=inconsistent-return-statements
             keyvalue_pairs = value.rstrip(";").split(";")
             result_dict = {}
             for pair in keyvalue_pairs:
-                key, values = [item.strip() for item in pair.split("=")]
+                key, values = (item.strip() for item in pair.split("="))
                 converted_values = []
                 for value_ in values.split(","):
                     value_ = value_.strip()

src/lsst/cmservice/common/bash.py

@@ -43,7 +43,7 @@ async def check_stamp_file(
     """
     if not os.path.exists(stamp_file):
         return None
-    with open(stamp_file, "rt", encoding="utf-8") as fin:
+    with open(stamp_file, encoding="utf-8") as fin:
         fields = yaml.safe_load(fin)
         status = StatusEnum[fields["status"]]
         return status
@@ -102,7 +102,7 @@ async def write_bash_script(
     except OSError:
         pass
 
-    with open(script_url, "wt", encoding="utf-8") as fout:
+    with open(script_url, "w", encoding="utf-8") as fout:
         if prepend:
             fout.write(f"{prepend}\n")
         if fake:

src/lsst/cmservice/common/utils.py

@@ -3,7 +3,7 @@
 import contextlib
 import os
 import sys
-from typing import Iterator
+from collections.abc import Iterator
 
 
 @contextlib.contextmanager

src/lsst/cmservice/db/campaign.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, Iterable, List, Optional
+from collections.abc import Iterable
+from typing import TYPE_CHECKING, Any
 
 from sqlalchemy import JSON
 from sqlalchemy.ext.asyncio import async_scoped_session
@@ -45,15 +46,15 @@ class Campaign(Base, ElementMixin):
     status: Mapped[StatusEnum] = mapped_column(default=StatusEnum.waiting)
     superseded: Mapped[bool] = mapped_column(default=False)
     handler: Mapped[str | None] = mapped_column()
-    data: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
-    child_config: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
-    collections: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
-    spec_aliases: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
-
-    spec_block_: Mapped["SpecBlock"] = relationship("SpecBlock", viewonly=True)
-    parent_: Mapped["Production"] = relationship("Production", viewonly=True)
-    s_: Mapped[List["Step"]] = relationship("Step", viewonly=True)
-    scripts_: Mapped[List["Script"]] = relationship("Script", viewonly=True)
+    data: Mapped[dict | list | None] = mapped_column(type_=JSON)
+    child_config: Mapped[dict | list | None] = mapped_column(type_=JSON)
+    collections: Mapped[dict | list | None] = mapped_column(type_=JSON)
+    spec_aliases: Mapped[dict | list | None] = mapped_column(type_=JSON)
+
+    spec_block_: Mapped[SpecBlock] = relationship("SpecBlock", viewonly=True)
+    parent_: Mapped[Production] = relationship("Production", viewonly=True)
+    s_: Mapped[list[Step]] = relationship("Step", viewonly=True)
+    scripts_: Mapped[list[Script]] = relationship("Script", viewonly=True)
 
     @hybrid_property
     def db_id(self) -> DbId:

src/lsst/cmservice/db/element.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, List
+from typing import TYPE_CHECKING, Any
 
 from sqlalchemy.ext.asyncio import async_scoped_session
 
@@ -33,7 +33,7 @@ async def get_scripts(
         script_name: str | None = None,
         remaining_only: bool = False,
         skip_superseded: bool = True,
-    ) -> List["Script"]:
+    ) -> list[Script]:
         """Return the `Script`s associated to an element
 
         Parameters
@@ -73,7 +73,7 @@ async def get_jobs(
         session: async_scoped_session,
         remaining_only: bool = False,
         skip_superseded: bool = True,
-    ) -> List["Job"]:
+    ) -> list[Job]:
         """Return the `Job`s associated to an element
 
         Parameters
@@ -128,12 +128,12 @@ async def retry_script(
         scripts = await self.get_scripts(session, script_name)
         if len(scripts) != 1:
             raise ValueError(
-                f"Expected one active script matching {script_name} for {self.fullname}, got {len(scripts)}"
+                f"Expected one active script matching {script_name} for {self.fullname}, got {len(scripts)}",
             )
         the_script = scripts[0]
         if the_script.status.value > StatusEnum.rejected.value:
             raise ValueError(
-                f"Can only retry failed/rejected scripts, {the_script.fullname} is {the_script.status.value}"
+                f"Can only retry failed/rejected scripts, {the_script.fullname} is {the_script.status.value}",
             )
         new_script = await the_script.copy_script(session)
         await the_script.update_values(session, superseded=True)
@@ -175,7 +175,7 @@ async def rescue_job(
     async def mark_job_rescued(
         self,
         session: async_scoped_session,
-    ) -> List["Job"]:
+    ) -> list[Job]:
         """Mark jobs as `rescued` once one of their siblings is `accepted`
 
         Parameters

src/lsst/cmservice/db/group.py

@@ -1,4 +1,5 @@
-from typing import TYPE_CHECKING, Any, Iterable, List, Optional
+from collections.abc import Iterable
+from typing import TYPE_CHECKING, Any
 
 from sqlalchemy import JSON
 from sqlalchemy.ext.asyncio import async_scoped_session
@@ -41,10 +42,10 @@ class Group(Base, ElementMixin):
     status: Mapped[StatusEnum] = mapped_column(default=StatusEnum.waiting)  # Status flag
     superseded: Mapped[bool] = mapped_column(default=False)  # Has this been supersede
     handler: Mapped[str | None] = mapped_column()
-    data: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
-    child_config: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
-    collections: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
-    spec_aliases: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
+    data: Mapped[dict | list | None] = mapped_column(type_=JSON)
+    child_config: Mapped[dict | list | None] = mapped_column(type_=JSON)
+    collections: Mapped[dict | list | None] = mapped_column(type_=JSON)
+    spec_aliases: Mapped[dict | list | None] = mapped_column(type_=JSON)
 
     spec_block_: Mapped["SpecBlock"] = relationship("SpecBlock", viewonly=True)
     c_: Mapped["Campaign"] = relationship(
@@ -63,8 +64,8 @@ class Group(Base, ElementMixin):
     )
 
     parent_: Mapped["Step"] = relationship("Step", viewonly=True)
-    scripts_: Mapped[List["Script"]] = relationship("Script", viewonly=True)
-    jobs_: Mapped[List["Job"]] = relationship("Job", viewonly=True)
+    scripts_: Mapped[list["Script"]] = relationship("Script", viewonly=True)
+    jobs_: Mapped[list["Job"]] = relationship("Job", viewonly=True)
 
     @hybrid_property
     def db_id(self) -> DbId:

src/lsst/cmservice/db/handler.py

@@ -65,7 +65,7 @@ def get_handler(
             with add_sys_path(Handler.plugin_dir):
                 handler_class = doImport(class_name)
             if isinstance(handler_class, types.ModuleType):
-                raise TypeError()
+                raise TypeError
             cached_handler = handler_class(spec_block_id, **kwargs)
             Handler.handler_cache[spec_block_id] = cached_handler
         return cached_handler

src/lsst/cmservice/db/job.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, List, Optional
+from typing import TYPE_CHECKING, Any
 
 from sqlalchemy import JSON
 from sqlalchemy.ext.asyncio import async_scoped_session
@@ -47,29 +47,29 @@ class Job(Base, ElementMixin):
     status: Mapped[StatusEnum] = mapped_column(default=StatusEnum.waiting)
     superseded: Mapped[bool] = mapped_column(default=False)
     handler: Mapped[str | None] = mapped_column()
-    data: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
-    child_config: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
-    collections: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
-    spec_aliases: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
-    wms_job_id: Mapped[Optional[int]] = mapped_column()
-    stamp_url: Mapped[Optional[str]] = mapped_column()
-
-    spec_block_: Mapped["SpecBlock"] = relationship("SpecBlock", viewonly=True)
-    s_: Mapped["Step"] = relationship(
+    data: Mapped[dict | list | None] = mapped_column(type_=JSON)
+    child_config: Mapped[dict | list | None] = mapped_column(type_=JSON)
+    collections: Mapped[dict | list | None] = mapped_column(type_=JSON)
+    spec_aliases: Mapped[dict | list | None] = mapped_column(type_=JSON)
+    wms_job_id: Mapped[int | None] = mapped_column()
+    stamp_url: Mapped[str | None] = mapped_column()
+
+    spec_block_: Mapped[SpecBlock] = relationship("SpecBlock", viewonly=True)
+    s_: Mapped[Step] = relationship(
         "Step",
         primaryjoin="Job.parent_id==Group.id",
         secondary="join(Group, Step)",
         secondaryjoin="Group.parent_id==Step.id",
         viewonly=True,
     )
-    c_: Mapped["Campaign"] = relationship(
+    c_: Mapped[Campaign] = relationship(
         "Campaign",
         primaryjoin="Job.parent_id==Group.id",
         secondary="join(Group, Step).join(Campaign)",
         secondaryjoin="and_(Group.parent_id==Step.id, Step.parent_id==Campaign.id) ",
         viewonly=True,
     )
-    p_: Mapped["Production"] = relationship(
+    p_: Mapped[Production] = relationship(
         "Production",
         primaryjoin="Job.parent_id==Group.id",
         secondary="join(Group, Step).join(Campaign).join(Production)",
@@ -80,18 +80,18 @@ class Job(Base, ElementMixin):
         ") ",
         viewonly=True,
     )
-    parent_: Mapped["Group"] = relationship("Group", viewonly=True)
-    scripts_: Mapped[List["Script"]] = relationship("Script", viewonly=True)
-    tasks_: Mapped[List["TaskSet"]] = relationship("TaskSet", viewonly=True)
-    products_: Mapped[List["ProductSet"]] = relationship("ProductSet", viewonly=True)
-    errors_: Mapped[List["PipetaskError"]] = relationship(
+    parent_: Mapped[Group] = relationship("Group", viewonly=True)
+    scripts_: Mapped[list[Script]] = relationship("Script", viewonly=True)
+    tasks_: Mapped[list[TaskSet]] = relationship("TaskSet", viewonly=True)
+    products_: Mapped[list[ProductSet]] = relationship("ProductSet", viewonly=True)
+    errors_: Mapped[list[PipetaskError]] = relationship(
         "PipetaskError",
         primaryjoin="Job.id==TaskSet.job_id",
         secondary="join(TaskSet, PipetaskError)",
         secondaryjoin="PipetaskError.task_id==TaskSet.id",
         viewonly=True,
     )
-    wms_reports_: Mapped[List["WmsTaskReport"]] = relationship("WmsTaskReport", viewonly=True)
+    wms_reports_: Mapped[list[WmsTaskReport]] = relationship("WmsTaskReport", viewonly=True)
 
     @hybrid_property
     def db_id(self) -> DbId:
@@ -148,4 +148,4 @@ async def copy_job(
         new_job: Job
             Newly created Job
         """
-        raise NotImplementedError()
+        raise NotImplementedError

src/lsst/cmservice/db/node.py

@@ -203,7 +203,7 @@ async def resolve_collections(
                         resolved_collections[name_].append(f1.format(**name_dict))
                     except KeyError as msg:
                         raise KeyError(
-                            f"Failed to resolve collection {name_} {f1} using: {str(name_dict)}",
+                            f"Failed to resolve collection {name_} {f1} using: {name_dict!s}",
                         ) from msg
             else:
                 try:
@@ -214,7 +214,7 @@ async def resolve_collections(
                     resolved_collections[name_] = f1.format(**name_dict)
                 except KeyError as msg:
                     raise KeyError(
-                        f"Failed to resolve collection {name_}, {f1} using: {str(name_dict)}",
+                        f"Failed to resolve collection {name_}, {f1} using: {name_dict!s}",
                     ) from msg
         return resolved_collections
 
@@ -357,7 +357,7 @@ async def get_spec_aliases(
         ret_dict = {}
         async with session.begin_nested():
             if self.level == LevelEnum.script:
-                raise NotImplementedError()
+                raise NotImplementedError
             if self.level.value > LevelEnum.campaign.value:
                 await session.refresh(self, attribute_names=["parent_"])
                 parent_data = await self.parent_.get_spec_aliases(session)
@@ -632,7 +632,7 @@ async def _clean_up_node(
         node: NodeMixin
             Node being cleaned
         """
-        raise NotImplementedError()
+        raise NotImplementedError
 
     async def process(
         self,

src/lsst/cmservice/db/pipetask_error.py

@@ -1,4 +1,4 @@
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING
 
 from sqlalchemy import JSON
 from sqlalchemy.orm import Mapped, mapped_column, relationship
@@ -26,7 +26,7 @@ class PipetaskError(Base, RowMixin):
     task_id: Mapped[int] = mapped_column(ForeignKey("task_set.id", ondelete="CASCADE"), index=True)
     quanta: Mapped[str] = mapped_column()
     diagnostic_message: Mapped[str] = mapped_column()
-    data_id: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
+    data_id: Mapped[dict | list | None] = mapped_column(type_=JSON)
 
     job_: Mapped["Job"] = relationship(
         "Job",

src/lsst/cmservice/db/pipetask_error_type.py

@@ -1,5 +1,5 @@
 import re
-from typing import TYPE_CHECKING, List
+from typing import TYPE_CHECKING
 
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 
@@ -23,7 +23,7 @@ class PipetaskErrorType(Base, RowMixin):
     task_name: Mapped[str] = mapped_column()
     diagnostic_message: Mapped[str] = mapped_column(unique=True)
 
-    errors_: Mapped[List["PipetaskError"]] = relationship("PipetaskError", viewonly=True)
+    errors_: Mapped[list["PipetaskError"]] = relationship("PipetaskError", viewonly=True)
 
     def __repr__(self) -> str:
         s = f"Id={self.id}\n"

src/lsst/cmservice/db/production.py

@@ -1,4 +1,5 @@
-from typing import TYPE_CHECKING, Iterable, List
+from collections.abc import Iterable
+from typing import TYPE_CHECKING
 
 from sqlalchemy.ext.asyncio import async_scoped_session
 from sqlalchemy.ext.hybrid import hybrid_property
@@ -21,7 +22,7 @@ class Production(Base, RowMixin):
     id: Mapped[int] = mapped_column(primary_key=True)
     name: Mapped[str] = mapped_column(index=True, unique=True)
 
-    c_: Mapped[List["Campaign"]] = relationship("Campaign", viewonly=True)
+    c_: Mapped[list["Campaign"]] = relationship("Campaign", viewonly=True)
 
     @hybrid_property
     def db_id(self) -> DbId:

src/lsst/cmservice/db/queue.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from datetime import datetime, timedelta
-from typing import TYPE_CHECKING, Any, Optional
+from typing import Any
 
 import pause
 from sqlalchemy import JSON, DateTime
@@ -20,9 +20,6 @@
 from .node import NodeMixin
 from .step import Step
 
-if TYPE_CHECKING:
-    pass
-
 
 class Queue(Base, NodeMixin):
     """Database table to implement processing queue"""
@@ -34,7 +31,7 @@ class Queue(Base, NodeMixin):
     time_updated: Mapped[datetime] = mapped_column(type_=DateTime)
     time_finished: Mapped[datetime | None] = mapped_column(type_=DateTime, default=None)
     interval: Mapped[float] = mapped_column(default=300.0)
-    options: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
+    options: Mapped[dict | list | None] = mapped_column(type_=JSON)
 
     element_level: Mapped[LevelEnum] = mapped_column()
     element_id: Mapped[int] = mapped_column()
@@ -43,10 +40,10 @@ class Queue(Base, NodeMixin):
     g_id: Mapped[int | None] = mapped_column(ForeignKey("group.id", ondelete="CASCADE"), index=True)
     j_id: Mapped[int | None] = mapped_column(ForeignKey("job.id", ondelete="CASCADE"), index=True)
 
-    c_: Mapped["Campaign"] = relationship("Campaign", viewonly=True)
-    s_: Mapped["Step"] = relationship("Step", viewonly=True)
-    g_: Mapped["Group"] = relationship("Group", viewonly=True)
-    j_: Mapped["Job"] = relationship("Job", viewonly=True)
+    c_: Mapped[Campaign] = relationship("Campaign", viewonly=True)
+    s_: Mapped[Step] = relationship("Step", viewonly=True)
+    g_: Mapped[Group] = relationship("Group", viewonly=True)
+    j_: Mapped[Job] = relationship("Job", viewonly=True)
 
     @hybrid_property
     def element_db_id(self) -> DbId:

src/lsst/cmservice/db/row.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, Sequence, TypeVar
+from collections.abc import Sequence
+from typing import TYPE_CHECKING, Any, TypeVar
 
 from fastapi import HTTPException
 from sqlalchemy import select