Ruff autofixes + enable additional checks in pre-commit

pyproject.toml

@@ -94,6 +94,9 @@ select = [
     "F",  # pyflakes
     "N",  # pep8-naming
     "W",  # pycodestyle
+    "COM", # pyflakes-commas
+    "FBT", # flake8-boolean-trap
+    "UP", # pyupgrade
 ]
 target-version = "py311"
 extend-select = [

src/lsst/cmservice/cli/commands.py

@@ -1,5 +1,6 @@
 import json
-from typing import Any, Sequence, TypeVar
+from collections.abc import Sequence
+from typing import Any, TypeVar
 
 import click
 import structlog

src/lsst/cmservice/cli/options.py

@@ -82,7 +82,7 @@ def convert(  # pylint:  disable=inconsistent-return-statements
             keyvalue_pairs = value.rstrip(";").split(";")
             result_dict = {}
             for pair in keyvalue_pairs:
-                key, values = [item.strip() for item in pair.split("=")]
+                key, values = (item.strip() for item in pair.split("="))
                 converted_values = []
                 for value_ in values.split(","):
                     value_ = value_.strip()

src/lsst/cmservice/client.py

@@ -165,6 +165,7 @@ def get_scripts(
         self,
         fullname: str,
         script_name: str,
+        *,
         remaining_only: bool = False,
         skip_superseded: bool = True,
     ) -> list[models.Script]:
@@ -184,6 +185,7 @@ def get_scripts(
     def get_jobs(
         self,
         fullname: str,
+        *,
         remaining_only: bool = False,
         skip_superseded: bool = True,
     ) -> list[models.Job]:

src/lsst/cmservice/common/bash.py

@@ -43,10 +43,9 @@ async def check_stamp_file(
     """
     if not os.path.exists(stamp_file):
         return None
-    with open(stamp_file, "rt", encoding="utf-8") as fin:
+    with open(stamp_file, encoding="utf-8") as fin:
         fields = yaml.safe_load(fin)
-        status = StatusEnum[fields["status"]]
-        return status
+        return StatusEnum[fields["status"]]
 
 
 async def write_bash_script(
@@ -102,7 +101,7 @@ async def write_bash_script(
     except OSError:
         pass
 
-    with open(script_url, "wt", encoding="utf-8") as fout:
+    with open(script_url, "w", encoding="utf-8") as fout:
         if prepend:
             fout.write(f"{prepend}\n")
         if fake:

src/lsst/cmservice/common/slurm.py

@@ -58,8 +58,7 @@ async def submit_slurm_job(
         ) as sbatch:
             assert sbatch.stdout
             line = sbatch.stdout.read().decode().strip()
-            job_id = line.split("|")[0]
-        return job_id
+            return line.split("|")[0]
     except TypeError as msg:
         raise TypeError(f"Bad slurm submit from {script_url}") from msg
 
@@ -85,12 +84,9 @@ async def check_slurm_job(
         assert sacct.stdout
         lines = sacct.stdout.read().decode().split("\n")
         if len(lines) < 2:
-            status = slurm_status_map["PENDING"]
-            return status
+            return slurm_status_map["PENDING"]
         tokens = lines[1].split("|")
         if len(tokens) < 2:
-            status = slurm_status_map["PENDING"]
-            return status
+            return slurm_status_map["PENDING"]
         slurm_status = tokens[1]
-        status = slurm_status_map[slurm_status]
-        return status
+        return slurm_status_map[slurm_status]

src/lsst/cmservice/common/utils.py

@@ -3,7 +3,7 @@
 import contextlib
 import os
 import sys
-from typing import Iterator
+from collections.abc import Iterator
 
 
 @contextlib.contextmanager

src/lsst/cmservice/db/campaign.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, Iterable, List, Optional
+from collections.abc import Iterable
+from typing import TYPE_CHECKING, Any
 
 from sqlalchemy import JSON
 from sqlalchemy.ext.asyncio import async_scoped_session
@@ -45,15 +46,15 @@ class Campaign(Base, ElementMixin):
     status: Mapped[StatusEnum] = mapped_column(default=StatusEnum.waiting)
     superseded: Mapped[bool] = mapped_column(default=False)
     handler: Mapped[str | None] = mapped_column()
-    data: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
-    child_config: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
-    collections: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
-    spec_aliases: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
-
-    spec_block_: Mapped["SpecBlock"] = relationship("SpecBlock", viewonly=True)
-    parent_: Mapped["Production"] = relationship("Production", viewonly=True)
-    s_: Mapped[List["Step"]] = relationship("Step", viewonly=True)
-    scripts_: Mapped[List["Script"]] = relationship("Script", viewonly=True)
+    data: Mapped[dict | list | None] = mapped_column(type_=JSON)
+    child_config: Mapped[dict | list | None] = mapped_column(type_=JSON)
+    collections: Mapped[dict | list | None] = mapped_column(type_=JSON)
+    spec_aliases: Mapped[dict | list | None] = mapped_column(type_=JSON)
+
+    spec_block_: Mapped[SpecBlock] = relationship("SpecBlock", viewonly=True)
+    parent_: Mapped[Production] = relationship("Production", viewonly=True)
+    s_: Mapped[list[Step]] = relationship("Step", viewonly=True)
+    scripts_: Mapped[list[Script]] = relationship("Script", viewonly=True)
 
     @hybrid_property
     def db_id(self) -> DbId:

src/lsst/cmservice/db/element.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, List
+from typing import TYPE_CHECKING, Any
 
 from sqlalchemy.ext.asyncio import async_scoped_session
 
@@ -31,9 +31,10 @@ async def get_scripts(
         self,
         session: async_scoped_session,
         script_name: str | None = None,
+        *,
         remaining_only: bool = False,
         skip_superseded: bool = True,
-    ) -> List["Script"]:
+    ) -> list[Script]:
         """Return the `Script`s associated to an element
 
         Parameters
@@ -71,9 +72,10 @@ async def get_scripts(
     async def get_jobs(
         self,
         session: async_scoped_session,
+        *,
         remaining_only: bool = False,
         skip_superseded: bool = True,
-    ) -> List["Job"]:
+    ) -> list[Job]:
         """Return the `Job`s associated to an element
 
         Parameters
@@ -128,12 +130,12 @@ async def retry_script(
         scripts = await self.get_scripts(session, script_name)
         if len(scripts) != 1:
             raise ValueError(
-                f"Expected one active script matching {script_name} for {self.fullname}, got {len(scripts)}"
+                f"Expected one active script matching {script_name} for {self.fullname}, got {len(scripts)}",
             )
         the_script = scripts[0]
         if the_script.status.value > StatusEnum.rejected.value:
             raise ValueError(
-                f"Can only retry failed/rejected scripts, {the_script.fullname} is {the_script.status.value}"
+                f"Can only retry failed/rejected scripts, {the_script.fullname} is {the_script.status.value}",
             )
         new_script = await the_script.copy_script(session)
         await the_script.update_values(session, superseded=True)
@@ -175,7 +177,7 @@ async def rescue_job(
     async def mark_job_rescued(
         self,
         session: async_scoped_session,
-    ) -> List["Job"]:
+    ) -> list[Job]:
         """Mark jobs as `rescued` once one of their siblings is `accepted`
 
         Parameters

src/lsst/cmservice/db/group.py

@@ -1,4 +1,5 @@
-from typing import TYPE_CHECKING, Any, Iterable, List, Optional
+from collections.abc import Iterable
+from typing import TYPE_CHECKING, Any
 
 from sqlalchemy import JSON
 from sqlalchemy.ext.asyncio import async_scoped_session
@@ -41,10 +42,10 @@ class Group(Base, ElementMixin):
     status: Mapped[StatusEnum] = mapped_column(default=StatusEnum.waiting)  # Status flag
     superseded: Mapped[bool] = mapped_column(default=False)  # Has this been supersede
     handler: Mapped[str | None] = mapped_column()
-    data: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
-    child_config: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
-    collections: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
-    spec_aliases: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
+    data: Mapped[dict | list | None] = mapped_column(type_=JSON)
+    child_config: Mapped[dict | list | None] = mapped_column(type_=JSON)
+    collections: Mapped[dict | list | None] = mapped_column(type_=JSON)
+    spec_aliases: Mapped[dict | list | None] = mapped_column(type_=JSON)
 
     spec_block_: Mapped["SpecBlock"] = relationship("SpecBlock", viewonly=True)
     c_: Mapped["Campaign"] = relationship(
@@ -63,8 +64,8 @@ class Group(Base, ElementMixin):
     )
 
     parent_: Mapped["Step"] = relationship("Step", viewonly=True)
-    scripts_: Mapped[List["Script"]] = relationship("Script", viewonly=True)
-    jobs_: Mapped[List["Job"]] = relationship("Job", viewonly=True)
+    scripts_: Mapped[list["Script"]] = relationship("Script", viewonly=True)
+    jobs_: Mapped[list["Job"]] = relationship("Job", viewonly=True)
 
     @hybrid_property
     def db_id(self) -> DbId:

src/lsst/cmservice/db/handler.py

@@ -65,7 +65,7 @@ def get_handler(
             with add_sys_path(Handler.plugin_dir):
                 handler_class = doImport(class_name)
             if isinstance(handler_class, types.ModuleType):
-                raise TypeError()
+                raise TypeError
             cached_handler = handler_class(spec_block_id, **kwargs)
             Handler.handler_cache[spec_block_id] = cached_handler
         return cached_handler

src/lsst/cmservice/db/job.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, List, Optional
+from typing import TYPE_CHECKING, Any
 
 from sqlalchemy import JSON
 from sqlalchemy.ext.asyncio import async_scoped_session
@@ -47,29 +47,29 @@ class Job(Base, ElementMixin):
     status: Mapped[StatusEnum] = mapped_column(default=StatusEnum.waiting)
     superseded: Mapped[bool] = mapped_column(default=False)
     handler: Mapped[str | None] = mapped_column()
-    data: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
-    child_config: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
-    collections: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
-    spec_aliases: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
-    wms_job_id: Mapped[Optional[int]] = mapped_column()
-    stamp_url: Mapped[Optional[str]] = mapped_column()
-
-    spec_block_: Mapped["SpecBlock"] = relationship("SpecBlock", viewonly=True)
-    s_: Mapped["Step"] = relationship(
+    data: Mapped[dict | list | None] = mapped_column(type_=JSON)
+    child_config: Mapped[dict | list | None] = mapped_column(type_=JSON)
+    collections: Mapped[dict | list | None] = mapped_column(type_=JSON)
+    spec_aliases: Mapped[dict | list | None] = mapped_column(type_=JSON)
+    wms_job_id: Mapped[int | None] = mapped_column()
+    stamp_url: Mapped[str | None] = mapped_column()
+
+    spec_block_: Mapped[SpecBlock] = relationship("SpecBlock", viewonly=True)
+    s_: Mapped[Step] = relationship(
         "Step",
         primaryjoin="Job.parent_id==Group.id",
         secondary="join(Group, Step)",
         secondaryjoin="Group.parent_id==Step.id",
         viewonly=True,
     )
-    c_: Mapped["Campaign"] = relationship(
+    c_: Mapped[Campaign] = relationship(
         "Campaign",
         primaryjoin="Job.parent_id==Group.id",
         secondary="join(Group, Step).join(Campaign)",
         secondaryjoin="and_(Group.parent_id==Step.id, Step.parent_id==Campaign.id) ",
         viewonly=True,
     )
-    p_: Mapped["Production"] = relationship(
+    p_: Mapped[Production] = relationship(
         "Production",
         primaryjoin="Job.parent_id==Group.id",
         secondary="join(Group, Step).join(Campaign).join(Production)",
@@ -80,18 +80,18 @@ class Job(Base, ElementMixin):
         ") ",
         viewonly=True,
     )
-    parent_: Mapped["Group"] = relationship("Group", viewonly=True)
-    scripts_: Mapped[List["Script"]] = relationship("Script", viewonly=True)
-    tasks_: Mapped[List["TaskSet"]] = relationship("TaskSet", viewonly=True)
-    products_: Mapped[List["ProductSet"]] = relationship("ProductSet", viewonly=True)
-    errors_: Mapped[List["PipetaskError"]] = relationship(
+    parent_: Mapped[Group] = relationship("Group", viewonly=True)
+    scripts_: Mapped[list[Script]] = relationship("Script", viewonly=True)
+    tasks_: Mapped[list[TaskSet]] = relationship("TaskSet", viewonly=True)
+    products_: Mapped[list[ProductSet]] = relationship("ProductSet", viewonly=True)
+    errors_: Mapped[list[PipetaskError]] = relationship(
         "PipetaskError",
         primaryjoin="Job.id==TaskSet.job_id",
         secondary="join(TaskSet, PipetaskError)",
         secondaryjoin="PipetaskError.task_id==TaskSet.id",
         viewonly=True,
     )
-    wms_reports_: Mapped[List["WmsTaskReport"]] = relationship("WmsTaskReport", viewonly=True)
+    wms_reports_: Mapped[list[WmsTaskReport]] = relationship("WmsTaskReport", viewonly=True)
 
     @hybrid_property
     def db_id(self) -> DbId:
@@ -114,7 +114,7 @@ async def get_create_kwargs(
         spec_block = await SpecBlock.get_row_by_fullname(session, spec_block_name)
         parent = await Group.get_row_by_fullname(session, parent_name)
 
-        ret_dict = {
+        return {
             "spec_block_id": spec_block.id,
             "parent_id": parent.id,
             "name": name,
@@ -126,8 +126,6 @@ async def get_create_kwargs(
             "spec_aliases": kwargs.get("spec_aliases", {}),
         }
 
-        return ret_dict
-
     async def copy_job(
         self,
         session: async_scoped_session,
@@ -148,4 +146,4 @@ async def copy_job(
         new_job: Job
             Newly created Job
         """
-        raise NotImplementedError()
+        raise NotImplementedError

src/lsst/cmservice/db/node.py

@@ -124,11 +124,10 @@ async def get_handler(
         else:
             spec_block = await self.get_spec_block(session)
             handler_class = spec_block.handler
-        handler = Handler.get_handler(
+        return Handler.get_handler(
             self.spec_block_id,
             handler_class,
         )
-        return handler
 
     def _split_fullname(self, fullname: str) -> dict:
         """Split a fullname into named fields
@@ -203,7 +202,7 @@ async def resolve_collections(
                         resolved_collections[name_].append(f1.format(**name_dict))
                     except KeyError as msg:
                         raise KeyError(
-                            f"Failed to resolve collection {name_} {f1} using: {str(name_dict)}",
+                            f"Failed to resolve collection {name_} {f1} using: {name_dict!s}",
                         ) from msg
             else:
                 try:
@@ -214,7 +213,7 @@ async def resolve_collections(
                     resolved_collections[name_] = f1.format(**name_dict)
                 except KeyError as msg:
                     raise KeyError(
-                        f"Failed to resolve collection {name_}, {f1} using: {str(name_dict)}",
+                        f"Failed to resolve collection {name_}, {f1} using: {name_dict!s}",
                     ) from msg
         return resolved_collections
 
@@ -357,7 +356,7 @@ async def get_spec_aliases(
         ret_dict = {}
         async with session.begin_nested():
             if self.level == LevelEnum.script:
-                raise NotImplementedError()
+                raise NotImplementedError
             if self.level.value > LevelEnum.campaign.value:
                 await session.refresh(self, attribute_names=["parent_"])
                 parent_data = await self.parent_.get_spec_aliases(session)
@@ -632,7 +631,7 @@ async def _clean_up_node(
         node: NodeMixin
             Node being cleaned
         """
-        raise NotImplementedError()
+        raise NotImplementedError
 
     async def process(
         self,
@@ -655,8 +654,7 @@ async def process(
             The status of the processing
         """
         handler = await self.get_handler(session)
-        status = await handler.process(session, self, **kwargs)
-        return status
+        return await handler.process(session, self, **kwargs)
 
     async def run_check(
         self,
@@ -679,5 +677,4 @@ async def run_check(
             The status of the processing
         """
         handler = await self.get_handler(session)
-        status = await handler.run_check(session, self, **kwargs)
-        return status
+        return await handler.run_check(session, self, **kwargs)