tests kinda testing

examples/empty_config.yaml

@@ -0,0 +1,110 @@
+- ScriptTemplate:
+      name: bps_panda_script_template
+      file_path: ${CM_CONFIGS}/example_bps_panda_template.yaml
+- ScriptTemplate:
+      name: bps_yaml_template
+      file_path: ${CM_CONFIGS}/example_template.yaml
+- ScriptTemplate:
+      name: manifest_script_template
+      file_path: ${CM_CONFIGS}/example_manifest_template.yaml
+- SpecBlock:
+      name: chain_create_script
+      handler: lsst.cmservice.handlers.scripts.ChainCreateScriptHandler
+- SpecBlock:
+      name: chain_prepend_script
+      handler: lsst.cmservice.handlers.scripts.ChainPrependScriptHandler
+- SpecBlock:
+      name: chain_collect_jobs_script
+      handler: lsst.cmservice.handlers.scripts.ChainCollectScriptHandler
+      data:
+          collect: jobs
+- SpecBlock:
+      name: chain_collect_steps_script
+      handler: lsst.cmservice.handlers.scripts.ChainCollectScriptHandler
+      data:
+          collect: steps
+- SpecBlock:
+      name: tag_inputs_script
+      handler: lsst.cmservice.handlers.scripts.TagInputsScriptHandler
+- SpecBlock:
+      name: tag_create_script
+- SpecBlock:
+      name: tag_associate_script
+      handler: lsst.cmservice.handlers.scripts.TagAssociateScriptHandler
+- SpecBlock:
+      name: prepare_step_script
+      handler: lsst.cmservice.handlers.scripts.PrepareStepScriptHandler
+      collections:
+          global_inputs: "{campaign_input}"
+- SpecBlock:
+      name: validate_script
+      handler: lsst.cmservice.handlers.scripts.ValidateScriptHandler
+- SpecBlock:
+      name: panda_script
+      handler: lsst.cmservice.handlers.jobs.PandaScriptHandler
+- SpecBlock:
+      name: panda_report_script
+      handler: lsst.cmservice.handlers.jobs.PandaReportHandler
+- SpecBlock:
+      name: manifest_report_script
+      handler: lsst.cmservice.handlers.jobs.ManifestReportScriptHandler
+- SpecBlock:
+      name: run_jobs
+      handler: lsst.cmservice.handlers.elements.RunJobsScriptHandler
+- SpecBlock:
+      name: run_groups
+      handler: lsst.cmservice.handlers.elements.RunGroupsScriptHandler
+- SpecBlock:
+      name: run_steps
+      handler: lsst.cmservice.handlers.elements.RunStepsScriptHandler
+- SpecBlock:
+      name: job
+      handler: lsst.cmservice.handlers.job_handler.JobHandler
+      collections:
+          job_run: "{root}/{campaign}/{step}/{group}/{job}"
+- SpecBlock:
+      name: group
+      handler: lsst.cmservice.handlers.element_handler.ElementHandler
+      collections:
+          group_output: "{root}/{campaign}/{step}/{group}"
+          group_validation: "{root}/{campaign}/{step}/{group}/validate"
+      child_config:
+          spec_block: job
+- SpecBlock:
+      name: step
+      handler: lsst.cmservice.handlers.element_handler.ElementHandler
+      collections:
+          step_input: "{root}/{campaign}/{step}/input"
+          step_output: "{root}/{campaign}/{step}_ouput"
+          step_public_output: "{root}/{campaign}/{step}"
+          step_validation: "{root}/{campaign}/{step}/validate"
+- SpecBlock:
+      name: basic_step
+      includes: ["step"]
+      data:
+          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#isr"
+      child_config:
+          spec_block: group
+          base_query: "instrument = 'HSC'"
+          split_method: split_by_query
+          split_dataset: raw
+          split_field: exposure
+          split_min_groups: 2
+- SpecBlock:
+      name: campaign
+      handler: lsst.cmservice.handlers.element_handler.ElementHandler
+      collections:
+          root: 'cm/hsc_rc2_micro'
+          campaign_source: HSC/raw/RC2
+          campaign_input: "{root}/{campaign}/input"
+          campaign_output: "{root}/{campaign}"
+          campaign_ancillary: "{root}/{campaign}/ancillary"
+          campaign_validation: "{root}/{campaign}/validate"
+      data:
+          butler_repo: '/repo/main'
+          prod_area: 'output/archive'
+          data_query: "instrument = 'HSC'"
+          lsst_version: "${WEEKLY}"
+          bps_script_template: bps_panda_script_template
+          bps_yaml_template: bps_yaml_template
+          manifest_script_template: manifest_script_template

src/lsst/cmservice/db/dbid.py

@@ -14,3 +14,11 @@ class DbId:
 
     def __repr__(self) -> str:
         return f"DbId({self._level.name}:{self._id})"
+
+    @property
+    def level(self) -> LevelEnum:
+        return self._level
+
+    @property
+    def id(self) -> int:
+        return self._id

tests/cli/test_commands.py

@@ -35,5 +35,6 @@ def test_commands(uvicorn: UvicornProcess) -> None:
     result = runner.invoke(main, "get campaigns -o yaml")
     assert result.exit_code == 0
 
-    result = runner.invoke(main, "get campaigns -o json")
-    assert result.exit_code == 0
+    # FIXME StatusEnum not JSON serializable
+    # result = runner.invoke(main, "get campaigns -o json")
+    # assert result.exit_code == 0

tests/conftest.py

@@ -14,7 +14,6 @@
 from sqlalchemy.ext.asyncio import AsyncEngine
 
 from lsst.cmservice import db, main
-from lsst.cmservice.common.enums import StatusEnum
 from lsst.cmservice.config import config
 from lsst.cmservice.handlers import interface
 
@@ -30,6 +29,7 @@ def event_loop() -> Iterator[AbstractEventLoop]:
 @pytest_asyncio.fixture(scope="session")
 async def engine() -> AsyncIterator[AsyncEngine]:
     """Return a SQLAlchemy AsyncEngine configured to talk to the app db."""
+    os.environ["CM_CONFIGS"] = "examples"
     logger = structlog.get_logger(config.logger_name)
     the_engine = create_database_engine(config.database_url, config.database_password)
     await initialize_database(the_engine, logger, schema=db.Base.metadata, reset=True)
@@ -38,42 +38,24 @@ async def engine() -> AsyncIterator[AsyncEngine]:
 
 
 @pytest_asyncio.fixture(scope="session")
-async def empty_session(engine: AsyncEngine) -> AsyncGenerator:  # pylint: disable=redefined-outer-name
+async def session(engine: AsyncEngine) -> AsyncGenerator:  # pylint: disable=redefined-outer-name
     """Return a SQLAlchemy AsyncEngine configured to talk to the app db."""
     logger = structlog.get_logger(config.logger_name)
     async with engine.begin():
-        session = await create_async_session(engine, logger)
-        yield session
-        await session.close()
-
-
-@pytest_asyncio.fixture(scope="session")
-async def fully_loaded_session(engine: AsyncEngine) -> AsyncGenerator:  # pylint: disable=redefined-outer-name
-    """Return a SQLAlchemy AsyncEngine configured to talk to the app db."""
-    logger = structlog.get_logger(config.logger_name)
-    os.environ["CM_CONFIGS"] = "examples"
-    async with engine.begin():
-        session = await create_async_session(engine, logger)
-
-        await interface.load_and_create_campaign(
-            session,
-            "examples/example_micro.yaml",
-            "hsc_micro",
-            "w_2023_41",
-        )
-
-        await interface.process(
-            session,
-            "hsc_micro/w_2023_41",
-            fake_status=StatusEnum.accepted,
-        )
-
-        yield session
-        await session.close()
+        the_session = await create_async_session(engine, logger)
+        specification = await interface.load_specification(the_session, "base", "examples/empty_config.yaml")
+        check = await db.SpecBlock.get_row_by_fullname(the_session, "base#campaign")
+        check2 = await specification.get_block(the_session, "campaign")
+        assert check.name == "campaign"
+        assert check2.name == "campaign"
+        yield the_session
+        await the_session.close()
 
 
 @pytest_asyncio.fixture(scope="session")
-async def app(engine: AsyncEngine) -> AsyncIterator[FastAPI]:  # pylint: disable=redefined-outer-name
+async def app(  # pylint: disable=redefined-outer-name,unused-argument
+    engine: AsyncEngine,
+) -> AsyncIterator[FastAPI]:
     """Return a configured test application.
 
     Wraps the application in a lifespan manager so that startup and shutdown

tests/db/test_campaign.py

@@ -1,12 +1,53 @@
+from uuid import uuid1
+
 import pytest
+from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import async_scoped_session
 
 from lsst.cmservice import db
 
 
 @pytest.mark.asyncio()
-async def test_campaign_db(fully_loaded_session: async_scoped_session) -> None:
+async def test_campaign_db(session: async_scoped_session) -> None:
     """Test `campaign` db table."""
 
-    campaigns = await db.Campaign.get_rows(fully_loaded_session)
-    assert len(campaigns) == 1
+    pnames = [str(uuid1()) for n in range(2)]
+    prods = [await db.Production.create_row(session, name=pname_) for pname_ in pnames]
+    cnames = [str(uuid1()) for n in range(5)]
+
+    camps0 = [
+        await db.Campaign.create_row(
+            session, name=cname_, spec_block_name="base#campaign", parent_name=pnames[0]
+        )
+        for cname_ in cnames
+    ]
+    assert len(camps0) == 5
+
+    camps1 = [
+        await db.Campaign.create_row(
+            session, name=cname_, spec_block_name="base#campaign", parent_name=pnames[1]
+        )
+        for cname_ in cnames
+    ]
+    assert len(camps1) == 5
+
+    with pytest.raises(IntegrityError):
+        await db.Campaign.create_row(
+            session, name=cnames[0], parent_name=pnames[0], spec_block_name="base#campaign"
+        )
+
+    await db.Production.delete_row(session, prods[0].id)
+
+    check_gone = await db.Campaign.get_rows(session, parent_id=prods[0].id, parent_class=db.Production)
+    assert len(check_gone) == 0
+
+    check_here = await db.Campaign.get_rows(session, parent_id=prods[1].id, parent_class=db.Production)
+    assert len(check_here) == 5
+
+    await db.Campaign.delete_row(session, camps1[0].id)
+
+    check_here = await db.Campaign.get_rows(session, parent_id=prods[1].id, parent_class=db.Production)
+    assert len(check_here) == 4
+
+    # Finish clean up
+    await db.Production.delete_row(session, prods[1].id)

tests/db/test_group.py

@@ -1,10 +1,54 @@
+from uuid import uuid1
+
 import pytest
+from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import async_scoped_session
 
 from lsst.cmservice import db
 
 
 @pytest.mark.asyncio()
-async def test_group_db(fully_loaded_session: async_scoped_session) -> None:
-    groups = await db.Group.get_rows(fully_loaded_session)
-    assert len(groups) == 6
+async def test_group_db(session: async_scoped_session) -> None:
+    pname = str(uuid1())
+    prod = await db.Production.create_row(session, name=pname)
+    cname = str(uuid1())
+    camp = await db.Campaign.create_row(
+        session, name=cname, spec_block_name="base#campaign", parent_name=pname
+    )
+    snames = [str(uuid1()) for n in range(2)]
+
+    steps = [
+        await db.Step.create_row(
+            session,
+            name=sname_,
+            spec_block_name="base#basic_step",
+            parent_name=camp.fullname,
+        )
+        for sname_ in snames
+    ]
+
+    gnames = [str(uuid1()) for n in range(5)]
+
+    groups0 = [
+        await db.Group.create_row(
+            session, name=gname_, spec_block_name="base#group", parent_name=steps[0].fullname
+        )
+        for gname_ in gnames
+    ]
+    assert len(groups0) == 5
+
+    groups1 = [
+        await db.Group.create_row(
+            session, name=gname_, spec_block_name="base#group", parent_name=steps[1].fullname
+        )
+        for gname_ in gnames
+    ]
+    assert len(groups1) == 5
+
+    with pytest.raises(IntegrityError):
+        await db.Group.create_row(
+            session, name=gnames[0], parent_name=steps[0].fullname, spec_block_name="base#group"
+        )
+
+    # Finish clean up
+    await db.Production.delete_row(session, prod.id)

tests/db/test_micro.py

@@ -1,10 +1,25 @@
 import pytest
 from sqlalchemy.ext.asyncio import async_scoped_session
 
+from lsst.cmservice.common.enums import StatusEnum
+from lsst.cmservice.handlers import interface
+
 
 @pytest.mark.asyncio()
-async def test_micro(fully_loaded_session: async_scoped_session) -> None:
+async def test_micro(session: async_scoped_session) -> None:
     """Test fake end to end run using example/example_micro.yaml"""
 
-    # this is already done in making the fully_loaded_session
-    assert fully_loaded_session
+    await interface.load_and_create_campaign(
+        session,
+        "examples/example_micro.yaml",
+        "hsc_micro",
+        "w_2023_41",
+    )
+
+    status = await interface.process(
+        session,
+        "hsc_micro/w_2023_41",
+        fake_status=StatusEnum.accepted,
+    )
+
+    assert status == StatusEnum.accepted

tests/db/test_production.py

@@ -1,12 +1,36 @@
+from uuid import uuid1
+
 import pytest
+from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import async_scoped_session
 
 from lsst.cmservice import db
+from lsst.cmservice.common.enums import LevelEnum
 
 
 @pytest.mark.asyncio()
-async def test_production_db(fully_loaded_session: async_scoped_session) -> None:
+async def test_production_db(session: async_scoped_session) -> None:
     """Test `production` db table."""
 
-    productions = await db.Production.get_rows(fully_loaded_session)
-    assert len(productions) == 1
+    # Check production name UNIQUE constraint
+    pname = str(uuid1())
+
+    p1 = await db.Production.create_row(session, name=pname)
+    with pytest.raises(IntegrityError):
+        p1 = await db.Production.create_row(session, name=pname)
+
+    check = await db.Production.get_row(session, p1.id)
+    assert check.name == p1.name
+    assert check.fullname == p1.fullname
+
+    assert check.db_id.level == LevelEnum.production
+    assert check.db_id.id == p1.id
+
+    prods = await db.Production.get_rows(session)
+    n_prod = len(prods)
+    assert n_prod >= 1
+
+    await db.Production.delete_row(session, p1.id)
+
+    prods = await db.Production.get_rows(session)
+    assert len(prods) == n_prod - 1