Skip to content

Commit

Permalink
Connect to microservice when looking for existing raws.
Browse files Browse the repository at this point in the history 
The microservice maps metadata to filenames without requiring that the
latter be a deterministic function of the former, which for Rubin
instruments it isn't.
  • Loading branch information
@kfindeisen
kfindeisen committed Aug 22, 2024
1 parent 8f23876 commit db85f4b
Show file tree
Hide file tree
Showing 2 changed files with 146 additions and 12 deletions.
78 changes: 69 additions & 9 deletions python/activator/raw.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@
import os
import re
import time
import urllib.parse

import requests

from lsst.obs.lsst import LsstCam, LsstComCam, LsstComCamSim
from lsst.obs.lsst.translators.lsst import LsstBaseTranslator
Expand Down Expand Up @@ -210,29 +213,86 @@ def get_prefix_from_snap(
----------
microservice : `str`
The URI of an optional microservice to assist the search.
instrument: `str`
instrument : `str`
The name of the instrument taking the image.
group: `str`
group : `str`
The group id from the visit, associating the snaps making up the visit.
detector: `int`
detector : `int`
The integer detector id for the image being sought.
snap: `int`
snap : `int`
The snap number within the group for the visit.
Returns
-------
prefix: `str` or None
prefix : `str` or `None`
The prefix to a path to the corresponding raw image object. If it
can be calculated, then the prefix may be the entire path. If no
prefix can be calculated, None is returned.
prefix can be calculated, `None` is returned.
"""

if instrument not in _LSST_CAMERA_LIST:
if microservice:
return _query_microservice(microservice=microservice,
instrument=instrument,
group=group,
detector=detector,
snap=snap,
)
elif instrument not in _LSST_CAMERA_LIST:
return f"{instrument}/{detector}/{group}/{snap}/"
# TODO DM-39022: use a microservice to determine paths for LSST cameras.
return None


def _query_microservice(
microservice: str, instrument: str, group: str, detector: int, snap: int
) -> str | None:
"""Look up a raw image's location from the raw image microservice.
Parameters
----------
microservice : `str`
The URI of the microservice to query.
instrument : `str`
The name of the instrument taking the image.
group : `str`
The group id from the visit, associating the snaps making up the visit.
detector : `int`
The integer detector id for the image being sought.
snap : `int`
The snap number within the group for the visit.
Returns
-------
key : `str` or `None`
The raw's object key within its bucket, or `None` if no image was found.
Raises
------
RuntimeError
Raised if this function could not connect to the microservice, or if the
microservice encountered an error.
"""
detector_name = _DETECTOR_FROM_INT[instrument][detector]
uri = f"{microservice}/{instrument}/{group}/{snap}/{detector_name}"
try:
response = requests.get(uri, timeout=1.0)
response.raise_for_status()
unpacked = response.json()
except requests.Timeout as e:
raise RuntimeError("Timed out connecting to raw microservice.") from e
except requests.RequestException as e:
raise RuntimeError("Could not query raw microservice.") from e

if unpacked["error"]:
raise RuntimeError("Raw microservice had an unidentified error.")
if unpacked["present"]:
# Need to return just the key, without the bucket
path = urllib.parse.urlparse(unpacked["uri"], allow_fragments=False).path
# TODO: validate output?
# Valid key does not start with a /
return path.lstrip("/")
else:
return None


def get_exp_id_from_oid(oid: str) -> int:
"""Calculate an exposure id from an image object's pathname.
Expand Down
80 changes: 77 additions & 3 deletions tests/test_raw.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,16 +130,88 @@ def test_writeread(self):
self.assertEqual(parsed['instrument'], str(self.instrument))
self.assertEqual(get_exp_id_from_oid(path), self.exposure)

def test_get_prefix(self):
"""Test that get_prefix_from_snap returns None for now."""
prefix = get_prefix_from_snap("", self.instrument, self.group, self.detector, self.snap)
def test_get_prefix_present(self):
"""Test that get_prefix_from_snap returns the service's prefix."""
microservice = "http://fake_host/fake_app"
path = f"{self.instrument}/arbitrary/path"
# Use self.detector_name to confirm pathinfo is being formed correctly
uri = microservice + f"/{self.instrument}/{self.group}/{self.snap}/{self.detector_name}"
message = {"error": False, "present": True, "uri": f"s3://{self.bucket}/{path}"}

with unittest.mock.patch("requests.get", **{"return_value.json.return_value": message}) as mock_get:
prefix = get_prefix_from_snap(microservice, self.instrument, self.group, self.detector, self.snap)
self.assertEqual(mock_get.call_args.args, (uri,))
self.assertEqual(prefix, path)

def test_get_prefix_noservice(self):
"""Test that get_prefix_from_snap returns None if there's no service configured."""
with unittest.mock.patch("requests.get") as mock_get:
prefix = get_prefix_from_snap("", self.instrument, self.group, self.detector, self.snap)
mock_get.assert_not_called()
self.assertIsNone(prefix)

def test_check_for_snap_present(self):
microservice = "http://fake_host/fake_app"
path = get_raw_path(self.instrument, self.detector, self.group, self.snap, self.exposure, self.filter)
message = {"error": False, "present": True, "uri": f"s3://{self.bucket}/{path}"}

fits_path = ResourcePath(f"s3://{self.bucket}").join(path)
with warnings.catch_warnings():
warnings.filterwarnings("ignore", "S3 does not support flushing objects", UserWarning)
with fits_path.open("wb"):
pass # Empty file is just fine

with unittest.mock.patch("requests.get", **{"return_value.json.return_value": message}):
oid = check_for_snap(boto3.client("s3"),
self.bucket,
instrument=self.instrument,
microservice=microservice,
group=self.group,
snap=self.snap,
detector=self.detector,
)
self.assertEqual(oid, path)

def test_check_for_snap_noservice(self):
path = get_raw_path(self.instrument, self.detector, self.group, self.snap, self.exposure, self.filter)
fits_path = ResourcePath(f"s3://{self.bucket}").join(path)
with warnings.catch_warnings():
warnings.filterwarnings("ignore", "S3 does not support flushing objects", UserWarning)
with fits_path.open("wb"):
pass # Empty file is just fine

oid = check_for_snap(boto3.client("s3"),
self.bucket,
instrument=self.instrument,
microservice="",
group=self.group,
snap=self.snap,
detector=self.detector,
)
self.assertEqual(oid, None)

def test_check_for_snap_absent(self):
microservice = "http://fake_host/fake_app"
path = get_raw_path(self.instrument, self.detector, self.group, self.snap, self.exposure, self.filter)
message = {"error": False, "present": True, "uri": f"s3://{self.bucket}/{path}"}

with unittest.mock.patch("requests.get", **{"return_value.json.return_value": message}):
oid = check_for_snap(boto3.client("s3"),
self.bucket,
instrument=self.instrument,
microservice=microservice,
group=self.group,
snap=self.snap,
detector=self.detector,
)
self.assertEqual(oid, None)


class LatissTest(LsstBase, unittest.TestCase):
def setUp(self):
self.instrument = "LATISS"
self.detector = 0
self.detector_name = "R00_S00"
self.snap = 0
self.exposure = 2022032100002
super().setUp()
Expand All @@ -157,6 +229,7 @@ class LsstComCamTest(LsstBase, unittest.TestCase):
def setUp(self):
self.instrument = "LSSTComCam"
self.detector = 4
self.detector_name = "R22_S11"
self.snap = 1
self.exposure = 2022032100003
super().setUp()
Expand All @@ -174,6 +247,7 @@ class LsstCamTest(LsstBase, unittest.TestCase):
def setUp(self):
self.instrument = "LSSTCam"
self.detector = 42
self.detector_name = "R11_S20"
self.snap = 0
self.exposure = 2022032100004
super().setUp()
Expand Down

0 comments on commit db85f4b

Please sign in to comment.