Merge branch 'tickets/DM-40822'

.github/workflows/build-base.yml

@@ -43,12 +43,6 @@ jobs:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Login to Google Artifact Registry
-        uses: docker/login-action@v2
-        with:
-          registry: us-central1-docker.pkg.dev
-          username: _json_key_base64
-          password: ${{ secrets.GAR_JSON_B64 }}
       - name: Determine base image eups tag
         run: |
           if [[ -n "$STACK_TAG" ]]; then
@@ -81,8 +75,7 @@ jobs:
           BRANCH=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
           [ "$BRANCH" == "merge" ] && BRANCH=$(echo "${{ github.head_ref }}" | sed -e 's,.*/\(.*\),\1,')
 
-          for IMAGE_ID in "ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME" \
-            "us-central1-docker.pkg.dev/prompt-proto/prompt/$IMAGE_NAME"; do
+          for IMAGE_ID in "ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME"; do
 
             STACK_TAG="$(< stack.tag)"
             if [ "$BRANCH" == "main" ]; then

.github/workflows/build-service.yml

@@ -92,12 +92,6 @@ jobs:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Login to Google Artifact Registry
-        uses: docker/login-action@v2
-        with:
-          registry: us-central1-docker.pkg.dev
-          username: _json_key_base64
-          password: ${{ secrets.GAR_JSON_B64 }}
       - name: Determine eups tag
         run: |
           docker run ghcr.io/${{ github.repository_owner }}/prompt-proto-base:"$BASE_TAG" bash -c "cat stack/miniconda*/ups_db/global.tags" > eups.tag
@@ -115,8 +109,7 @@ jobs:
           BRANCH=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
           [ "$BRANCH" == "merge" ] && BRANCH=$(echo "${{ github.head_ref }}" | sed -e 's,.*/\(.*\),\1,')
 
-          for IMAGE_ID in "ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME" \
-                          "us-central1-docker.pkg.dev/prompt-proto/prompt/$IMAGE_NAME"; do
+          for IMAGE_ID in "ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME"; do
             if [ "$BRANCH" == "main" ]; then
               VERSION="$BASE_TAG"
             else

bin.src/make_remote_butler.py

@@ -24,11 +24,6 @@
 """Simple script for creating a repository at a remote URI, given
 a source repository and export file.
 
-For most values of --target-repo and --seed-config, this script is only useful
-if run from the prompt-proto project on Google Cloud (because of access
-restrictions to **both** the repository's storage location and its registry
-database).
-
 The user is responsible for clearing any old copies of the repository from
 both the target URI and the registry database.
 """

python/activator/logger.py

@@ -19,8 +19,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-__all__ = ["GCloudStructuredLogFormatter", "UsdfJsonFormatter", "setup_google_logger", "setup_usdf_logger",
-           "RecordFactoryContextAdapter"]
+__all__ = ["UsdfJsonFormatter", "setup_usdf_logger", "RecordFactoryContextAdapter"]
 
 import collections.abc
 from contextlib import contextmanager
@@ -99,34 +98,6 @@ def _set_context_logger():
     logging.setLogRecordFactory(RecordFactoryContextAdapter(logging.getLogRecordFactory()))
 
 
-# TODO: replace with something more extensible, once we know what needs to
-# vary besides the formatter (handler type?).
-def setup_google_logger(labels=None):
-    """Set global logging settings for prompt_prototype.
-
-    Calling this function makes `GCloudStructuredLogFormatter` the root
-    formatter and redirects all warnings to go through it.
-
-    Parameters
-    ----------
-    labels : `dict` [`str`, `str`]
-        Any metadata that should be attached to all logs. See
-        ``LogEntry.labels`` in Google Cloud REST API documentation.
-
-    Returns
-    -------
-    handler : `logging.Handler`
-        The handler used by the root logger.
-    """
-    _set_context_logger()
-    log_handler = logging.StreamHandler()
-    log_handler.setFormatter(GCloudStructuredLogFormatter(labels))
-    logging.basicConfig(handlers=[log_handler])
-    _channel_all_to_pylog()
-    _set_lsst_logging_levels()
-    return log_handler
-
-
 def setup_usdf_logger(labels=None):
     """Set global logging settings for prompt_prototype.
 
@@ -151,38 +122,6 @@ def setup_usdf_logger(labels=None):
     return log_handler
 
 
-class GCloudStructuredLogFormatter(logging.Formatter):
-    """A formatter that can be parsed by the Google Cloud logging agent.
-
-    The formatter's output is a JSON-encoded message containing keywords
-    recognized by the logging agent.
-
-    Parameters
-    ----------
-    labels : `dict` [`str`, `str`]
-        Any metadata that should be attached to the log. See ``LogEntry.labels``
-        in Google Cloud REST API documentation.
-    """
-    def __init__(self, labels=None):
-        super().__init__()
-
-        if labels:
-            self._labels = labels
-        else:
-            self._labels = {}
-
-    def format(self, record):
-        # format updates record.message, but the full info is *only* in the return value.
-        msg = super().format(record)
-
-        entry = {
-            "severity": record.levelname,
-            "logging.googleapis.com/labels": self._labels | record.logging_context,
-            "message": msg,
-        }
-        return json.dumps(entry, default=_encode_json_extras)
-
-
 class UsdfJsonFormatter(logging.Formatter):
     """A formatter that can be parsed by the Loki/Grafana system at USDF.
 

python/tester/upload.py

@@ -155,9 +155,6 @@ def get_samples_non_lsst(bucket, instrument):
     # TODO: set up a lookup-friendly class to represent the return value
 
     # TODO: replace this dict with something more scalable.
-    #     One option is to attach metadata to the Google Storage objects at
-    #     upload time, another is to download the blob and actually read
-    #     its header.
     hsc_metadata = {
         59126: {"ra": 149.28531249999997, "dec": 2.935002777777778, "rot": 270.0, "time": 1457332820.0},
         59134: {"ra": 149.45749166666664, "dec": 2.926961111111111, "rot": 270.0, "time": 1457333685.0},

tests/test_logger.py

@@ -29,7 +29,7 @@
 
 import pytest
 
-from activator.logger import GCloudStructuredLogFormatter, UsdfJsonFormatter, \
+from activator.logger import UsdfJsonFormatter, \
     _parse_log_levels, RecordFactoryContextAdapter
 
 
@@ -59,148 +59,6 @@ def test_root_(self):
                          )
 
 
-class GoogleFormatterTest(unittest.TestCase):
-    """Test GCloudStructuredLogFormatter with fake log messages.
-    """
-    def setUp(self):
-        super().setUp()
-
-        # Buffer for log output.
-        # Can't use assertLogs, because it inserts its own handler/formatter.
-        self.output = io.StringIO()
-        self.addCleanup(io.StringIO.close, self.output)
-
-        # GCloudStructuredLogFormatter assumes a logging_context field is present.
-        old_factory = logging.getLogRecordFactory()
-        self.addCleanup(logging.setLogRecordFactory, old_factory)
-        logging.setLogRecordFactory(RecordFactoryContextAdapter(old_factory))
-
-        log_handler = logging.StreamHandler(self.output)
-        log_handler.setFormatter(GCloudStructuredLogFormatter(
-            labels={"instrument": "NotACam"},
-        ))
-        # Unique logger per test
-        self.log = logging.getLogger(self.id())
-        self.log.propagate = False
-        self.log.addHandler(log_handler)
-        self.log.setLevel(logging.DEBUG)
-
-    def _check_log(self, outputs, level, labels, texts):
-        """Check that the log output is formatted correctly.
-
-        Parameters
-        ----------
-        outputs : `list` [`str`]
-            A list of the formatted log messages.
-        level : `str`
-            The emitted log level.
-        labels : `dict` [`str`, `str`]
-            The labels attached to the log message.
-        texts : `list` [`str`]
-            The expected log messages.
-        """
-        self.assertEqual(len(outputs), len(texts))
-        for output, text in zip(outputs, texts):
-            parsed = json.loads(output)
-            self.assertEqual(parsed["severity"], level)
-            self.assertEqual(parsed["message"], text)
-            self.assertEqual(parsed["logging.googleapis.com/labels"], labels)
-
-    def test_direct(self):
-        """Test the translation of verbatim log messages.
-        """
-        msg = "Consider a spherical cow..."
-        self.log.info(msg)
-        self._check_log(self.output.getvalue().splitlines(),
-                        "INFO", {"instrument": "NotACam"},
-                        [msg])
-
-    def test_args(self):
-        """Test the translation of arg-based log messages.
-        """
-        msg = "Consider a %s..."
-        args = "rotund bovine"
-        self.log.warning(msg, args)
-        self._check_log(self.output.getvalue().splitlines(),
-                        "WARNING", {"instrument": "NotACam"},
-                        [msg % args])
-
-    def test_quotes(self):
-        """Test handling of messages containing single or double quotes.
-        """
-        msgs = ["Consider a so-called 'spherical cow'.",
-                'Consider a so-called "spherical cow".',
-                ]
-        for msg in msgs:
-            self.log.info(msg)
-        self._check_log(self.output.getvalue().splitlines(),
-                        "INFO", {"instrument": "NotACam"},
-                        msgs)
-
-    def test_multiline(self):
-        """Test handling of messages that split across multiple lines.
-        """
-        msg = """This is a multiline
-              message with internal line
-              breaks."""
-        self.log.error(msg)
-        self._check_log(self.output.getvalue().splitlines(),
-                        "ERROR", {"instrument": "NotACam"},
-                        [msg])
-
-    def test_exception(self):
-        """Test that exception messages include the stack trace.
-        """
-        try:
-            raise RuntimeError("I take exception to that!")
-        except RuntimeError as e:
-            self.log.exception(e)
-        shredded = self.output.getvalue().splitlines()
-        self.assertEqual(len(shredded), 1)
-        self.assertIn("Traceback (most recent call last)", shredded[0])
-
-    def test_context(self):
-        """Test handling of messages that have free-form context.
-        """
-        msg = "Consider a spherical exposure."
-        exposures = {1, 2, 3}
-        visit = 42
-        ratio = 3.5
-        group = "group A"
-        settings = {"option": True}
-        with logging.getLogRecordFactory().add_context(
-            exposures=exposures,
-            visit=visit,
-            ratio=ratio,
-            group=group,
-            settings=settings,
-        ):
-            self.log.info(msg)
-        self._check_log(self.output.getvalue().splitlines(),
-                        "INFO",
-                        {"instrument": "NotACam",
-                         "exposures": list(exposures),
-                         "visit": visit,
-                         "ratio": ratio,
-                         "group": group,
-                         "settings": settings,
-                         },
-                        [msg])
-
-    def test_side_effects(self):
-        """Test that format still modifies exposure records in the same way
-        as Formatter.format.
-        """
-        msg = "Consider a %s..."
-        args = "rotund bovine"
-        factory = logging.getLogRecordFactory()
-        record = factory(self.id(), logging.INFO, "file.py", 42, msg, args, None)
-        formatter = GCloudStructuredLogFormatter()
-        formatter.format(record)
-        # If format has no side effects, record.message does not exist.
-        self.assertEqual(record.message, msg % args)
-
-
 class UsdfJsonFormatterTest(unittest.TestCase):
     """Test UsdfJsonFormatter with fake log messages.
     """