testing pipeline

Makefile

@@ -42,7 +42,7 @@ run: deps ## Start the api locally on port 28900.
 	@FLASK_APP=${FLASK_APP} poetry run gunicorn "app.main:create_app()" -b 0.0.0.0:${BERLIN_API_PORT} -c gunicorn_config.py
 
 run-container: build ## Runs a container from the pre-build image 
-	docker run -p 28900:28900 --env BERLIN_API_BUILD_TIME='${BERLIN_API_BUILD_TIME}' -e BERLIN_API_BUILD_TIME="${BERLIN_API_BUILD_TIME}" -e BERLIN_API_VERSION="${BERLIN_API_VERSION}" -ti berlin_api
+	docker run -p 28940:28900 --env BERLIN_API_BUILD_TIME='${BERLIN_API_BUILD_TIME}' -e BERLIN_API_BUILD_TIME="${BERLIN_API_BUILD_TIME}" -e BERLIN_API_VERSION="${BERLIN_API_VERSION}" -ti berlin_api
 
 test: deps ## runs all tests
 	poetry run pytest -v tests/
@@ -63,4 +63,3 @@ help: ## Show this help.
 		if (/^[a-zA-Z_-]+:.*?##.*$$/) {printf "    ${YELLOW}%-20s${GREEN}%s${RESET}\n", $$1, $$2} \
 		else if (/^## .*$$/) {printf "  ${CYAN}%s${RESET}\n", substr($$1,4)} \
 		}' $(MAKEFILE_LIST)
-

README.md

@@ -66,26 +66,36 @@ curl 'http://localhost:28900/berlin/search?q=house+prices+in+londo&state=gb' | j
 
 replacing `localhost` with the local endpoint (`jq` used for formatting).
 
-This will return results of the form:
-
 ```json
 {
   "matches": [
     {
-      "encoding": "UN-LOCODE",
-      "id": "ca:lod",
-      "key": "UN-LOCODE-ca:lod",
-      "words": [
-        "london"
-      ]
+      "loc": {
+        "encoding": "UN-LOCODE",
+        "id": "ca:lod",
+        "key": "UN-LOCODE-ca:lod",
+        "words": [
+          "london"
+        ]
+      },
+      "match": {
+        "score": 1008,
+        "offset": [17, 22]
+      }
     },
     {
-      "encoding": "UN-LOCODE",
-      "id": "us:ldn",
-      "key": "UN-LOCODE-us:ldn",
-      "words": [
-        "london"
-      ]
+      "loc": {
+        "encoding": "UN-LOCODE",
+        "id": "us:ldn",
+        "key": "UN-LOCODE-us:ldn",
+        "words": [
+          "london"
+        ]
+      },
+      "match": {
+        "score": 1008,
+        "offset": [17, 22]
+      }
     }
     ...
   ]

app/logger.py

@@ -1,28 +1,68 @@
+import logging
+import logging.config
 from datetime import datetime
 
 import structlog
+import structlog._log_levels
 
 from app.settings import settings
 
 
-def configure_logging():
+def add_severity_level(logger, method_name, event_dict):
+    if method_name == "info":
+        event_dict[0][0]["severity"] = 0
+    elif method_name == "error":
+        event_dict[0][0]["severity"] = 1
+
+    return event_dict
+
+
+def setup_logging():
+    shared_processors = []
+    processors = shared_processors + [
+        structlog.stdlib.PositionalArgumentsFormatter(),
+        structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
+        add_severity_level,
+    ]
     structlog.configure(
-        processors=[
-            structlog.processors.TimeStamper(fmt="iso", utc=True),
-            structlog.processors.dict_tracebacks,
-            structlog.processors.JSONRenderer(),
-        ],
-        context_class=structlog.threadlocal.wrap_dict(dict),
+        cache_logger_on_first_use=True,
+        wrapper_class=structlog.make_filtering_bound_logger(logging.INFO),
+        processors=processors,
+        context_class=dict,
         logger_factory=structlog.stdlib.LoggerFactory(),
     )
 
+    stdlib_config = {
+        "version": 1,
+        "disable_existing_loggers": False,
+        "formatters": {
+            "json": {
+                "()": structlog.stdlib.ProcessorFormatter,
+                "processor": structlog.processors.JSONRenderer(),
+                "foreign_pre_chain": shared_processors,
+            },
+        },
+        "handlers": {
+            "stream": {
+                "level": "DEBUG",
+                "formatter": "json",
+                "class": "logging.StreamHandler",
+                "stream": "ext://sys.stderr",
+            },
+        },
+        "loggers": {
+            "": {
+                "handlers": ["stream"],
+                "level": "DEBUG",
+                "propagate": True,
+            },
+        },
+    }
+    logging.config.dictConfig(stdlib_config)
 
-def setup_logger():
     return structlog.get_logger(
         namespace=settings.NAMESPACE,
         created_at=datetime.utcnow().isoformat(),
+        event="",
+        severity=0,  # default
     )
-
-
-configure_logging()
-logger = setup_logger()

app/main.py

@@ -1,22 +1,11 @@
-import logging
-import sys
-
 from flask import Flask
 
-from app.logger import logger
-from app.settings import get_custom_settings, settings
+from app.logger import setup_logging
+from app.settings import settings
 from app.views.berlin import berlin_blueprint
 from app.views.health import health_blueprint
 
-logging.basicConfig(
-    format="%(message)s",
-    stream=sys.stdout,
-    level=logging.INFO,
-)
-
-logger.info(
-    "initial configuration", data=get_custom_settings(), level="INFO", severity=0
-)
+logger = setup_logging()
 
 
 def create_app():

app/models.py

@@ -3,6 +3,26 @@
 
 from berlin import Location
 
+from app.logger import setup_logging
+
+logger = setup_logging()
+
+
+@dataclass
+class MatchModel:
+    score: int | None
+    offset: list[int] | None
+
+    @classmethod
+    def from_location(cls, loc: Location) -> "MatchModel":
+        try:
+            return cls(score=loc.get_score(), offset=loc.get_offset())
+        except AttributeError:
+            logger.error("no offset or score available")
+
+    def to_json(self):
+        return asdict(self)
+
 
 @dataclass
 class LocationModel:

app/views/berlin.py

@@ -1,9 +1,10 @@
 from flask import Blueprint, jsonify, request
 
-from app.logger import logger
-from app.models import LocationModel
+from app.logger import setup_logging
+from app.models import LocationModel, MatchModel
 from app.store import get_db
 
+logger = setup_logging()
 db = get_db()
 
 berlin_blueprint = Blueprint("berlin", __name__)
@@ -17,9 +18,7 @@ def berlin_code(key):
     except Exception as e:
         logger.error(
             event="error retrieving key from database ",
-            stack_trace=str(e),
-            level="ERROR",
-            severity=1,
+            error=str(e),
         )
 
         return jsonify({"key": key, "error": "Not found"}), 404
@@ -36,24 +35,37 @@ def berlin_search():
     lev_distance = request.args.get("lev_distance", type=int) or 2
 
     try:
-        log_message = f"Querying database with q={q}, state={state}, limit={limit}, lev_distance={lev_distance}"
-        logger.info(log_message, severity=0)
+        logger.info(
+            event="Querying database",
+            data={
+                "q": q,
+                "state": state,
+                "limit": limit,
+                "lev_distance": lev_distance,
+            },
+        )
 
         result = db.query(q, state=state, limit=limit, lev_distance=lev_distance)
 
-        locations = {
-            "matches": [
-                LocationModel.from_location(loc, db).to_json() for loc in result
-            ]
-        }
+        matches = [
+            {
+                "loc": LocationModel.from_location(loc, db).to_json(),
+                "scores": MatchModel.from_location(loc).to_json(),
+            }
+            for loc in result
+        ]
+
+        start_idx = matches[0]["scores"]["offset"][0]
+        end_idx = matches[0]["scores"]["offset"][1]
+        q = q[:start_idx] + q[end_idx:]
+
+        locations = {"query": q, "matches": matches}
         return jsonify(locations), 200
 
     except Exception as e:
         logger.error(
             event="error querying the database ",
-            stack_trace=str(e),
-            level="ERROR",
-            severity=1,
+            error=str(e),
         )
         return (
             jsonify({"error": "error querying the database"}),

gunicorn_config.py

@@ -39,10 +39,6 @@ def json_record(
                 "method": record.args["m"],
                 "path": url,
                 "status": str(record.args["s"]),
-                "user_agent": record.args["a"],
-                "referer": record.args["f"],
-                "duration_in_ms": record.args["M"],
-                "pid": record.args["p"],
             },
             severity=severity,
         )
@@ -66,6 +62,7 @@ def json_record(
             0 if record.levelname == "INFO" else 1 if record.levelname == "ERROR" else 2
         )
         payload.pop("time", None)
+        payload.pop("taskName", None)
         payload.pop("message", None)
 
         return payload