Sensor failure recovery, support calibration to the antenna, save onboard cals separately, cal on startup

.github/workflows/github-actions-test.yml

@@ -75,8 +75,8 @@ jobs:
           source ./env.template
           export MOCK_SIGAN=1
           export MOCK_SIGAN_RANDOM=1
-          docker-compose build --no-cache
-          docker-compose up -d
+          docker compose build --no-cache
+          docker compose up -d
 
       - name: Wait for containers # wait for containers to finish starting
         run: sleep 45

README.md

@@ -422,12 +422,11 @@ specific to the sensor you are using.
 
 ### Sensor Calibration File
 
-By default, scos-sensor will use `configs/default_calibration.json` as the sensor
-calibration file. However, if`configs/sensor_calibration.json` or
-`configs/sigan_calibration.json` exist they will be used instead of the default
-calibration file. Sensor calibration files allow scos-sensor to apply a gain based
-on a laboratory calibration of the sensor and may also contain other useful
-metadata that characterizes the sensor performance. For additional
+By default, scos-sensor will use `configs/default_sensor_calibration.json` as the sensor
+calibration file. However, if `configs/sensor_calibration.json` exists it will be used
+instead of the default calibration file. Sensor calibration files allow SCOS Sensor to
+scale data based on a laboratory or in-field calibration of the sensor and may also
+contain other useful metadata that characterizes the sensor performance. For additional
 information on the calibration data, see the
 [NTIA-Sensor SigMF Calibration Object](https://github.com/NTIA/sigmf-ns-ntia/blob/master/ntia-sensor.sigmf-ext.md#08-the-calibration-object).
 The default calibration file is shown below:

docker-compose.yml → compose.yaml

@@ -1,5 +1,3 @@
-version: '3'
-
 services:
   db:
     image: postgres:15-alpine
@@ -45,6 +43,7 @@ services:
       - ADDITIONAL_USER_NAMES
       - ADDITIONAL_USER_PASSWORD
       - AUTHENTICATION
+      - CALIBRATION_EXPIRATION_LIMIT
       - CALLBACK_AUTHENTICATION
       - CALLBACK_SSL_VERIFICATION
       - CALLBACK_TIMEOUT
@@ -70,6 +69,8 @@ services:
       - SIGAN_CLASS
       - SIGAN_POWER_SWITCH
       - SIGAN_POWER_CYCLE_STATES
+      - STARTUP_CALIBRATION_ACTION
+      - RAY_INIT
       - RUNNING_MIGRATIONS
       - USB_DEVICE
     expose:

env.template

@@ -18,8 +18,7 @@ ADMIN_PASSWORD=password
 
 # set to CERT to enable scos-sensor certificate authentication
 AUTHENTICATION=TOKEN
-
-BASE_IMAGE=ghcr.io/ntia/scos-tekrsa/tekrsa_usb:0.2.3
+CALIBRATION_EXPIRATION_LIMIT=360
 # Default callback api/results
 # Set to CERT for certificate authentication
 CALLBACK_AUTHENTICATION=TOKEN
@@ -34,8 +33,6 @@ CALLBACK_TIMEOUT=2
 # Use either true or false
 DEBUG=true
 
-DEVICE_MODEL=RSA507A
-
 # Use latest as default for local development
 DOCKER_TAG=latest
 
@@ -75,8 +72,12 @@ SCOS_SENSOR_GIT_TAG="$(git describe --tags)"
 # SECURITY WARNING: generate unique key with `manage.py generate_secret_key`
 SECRET_KEY="$(python3 -c 'import secrets; print(secrets.token_urlsafe(64))')"
 
+# Signal analyzer selection/setup
 SIGAN_CLASS=TekRSASigan
 SIGAN_MODULE=scos_tekrsa.hardware.tekrsa_sigan
+USB_DEVICE=Tektronix
+DEVICE_MODEL=RSA507A
+BASE_IMAGE=ghcr.io/ntia/scos-tekrsa/tekrsa_usb:0.2.3
 
 # SECURITY WARNING: You should be using certs from a trusted authority.
 #                   If you don't have any, try letsencrypt or a similar service.
@@ -86,8 +87,12 @@ SSL_CA_PATH=scos_test_ca.crt
 SSL_CERT_PATH=sensor01.pem
 SSL_KEY_PATH=sensor01.pem
 
-USB_DEVICE=Tektronix
-
+# Calibration action selection
+#    The action specified here will be used to attempt an onboard
+#    sensor calibration on startup, if no onboard calibration data
+#    is available on startup. The specified action must be available.
+CALIBRATE_ON_STARTUP=true
+STARTUP_CALIBRATION_ACTION=SEA_CBRS_Calibrate_Baseline
 
 # Debug dependant settings
 if $DEBUG; then

gunicorn/config.py

@@ -2,7 +2,6 @@
 import sys
 from multiprocessing import cpu_count
 
-
 bind = ":8000"
 workers = 1
 worker_class = "gthread"

src/handlers/apps.py

@@ -5,7 +5,7 @@
 from scos_actions.signals import (
     location_action_completed,
     measurement_action_completed,
-    trigger_api_restart
+    trigger_api_restart,
 )
 
 logger = logging.getLogger(__name__)
@@ -20,7 +20,6 @@ def ready(self):
             db_location_deleted,
             db_location_updated,
             location_action_completed_callback,
-
         )
         from handlers.measurement_handler import measurement_action_completed_callback
 
@@ -40,5 +39,3 @@ def ready(self):
 
         trigger_api_restart.connect(trigger_api_restart_callback)
         logger.debug("trigger_api_restart_callback registered to trigger_api_restart")
-
-

src/initialization/__init__.py

@@ -1,31 +1,90 @@
+import importlib
 import logging
-import sys
-import types
+import time
+from os import path
 from pathlib import Path
-from subprocess import check_output
+from typing import Optional, Union
 
 from django.conf import settings
+from its_preselector.configuration_exception import ConfigurationException
+from its_preselector.controlbyweb_web_relay import ControlByWebWebRelay
+from its_preselector.preselector import Preselector
+from scos_actions.calibration.differential_calibration import DifferentialCalibration
+from scos_actions.calibration.sensor_calibration import SensorCalibration
+from scos_actions.hardware.utils import power_cycle_sigan
+from scos_actions.utils import load_from_json
 
 from utils.signals import register_component_with_status
 
 from .action_loader import ActionLoader
 from .capabilities_loader import CapabilitiesLoader
 from .sensor_loader import SensorLoader
 from .status_monitor import StatusMonitor
+from .utils import get_usb_device_exists, set_container_unhealthy
 
 logger = logging.getLogger(__name__)
 
 status_monitor = StatusMonitor()
 
 
-def get_usb_device_exists() -> bool:
-    logger.debug("Checking for USB...")
-    if not settings.RUNNING_TESTS and settings.USB_DEVICE is not None:
-        usb_devices = check_output("lsusb").decode(sys.stdout.encoding)
-        logger.debug("Checking for " + settings.USB_DEVICE)
-        logger.debug("Found " + usb_devices)
-        return settings.USB_DEVICE in usb_devices
-    return True
+def load_preselector_from_file(
+    preselector_module, preselector_class, preselector_config_file: Path
+):
+    if preselector_config_file is None:
+        return None
+    else:
+        try:
+            preselector_config = load_from_json(preselector_config_file)
+            return load_preselector(
+                preselector_config, preselector_module, preselector_class
+            )
+        except ConfigurationException:
+            logger.exception(
+                f"Unable to create preselector defined in: {preselector_config_file}"
+            )
+    return None
+
+
+def load_preselector(
+    preselector_config: str,
+    module: str,
+    preselector_class_name: str,
+    sensor_definition: dict,
+) -> Preselector:
+    logger.debug(
+        f"loading {preselector_class_name} from {module} with config: {preselector_config}"
+    )
+    if module is not None and preselector_class_name is not None:
+        preselector_module = importlib.import_module(module)
+        preselector_constructor = getattr(preselector_module, preselector_class_name)
+        preselector_config = load_from_json(preselector_config)
+        ps = preselector_constructor(sensor_definition, preselector_config)
+        register_component_with_status.send(ps, component=ps)
+    else:
+        ps = None
+    return ps
+
+
+def load_switches(switch_dir: Path) -> dict:
+    logger.debug(f"Loading switches in {switch_dir}")
+    switch_dict = {}
+    try:
+        if switch_dir is not None and switch_dir.is_dir():
+            for f in switch_dir.iterdir():
+                file_path = f.resolve()
+                logger.debug(f"loading switch config {file_path}")
+                conf = load_from_json(file_path)
+                try:
+                    switch = ControlByWebWebRelay(conf)
+                    logger.debug(f"Adding {switch.id}")
+                    switch_dict[switch.id] = switch
+                    logger.debug(f"Registering switch status for {switch.name}")
+                    register_component_with_status.send(__name__, component=switch)
+                except ConfigurationException:
+                    logger.error(f"Unable to configure switch defined in: {file_path}")
+    except Exception as ex:
+        logger.error(f"Unable to load switches {ex}")
+    return switch_dict
 
 
 def status_registration_handler(sender, **kwargs):
@@ -42,32 +101,108 @@ def set_container_unhealthy():
         Path(settings.SDR_HEALTHCHECK_FILE).touch()
 
 
+def get_calibration(
+    cal_file_path: str, cal_type: str
+) -> Optional[Union[DifferentialCalibration, SensorCalibration]]:
+    """
+    Load calibration data from file.
+
+    :param cal_file_path: Path to the JSON calibration file.
+    :param cal_type: Calibration type to load: "onboard", "sensor" or "differential"
+    :return: The ``Calibration`` object, if loaded, or ``None`` if loading failed.
+    """
+    try:
+        cal = None
+        if cal_file_path is None or cal_file_path == "":
+            logger.error("No calibration file specified, reverting to none.")
+        elif not path.exists(cal_file_path):
+            logger.error(f"{cal_file_path} does not exist, reverting to none.")
+        else:
+            logger.debug(f"Loading calibration file: {cal_file_path}")
+            # Create calibration object
+            cal_file_path = Path(cal_file_path)
+            if cal_type.lower() in ["sensor", "onboard"]:
+                cal = SensorCalibration.from_json(cal_file_path)
+            elif cal_type.lower() == "differential":
+                cal = DifferentialCalibration.from_json(cal_file_path)
+            else:
+                logger.error(f"Unknown calibration type: {cal_type}")
+                raise ValueError
+    except Exception:
+        cal = None
+        logger.exception(
+            f"Unable to load {cal_type} calibration file, reverting to none"
+        )
+    finally:
+        return cal
+
+
 try:
+    sensor_loader = None
     register_component_with_status.connect(status_registration_handler)
-    usb_device_exists = get_usb_device_exists()
-    if usb_device_exists:
-        action_loader = ActionLoader()
-        logger.debug(f"Actions ActionLoader has {len(action_loader.actions)} actions")
-        capabilities_loader = CapabilitiesLoader()
-        logger.debug("Calling sensor loader.")
-        sensor_loader = SensorLoader(capabilities_loader.capabilities)
+    action_loader = ActionLoader()
+    logger.debug(f"Actions ActionLoader has {len(action_loader.actions)} actions")
+    capabilities_loader = CapabilitiesLoader()
+    switches = load_switches(settings.SWITCH_CONFIGS_DIR)
+    preselector = load_preselector(
+        settings.PRESELECTOR_CONFIG,
+        settings.PRESELECTOR_MODULE,
+        settings.PRESELECTOR_CLASS,
+        capabilities_loader.capabilities["sensor"],
+    )
+
+    if get_usb_device_exists():
+        logger.debug("Initializing Sensor...")
+        sensor_loader = SensorLoader(
+            capabilities_loader.capabilities, switches, preselector
+        )
+
+    else:
+        logger.debug("Power cycling sigan")
+        try:
+            power_cycle_sigan(switches)
+        except Exception as power_cycle_exception:
+            logger.error(f"Unable to power cycle sigan: {power_cycle_exception}")
+        set_container_unhealthy()
+        time.sleep(60)
+
+    if not settings.RUNNING_MIGRATIONS:
         if (
-            not settings.RUNNING_MIGRATIONS
-            and not sensor_loader.sensor.signal_analyzer.healthy()
+            sensor_loader.sensor.signal_analyzer is None
+            or not sensor_loader.sensor.signal_analyzer.healthy()
         ):
+            try:
+                power_cycle_sigan(switches)
+            except Exception as power_cycle_exception:
+                logger.error(f"Unable to power cycle sigan: {power_cycle_exception}")
             set_container_unhealthy()
-    else:
-        action_loader = types.SimpleNamespace()
-        action_loader.actions = {}
-        capabilities_loader = types.SimpleNamespace()
-        capabilities_loader.capabilities = {}
-        sensor_loader = types.SimpleNamespace()
-        sensor_loader.sensor = types.SimpleNamespace()
-        sensor_loader.sensor.signal_analyzer = None
-        sensor_loader.preselector = None
-        sensor_loader.switches = {}
-        sensor_loader.capabilities = {}
-        logger.warning("Usb is not ready. Marking container as unhealthy")
-        set_container_unhealthy()
-except:
-    logger.exception("Error during initialization")
+            time.sleep(60)
+
+        # Calibration loading
+        if not settings.RUNNING_TESTS:
+            # Load the onboard cal file as the sensor calibration, if it exists
+            onboard_cal = get_calibration(settings.ONBOARD_CALIBRATION_FILE, "onboard")
+            if onboard_cal is not None:
+                sensor_loader.sensor.sensor_calibration = onboard_cal
+            else:
+                # Otherwise, try using the sensor calibration file
+                sensor_cal = get_calibration(settings.SENSOR_CALIBRATION_FILE, "sensor")
+                if sensor_cal is not None:
+                    sensor_loader.sensor.sensor_calibration = sensor_cal
+
+            # Now load the differential calibration, if it exists
+            differential_cal = get_calibration(
+                settings.DIFFERENTIAL_CALIBRATION_FILE,
+                "differential",
+            )
+            sensor_loader.sensor.differential_calibration = differential_cal
+
+        import ray
+
+        if settings.RAY_INIT and not ray.is_initialized():
+            # Dashboard is only enabled if ray[default] is installed
+            logger.debug("Initializing ray.")
+            ray.init()
+except BaseException as error:
+    logger.exception(f"Error during initialization: {error}")
+    set_container_unhealthy()