From 63f4dbd0b3c8475984e789cb22eb87b542713b58 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 25 Jan 2024 19:03:20 -0500 Subject: [PATCH 01/56] add default differential cal file --- configs/default_differential_calibration.json | 7 +++++++ ...lt_calibration.json => default_sensor_calibration.json} | 0 2 files changed, 7 insertions(+) create mode 100644 configs/default_differential_calibration.json rename configs/{default_calibration.json => default_sensor_calibration.json} (100%) diff --git a/configs/default_differential_calibration.json b/configs/default_differential_calibration.json new file mode 100644 index 00000000..4fd30316 --- /dev/null +++ b/configs/default_differential_calibration.json @@ -0,0 +1,7 @@ +{ + "calibration_parameters": [], + "calibration_data": { + "loss": 0.0 + }, + "calibration_reference": "DEFAULT CALIBRATION" +} diff --git a/configs/default_calibration.json b/configs/default_sensor_calibration.json similarity index 100% rename from configs/default_calibration.json rename to configs/default_sensor_calibration.json From bac310960013b99984f200a3891e87c821e08f42 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 25 Jan 2024 19:07:01 -0500 Subject: [PATCH 02/56] remove sigan calibration --- README.md | 9 +++--- src/initialization/sensor_loader.py | 43 +---------------------------- src/requirements.in | 2 +- src/requirements.txt | 11 ++++++-- src/sensor/settings.py | 6 ---- 5 files changed, 15 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 97399b36..47d5b062 100644 --- a/README.md +++ b/README.md @@ -394,11 +394,10 @@ specific to the sensor you are using. ### Sensor Calibration File By default, scos-sensor will use `configs/default_calibration.json` as the sensor -calibration file. However, if`configs/sensor_calibration.json` or -`configs/sigan_calibration.json` exist they will be used instead of the default -calibration file. Sensor calibration files allow scos-sensor to pply a gain based -on a laboratory calibration of the sensor and may also contain other useful -metadata that characterizes the sensor performance. For additional +calibration file. However, if `configs/sensor_calibration.json` exists it will be used +instead of the default calibration file. Sensor calibration files allow SCOS Sensor to +scale data based on a laboratory or in-field calibration of the sensor and may also +contain other useful metadata that characterizes the sensor performance. For additional information on the calibration data, see the [NTIA-Sensor SigMF Calibration Object](https://github.com/NTIA/sigmf-ns-ntia/blob/master/ntia-sensor.sigmf-ext.md#08-the-calibration-object). The default calibration file is shown below: diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index af0094e4..6375c87c 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -41,7 +41,6 @@ def sensor(self) -> Sensor: def load_sensor(sensor_capabilities: dict) -> Sensor: switches = {} - sigan_cal = None sensor_cal = None preselector = None location = None @@ -60,9 +59,6 @@ def load_sensor(sensor_capabilities: dict) -> Sensor: sensor_cal = get_sensor_calibration( settings.SENSOR_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE ) - sigan_cal = get_sigan_calibration( - settings.SIGAN_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE - ) preselector = load_preselector( settings.PRESELECTOR_CONFIG, settings.PRESELECTOR_MODULE, @@ -79,9 +75,7 @@ def load_sensor(sensor_capabilities: dict) -> Sensor: "Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE ) sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) - sigan = sigan_constructor( - sensor_cal=sensor_cal, sigan_cal=sigan_cal, switches=switches - ) + sigan = sigan_constructor(switches=switches) register_component_with_status.send(sigan, component=sigan) else: logger.info("Running migrations. Not loading signal analyzer.") @@ -158,41 +152,6 @@ def load_preselector( return ps -def get_sigan_calibration( - sigan_cal_file_path: str, default_cal_file_path: str -) -> Calibration: - """ - Load signal analyzer calibration data from file. - - :param sigan_cal_file_path: Path to JSON file containing signal - analyzer calibration data. - :param default_cal_file_path: Path to the default cal file. - :return: The signal analyzer ``Calibration`` object. - """ - try: - sigan_cal = None - if sigan_cal_file_path is None or sigan_cal_file_path == "": - logger.warning( - "No sigan calibration file specified. Not loading calibration file." - ) - elif not path.exists(sigan_cal_file_path): - logger.warning( - sigan_cal_file_path - + " does not exist. Not loading sigan calibration file." - ) - else: - logger.debug(f"Loading sigan cal file: {sigan_cal_file_path}") - default = check_for_default_calibration( - sigan_cal_file_path, default_cal_file_path, "Sigan" - ) - sigan_cal = load_from_json(sigan_cal_file_path, default) - sigan_cal.is_default = default - except Exception: - sigan_cal = None - logger.exception("Unable to load sigan calibration data, reverting to none") - return sigan_cal - - def get_sensor_calibration( sensor_cal_file_path: str, default_cal_file_path: str ) -> Calibration: diff --git a/src/requirements.in b/src/requirements.in index 3562dcd1..2dbcbd91 100644 --- a/src/requirements.in +++ b/src/requirements.in @@ -11,7 +11,7 @@ packaging>=23.0, <24.0 psycopg2-binary>=2.0, <3.0 requests-mock>=1.0, <2.0 requests_oauthlib>=1.0, <2.0 -scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@discover_action_types +scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@calibrate_to_antenna # The following are sub-dependencies for which SCOS Sensor enforces a # higher minimum patch version than the dependencies which require them. diff --git a/src/requirements.txt b/src/requirements.txt index 05770207..233159bf 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -20,6 +20,8 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via ray +colorama==0.4.6 + # via click cryptography==41.0.7 # via -r requirements.in defusedxml==0.7.1 @@ -91,6 +93,8 @@ numpy==1.24.4 # scos-actions # sigmf # tekrsa-api-wrap +oauthlib==3.2.2 + # via requests-oauthlib packaging==23.2 # via # -r requirements.in @@ -132,8 +136,11 @@ requests==2.31.0 # its-preselector # ray # requests-mock + # requests-oauthlib requests-mock==1.11.0 # via -r requirements.in +requests-oauthlib==1.3.1 + # via -r requirements.in rpds-py==0.13.2 # via # jsonschema @@ -144,9 +151,9 @@ ruamel-yaml-clib==0.2.8 # via ruamel-yaml scipy==1.10.1 # via scos-actions -scos-actions @ git+https://github.com/NTIA/scos-actions@discover_action_types +scos-actions @ git+https://github.com/NTIA/scos-actions@calibrate_to_antenna # via scos-tekrsa -scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@discover_action_types +scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@calibrate_to_antenna # via -r requirements.in sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive # via scos-actions diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 20154a19..0c73cb91 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -80,15 +80,9 @@ else: SENSOR_CALIBRATION_FILE = DEFAULT_CALIBRATION_FILE -if path.exists(path.join(CONFIG_DIR, "sigan_calibration.json")): - SIGAN_CALIBRATION_FILE = path.join(CONFIG_DIR, "sigan_calibration.json") -else: - SIGAN_CALIBRATION_FILE = DEFAULT_CALIBRATION_FILE - if path.exists(path.join(CONFIG_DIR, "sensor_definition.json")): SENSOR_DEFINITION_FILE = path.join(CONFIG_DIR, "sensor_definition.json") os.environ["SENSOR_CALIBRATION_FILE"] = SENSOR_CALIBRATION_FILE -os.environ["SIGAN_CALIBRATION_FILE"] = SIGAN_CALIBRATION_FILE MEDIA_ROOT = path.join(REPO_ROOT, "files") PRESELECTOR_CONFIG = path.join(CONFIG_DIR, "preselector_config.json") From 4f150fa383426782309a223c659ca48898c6cc71 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 25 Jan 2024 20:06:02 -0500 Subject: [PATCH 03/56] update requirements --- src/requirements-dev.txt | 15 ++++++++++++--- src/requirements.txt | 2 -- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index 18ed0aba..b97088e0 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -59,7 +59,9 @@ colorama==0.4.6 colorful==0.5.5 # via ray coverage[toml]==7.3.2 - # via pytest-cov + # via + # coverage + # pytest-cov cryptography==41.0.7 # via -r requirements.txt defusedxml==0.7.1 @@ -182,6 +184,10 @@ numpy==1.24.4 # tekrsa-api-wrap nvidia-ml-py==12.535.133 # via gpustat +oauthlib==3.2.2 + # via + # -r requirements.txt + # requests-oauthlib opencensus==0.11.3 # via ray opencensus-context==0.1.3 @@ -285,8 +291,11 @@ requests==2.31.0 # its-preselector # ray # requests-mock + # requests-oauthlib requests-mock==1.11.0 # via -r requirements.txt +requests-oauthlib==1.3.1 + # via -r requirements.txt rpds-py==0.13.2 # via # -r requirements.txt @@ -306,11 +315,11 @@ scipy==1.10.1 # via # -r requirements.txt # scos-actions -scos-actions @ git+https://github.com/NTIA/scos-actions@discover_action_types +scos-actions @ git+https://github.com/NTIA/scos-actions@calibrate_to_antenna # via # -r requirements.txt # scos-tekrsa -scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@discover_action_types +scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@calibrate_to_antenna # via -r requirements.txt sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive # via diff --git a/src/requirements.txt b/src/requirements.txt index 233159bf..b0abfcf8 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -20,8 +20,6 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via ray -colorama==0.4.6 - # via click cryptography==41.0.7 # via -r requirements.in defusedxml==0.7.1 From a2353a4400d36ce165eecbbfdc604fa9dd883bc3 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 25 Jan 2024 20:07:08 -0500 Subject: [PATCH 04/56] update sensor loader for calibration changes now handle sensor and differential calibrations --- README.md | 2 +- src/initialization/sensor_loader.py | 74 ++++++++++++++++++----------- src/sensor/settings.py | 26 +++++++--- 3 files changed, 66 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 47d5b062..e56603f8 100644 --- a/README.md +++ b/README.md @@ -393,7 +393,7 @@ specific to the sensor you are using. ### Sensor Calibration File -By default, scos-sensor will use `configs/default_calibration.json` as the sensor +By default, scos-sensor will use `configs/default_sensor_calibration.json` as the sensor calibration file. However, if `configs/sensor_calibration.json` exists it will be used instead of the default calibration file. Sensor calibration files allow SCOS Sensor to scale data based on a laboratory or in-field calibration of the sensor and may also diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index 6375c87c..cb940899 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -3,13 +3,15 @@ import signal from os import path from pathlib import Path +from typing import Union from django.conf import settings from its_preselector.configuration_exception import ConfigurationException from its_preselector.controlbyweb_web_relay import ControlByWebWebRelay from its_preselector.preselector import Preselector from scos_actions import utils -from scos_actions.calibration.calibration import Calibration, load_from_json +from scos_actions.calibration.differential_calibration import DifferentialCalibration +from scos_actions.calibration.sensor_calibration import SensorCalibration from scos_actions.hardware.sensor import Sensor from scos_actions.metadata.utils import construct_geojson_point @@ -40,8 +42,10 @@ def sensor(self) -> Sensor: def load_sensor(sensor_capabilities: dict) -> Sensor: + gps = None # TODO Not Implemented switches = {} sensor_cal = None + differential_cal = None preselector = None location = None if not settings.RUNNING_TESTS: @@ -56,8 +60,15 @@ def load_sensor(sensor_capabilities: dict) -> Sensor: sensor_loc["z"] if "z" in sensor_loc else None, ) switches = load_switches(settings.SWITCH_CONFIGS_DIR) - sensor_cal = get_sensor_calibration( - settings.SENSOR_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE + sensor_cal = get_calibration( + settings.SENSOR_CALIBRATION_FILE, + settings.DEFAULT_SENSOR_CALIBRATION_FILE, + "sensor", + ) + differential_cal = get_calibration( + settings.DIFFERENTIAL_CALIBRATION_FILE, + settings.DEFAULT_DIFFERENTIAL_CALIBRATION_FILE, + "differential", ) preselector = load_preselector( settings.PRESELECTOR_CONFIG, @@ -84,10 +95,13 @@ def load_sensor(sensor_capabilities: dict) -> Sensor: sensor = Sensor( signal_analyzer=sigan, + gps=gps, preselector=preselector, switches=switches, capabilities=sensor_capabilities, location=location, + sensor_cal=sensor_cal, + differential_cal=differential_cal, ) return sensor @@ -152,39 +166,43 @@ def load_preselector( return ps -def get_sensor_calibration( - sensor_cal_file_path: str, default_cal_file_path: str -) -> Calibration: +def get_calibration( + cal_file_path: str, default_cal_file_path: str, cal_type: str +) -> Union[DifferentialCalibration, SensorCalibration]: """ - Load sensor calibration data from file. + Load calibration data from file. - :param sensor_cal_file_path: Path to JSON file containing sensor - calibration data. - :param default_cal_file_path: Name of the default calibration file. - :return: The sensor ``Calibration`` object. + :param cal_file_path: Path to the JSON calibration file. + :param default_cal_file_path: Path to the default calibration file. + :param cal_type: Calibration type to load, either "sensor" or "differential" + :return: The ``Calibration`` object. """ try: - sensor_cal = None - if sensor_cal_file_path is None or sensor_cal_file_path == "": - logger.warning( - "No sensor calibration file specified. Not loading calibration file." - ) - elif not path.exists(sensor_cal_file_path): - logger.warning( - sensor_cal_file_path - + " does not exist. Not loading sensor calibration file." - ) + cal = None + if cal_file_path is None or cal_file_path == "": + logger.error("No calibration file specified.") + raise ValueError + elif not path.exists(cal_file_path): + logger.error(f"{cal_file_path} does not exist.") + raise FileNotFoundError else: - logger.debug(f"Loading sensor cal file: {sensor_cal_file_path}") + logger.debug(f"Loading calibration file: {cal_file_path}") default = check_for_default_calibration( - sensor_cal_file_path, default_cal_file_path, "Sensor" + cal_file_path, default_cal_file_path, cal_type ) - sensor_cal = load_from_json(sensor_cal_file_path, default) - sensor_cal.is_default = default + # Create calibration object + if cal_type.lower() == "sensor": + cal = SensorCalibration.from_json(cal_file_path, default) + elif cal_type.lower() == "differential": + cal = DifferentialCalibration.from_json(cal_file_path, default) + else: + logger.error(f"Unknown calibration type: {cal_type}") + raise ValueError except Exception: - sensor_cal = None - logger.exception("Unable to load sensor calibration data, reverting to none") - return sensor_cal + cal = None + logger.exception("Unable to load calibration file, reverting to none") + finally: + return cal def check_for_default_calibration( diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 0c73cb91..4fd65e28 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -72,16 +72,28 @@ ACTIONS_DIR = path.join(CONFIG_DIR, "actions") DRIVERS_DIR = path.join(REPO_ROOT, "drivers") -DEFAULT_CALIBRATION_FILE = path.join(CONFIG_DIR, "default_calibration.json") -os.environ["DEFAULT_CALIBRATION_FILE"] = str(DEFAULT_CALIBRATION_FILE) +DEFAULT_SENSOR_CALIBRATION_FILE = path.join( + CONFIG_DIR, "default_sensor_calibration.json" +) +DEFAULT_DIFFERENTIAL_CALIBRATION_FILE = path.join( + CONFIG_DIR, "default_differential_calibration.json" +) +os.environ["DEFAULT_SENSOR_CALIBRATION_FILE"] = str(DEFAULT_SENSOR_CALIBRATION_FILE) +os.environ["DEFAULT_DIFFERENTIAL_CALIBRATION_FILE"] = str( + DEFAULT_DIFFERENTIAL_CALIBRATION_FILE +) # JSON configs -if path.exists(path.join(CONFIG_DIR, "sensor_calibration.json")): - SENSOR_CALIBRATION_FILE = path.join(CONFIG_DIR, "sensor_calibration.json") +if path.exists(sensor_cal_path := path.join(CONFIG_DIR, "sensor_calibration.json")): + SENSOR_CALIBRATION_FILE = sensor_cal_path +else: + SENSOR_CALIBRATION_FILE = DEFAULT_SENSOR_CALIBRATION_FILE +if path.exists(diff_cal_path := path.join(CONFIG_DIR, "differential_calibration.json")): + DIFFERENTIAL_CALIBRATION_FILE = diff_cal_path else: - SENSOR_CALIBRATION_FILE = DEFAULT_CALIBRATION_FILE + DIFFERENTIAL_CALIBRATION_FILE = DEFAULT_DIFFERENTIAL_CALIBRATION_FILE -if path.exists(path.join(CONFIG_DIR, "sensor_definition.json")): - SENSOR_DEFINITION_FILE = path.join(CONFIG_DIR, "sensor_definition.json") +if path.exists(sensor_def_path := path.join(CONFIG_DIR, "sensor_definition.json")): + SENSOR_DEFINITION_FILE = sensor_def_path os.environ["SENSOR_CALIBRATION_FILE"] = SENSOR_CALIBRATION_FILE MEDIA_ROOT = path.join(REPO_ROOT, "files") PRESELECTOR_CONFIG = path.join(CONFIG_DIR, "preselector_config.json") From c0eafb4313f15372d76fd3abcf75ebdc8ecc9f9b Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 7 Feb 2024 16:29:17 -0500 Subject: [PATCH 05/56] convert calibration file string to path before passing --- src/initialization/sensor_loader.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index 0b8277a7..a09a2c84 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -199,14 +199,15 @@ def get_calibration( raise FileNotFoundError else: logger.debug(f"Loading calibration file: {cal_file_path}") - default = check_for_default_calibration( + is_default = check_for_default_calibration( cal_file_path, default_cal_file_path, cal_type ) # Create calibration object + cal_file_path = Path(cal_file_path) if cal_type.lower() == "sensor": - cal = SensorCalibration.from_json(cal_file_path, default) + cal = SensorCalibration.from_json(cal_file_path, is_default) elif cal_type.lower() == "differential": - cal = DifferentialCalibration.from_json(cal_file_path, default) + cal = DifferentialCalibration.from_json(cal_file_path, is_default) else: logger.error(f"Unknown calibration type: {cal_type}") raise ValueError From e0ff8a6e279e4016ee127b6a6fbd46f0ed5971fa Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 5 Mar 2024 12:52:32 -0500 Subject: [PATCH 06/56] Group related sigan variables, add auto calibration config --- env.template | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/env.template b/env.template index f78e123e..2bdf6157 100644 --- a/env.template +++ b/env.template @@ -19,7 +19,6 @@ ADMIN_PASSWORD=password # set to CERT to enable scos-sensor certificate authentication AUTHENTICATION=TOKEN -BASE_IMAGE=ghcr.io/ntia/scos-tekrsa/tekrsa_usb:0.2.3 # Default callback api/results # Set to CERT for certificate authentication CALLBACK_AUTHENTICATION=TOKEN @@ -34,8 +33,6 @@ CALLBACK_TIMEOUT=2 # Use either true or false DEBUG=true -DEVICE_MODEL=RSA507A - # Use latest as default for local development DOCKER_TAG=latest @@ -75,8 +72,12 @@ SCOS_SENSOR_GIT_TAG="$(git describe --tags)" # SECURITY WARNING: generate unique key with `manage.py generate_secret_key` SECRET_KEY="$(python3 -c 'import secrets; print(secrets.token_urlsafe(64))')" +# Signal analyzer selection/setup SIGAN_CLASS=TekRSASigan SIGAN_MODULE=scos_tekrsa.hardware.tekrsa_sigan +USB_DEVICE=Tektronix +DEVICE_MODEL=RSA507A +BASE_IMAGE=ghcr.io/ntia/scos-tekrsa/tekrsa_usb:0.2.3 # SECURITY WARNING: You should be using certs from a trusted authority. # If you don't have any, try letsencrypt or a similar service. @@ -86,8 +87,12 @@ SSL_CA_PATH=scos_test_ca.crt SSL_CERT_PATH=sensor01.pem SSL_KEY_PATH=sensor01.pem -USB_DEVICE=Tektronix - +# Calibration action selection +# The action specified here will be used to attempt an onboard +# sensor calibration on startup, if no onboard calibration data +# is available on startup. The specified action must be available. +CALIBRATE_ON_STARTUP=true +CALIBRATION_ACTION=SEA_CBRS_Calibrate_Baseline # Debug dependant settings if $DEBUG; then From a9999c7fec6997558b5e0897518a5108d9eafb7c Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 5 Mar 2024 12:53:16 -0500 Subject: [PATCH 07/56] remove default calibration files --- configs/default_differential_calibration.json | 7 ---- configs/default_sensor_calibration.json | 16 -------- src/initialization/sensor_loader.py | 39 +++++-------------- 3 files changed, 10 insertions(+), 52 deletions(-) delete mode 100644 configs/default_differential_calibration.json delete mode 100644 configs/default_sensor_calibration.json diff --git a/configs/default_differential_calibration.json b/configs/default_differential_calibration.json deleted file mode 100644 index 4fd30316..00000000 --- a/configs/default_differential_calibration.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "calibration_parameters": [], - "calibration_data": { - "loss": 0.0 - }, - "calibration_reference": "DEFAULT CALIBRATION" -} diff --git a/configs/default_sensor_calibration.json b/configs/default_sensor_calibration.json deleted file mode 100644 index 5b951871..00000000 --- a/configs/default_sensor_calibration.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "calibration_data":{ - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 0, - "noise_figure": 0.0, - "1db_compression_point": null, - "enbw": null, - "temperature": 26.85 - }, - "last_calibration_datetime": "1970-01-01T00:00:00.000000Z", - "calibration_parameters": [], - "clock_rate_lookup_by_sample_rate": [ - ], - "sensor_uid": "DEFAULT CALIBRATION", - "calibration_reference": "noise source output" -} diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index a09a2c84..f1878068 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -2,7 +2,7 @@ import logging from os import path from pathlib import Path -from typing import Union +from typing import Optional, Union from django.conf import settings from its_preselector.configuration_exception import ConfigurationException @@ -58,14 +58,9 @@ def load_sensor(sensor_capabilities: dict) -> Sensor: sensor_loc["z"] if "z" in sensor_loc else None, ) switches = load_switches(settings.SWITCH_CONFIGS_DIR) - sensor_cal = get_calibration( - settings.SENSOR_CALIBRATION_FILE, - settings.DEFAULT_SENSOR_CALIBRATION_FILE, - "sensor", - ) + sensor_cal = get_calibration(settings.SENSOR_CALIBRATION_FILE, "sensor") differential_cal = get_calibration( settings.DIFFERENTIAL_CALIBRATION_FILE, - settings.DEFAULT_DIFFERENTIAL_CALIBRATION_FILE, "differential", ) preselector = load_preselector( @@ -179,15 +174,14 @@ def load_preselector( def get_calibration( - cal_file_path: str, default_cal_file_path: str, cal_type: str -) -> Union[DifferentialCalibration, SensorCalibration]: + cal_file_path: str, cal_type: str +) -> Optional[Union[DifferentialCalibration, SensorCalibration]]: """ Load calibration data from file. :param cal_file_path: Path to the JSON calibration file. - :param default_cal_file_path: Path to the default calibration file. :param cal_type: Calibration type to load, either "sensor" or "differential" - :return: The ``Calibration`` object. + :return: The ``Calibration`` object, if loaded, or ``None`` if loading failed. """ try: cal = None @@ -199,32 +193,19 @@ def get_calibration( raise FileNotFoundError else: logger.debug(f"Loading calibration file: {cal_file_path}") - is_default = check_for_default_calibration( - cal_file_path, default_cal_file_path, cal_type - ) # Create calibration object cal_file_path = Path(cal_file_path) if cal_type.lower() == "sensor": - cal = SensorCalibration.from_json(cal_file_path, is_default) + cal = SensorCalibration.from_json(cal_file_path) elif cal_type.lower() == "differential": - cal = DifferentialCalibration.from_json(cal_file_path, is_default) + cal = DifferentialCalibration.from_json(cal_file_path) else: logger.error(f"Unknown calibration type: {cal_type}") raise ValueError except Exception: cal = None - logger.exception("Unable to load calibration file, reverting to none") + logger.exception( + f"Unable to load {cal_type} calibration file, reverting to none" + ) finally: return cal - - -def check_for_default_calibration( - cal_file_path: str, default_cal_path: str, cal_type: str -) -> bool: - default_cal = False - if cal_file_path == default_cal_path: - default_cal = True - logger.warning( - f"***************LOADING DEFAULT {cal_type} CALIBRATION***************" - ) - return default_cal From caa811a7b2c2b253410761617cbb9840ee4ea07f Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 5 Mar 2024 17:15:51 -0500 Subject: [PATCH 08/56] remove default calibration files and update cal loading --- src/initialization/sensor_loader.py | 38 ++++++++++++++++++++++------- src/sensor/settings.py | 38 ++++++++++++++++++----------- 2 files changed, 53 insertions(+), 23 deletions(-) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index f1878068..63e130ba 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -12,6 +12,7 @@ from scos_actions.calibration.differential_calibration import DifferentialCalibration from scos_actions.calibration.sensor_calibration import SensorCalibration from scos_actions.hardware.sensor import Sensor +from scos_actions.hardware.sigan_iface import SIGAN_SETTINGS_KEYS from scos_actions.metadata.utils import construct_geojson_point from utils.signals import register_component_with_status @@ -58,11 +59,6 @@ def load_sensor(sensor_capabilities: dict) -> Sensor: sensor_loc["z"] if "z" in sensor_loc else None, ) switches = load_switches(settings.SWITCH_CONFIGS_DIR) - sensor_cal = get_calibration(settings.SENSOR_CALIBRATION_FILE, "sensor") - differential_cal = get_calibration( - settings.DIFFERENTIAL_CALIBRATION_FILE, - "differential", - ) preselector = load_preselector( settings.PRESELECTOR_CONFIG, settings.PRESELECTOR_MODULE, @@ -87,6 +83,7 @@ def load_sensor(sensor_capabilities: dict) -> Sensor: except Exception as ex: logger.warning(f"unable to create signal analyzer: {ex}") + # Create sensor before handling calibrations sensor = Sensor( signal_analyzer=sigan, # TODO GPS Not Implemented @@ -94,9 +91,32 @@ def load_sensor(sensor_capabilities: dict) -> Sensor: preselector=preselector, switches=switches, location=location, - sensor_cal=sensor_cal, - differential_cal=differential_cal, + sensor_calibration=None, + differential_cal=None, ) + + # Calibration loading + if not settings.RUNNING_TESTS: + # Load the onboard cal file as the sensor calibration, if it exists + onboard_cal = get_calibration(settings.ONBOARD_CALIBRATION_FILE, "onboard") + if onboard_cal is not None: + sensor.sensor_calibration = onboard_cal + else: + # Otherwise, try using the sensor calibration file + sensor_cal = get_calibration(settings.SENSOR_CALIBRATION_FILE, "sensor") + if sensor_cal is not None: + sensor.sensor_calibration = sensor_cal + # Now run the calibration action defined in the environment + # TODO + # This will create an onboard_cal file if needed, and set it + # as the sensor's sensor_calibration. + # Now load the differential calibration, if it exists + differential_cal = get_calibration( + settings.DIFFERENTIAL_CALIBRATION_FILE, + "differential", + ) + sensor.differential_calibration = differential_cal + return sensor @@ -180,7 +200,7 @@ def get_calibration( Load calibration data from file. :param cal_file_path: Path to the JSON calibration file. - :param cal_type: Calibration type to load, either "sensor" or "differential" + :param cal_type: Calibration type to load: "onboard", "sensor" or "differential" :return: The ``Calibration`` object, if loaded, or ``None`` if loading failed. """ try: @@ -195,7 +215,7 @@ def get_calibration( logger.debug(f"Loading calibration file: {cal_file_path}") # Create calibration object cal_file_path = Path(cal_file_path) - if cal_type.lower() == "sensor": + if cal_type.lower() in ["sensor", "onboard"]: cal = SensorCalibration.from_json(cal_file_path) elif cal_type.lower() == "differential": cal = DifferentialCalibration.from_json(cal_file_path) diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 995f93c7..e220e000 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -69,29 +69,39 @@ ACTIONS_DIR = path.join(CONFIG_DIR, "actions") DRIVERS_DIR = path.join(REPO_ROOT, "drivers") -DEFAULT_SENSOR_CALIBRATION_FILE = path.join( - CONFIG_DIR, "default_sensor_calibration.json" -) -DEFAULT_DIFFERENTIAL_CALIBRATION_FILE = path.join( - CONFIG_DIR, "default_differential_calibration.json" -) -os.environ["DEFAULT_SENSOR_CALIBRATION_FILE"] = str(DEFAULT_SENSOR_CALIBRATION_FILE) -os.environ["DEFAULT_DIFFERENTIAL_CALIBRATION_FILE"] = str( - DEFAULT_DIFFERENTIAL_CALIBRATION_FILE -) -# JSON configs +########### Calibration Files ################## + +# Onboard calibration file; should only be autogenerated by actions. +# If present and parseable, this will be used in place of the SENSOR_CALIBRATION_FILE. +if path.exists( + onboard_cal_path := path.join(CONFIG_DIR, "onboard_sensor_calibration.json") +): + ONBOARD_SENSOR_CALIBRATION_FILE = onboard_cal_path +else: + ONBOARD_SENSOR_CALIBRATION_FILE = None +os.environ["ONBOARD_SENSOR_CALIBRATION_FILE"] = ONBOARD_SENSOR_CALIBRATION_FILE + +# Sensor calibration file; should be provided manually. Will not be +# overwritten when calibration actions. if path.exists(sensor_cal_path := path.join(CONFIG_DIR, "sensor_calibration.json")): SENSOR_CALIBRATION_FILE = sensor_cal_path else: - SENSOR_CALIBRATION_FILE = DEFAULT_SENSOR_CALIBRATION_FILE + SENSOR_CALIBRATION_FILE = None +os.environ["SENSOR_CALIBRATION_FILE"] = SENSOR_CALIBRATION_FILE + +# Differential calibration file; should be provided manually. If present, +# this will be used to apply additional scaling on top of the sensor or onboard +# calibration. This can be used to shift the data reference point from the onboard +# calibration terminal to another point in the signal path. if path.exists(diff_cal_path := path.join(CONFIG_DIR, "differential_calibration.json")): DIFFERENTIAL_CALIBRATION_FILE = diff_cal_path else: - DIFFERENTIAL_CALIBRATION_FILE = DEFAULT_DIFFERENTIAL_CALIBRATION_FILE + DIFFERENTIAL_CALIBRATION_FILE = None +os.environ["DIFFERENTIAL_CALIBRATION_FILE"] = DIFFERENTIAL_CALIBRATION_FILE +# Sensor Definition File if path.exists(sensor_def_path := path.join(CONFIG_DIR, "sensor_definition.json")): SENSOR_DEFINITION_FILE = sensor_def_path -os.environ["SENSOR_CALIBRATION_FILE"] = SENSOR_CALIBRATION_FILE MEDIA_ROOT = path.join(REPO_ROOT, "files") PRESELECTOR_CONFIG = path.join(CONFIG_DIR, "preselector_config.json") From 073b1ef5c6f368f0fcd9ec1c8486914339f49c34 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 5 Mar 2024 17:24:39 -0500 Subject: [PATCH 09/56] add calibrate_on_startup --- env.template | 2 +- src/initialization/sensor_loader.py | 30 ++++++++++++++++++++++------- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/env.template b/env.template index 2bdf6157..7bf49093 100644 --- a/env.template +++ b/env.template @@ -92,7 +92,7 @@ SSL_KEY_PATH=sensor01.pem # sensor calibration on startup, if no onboard calibration data # is available on startup. The specified action must be available. CALIBRATE_ON_STARTUP=true -CALIBRATION_ACTION=SEA_CBRS_Calibrate_Baseline +STARTUP_CALIBRATION_ACTION=SEA_CBRS_Calibrate_Baseline # Debug dependant settings if $DEBUG; then diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index 63e130ba..d1281033 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -5,6 +5,7 @@ from typing import Optional, Union from django.conf import settings +from environs import Env from its_preselector.configuration_exception import ConfigurationException from its_preselector.controlbyweb_web_relay import ControlByWebWebRelay from its_preselector.preselector import Preselector @@ -17,7 +18,10 @@ from utils.signals import register_component_with_status +from ..capabilities import actions_by_name + logger = logging.getLogger(__name__) +env = Env() class SensorLoader: @@ -72,9 +76,7 @@ def load_sensor(sensor_capabilities: dict) -> Sensor: check_for_required_sigan_settings() sigan_module_setting = settings.SIGAN_MODULE sigan_module = importlib.import_module(sigan_module_setting) - logger.info( - "Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE - ) + logger.info(f"Creating {settings.SIGAN_CLASS} from {settings.SIGAN_MODULE}") sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) sigan = sigan_constructor(switches=switches) register_component_with_status.send(sigan, component=sigan) @@ -106,10 +108,24 @@ def load_sensor(sensor_capabilities: dict) -> Sensor: sensor_cal = get_calibration(settings.SENSOR_CALIBRATION_FILE, "sensor") if sensor_cal is not None: sensor.sensor_calibration = sensor_cal - # Now run the calibration action defined in the environment - # TODO - # This will create an onboard_cal file if needed, and set it - # as the sensor's sensor_calibration. + + # Now run the calibration action defined in the environment + # This will create an onboard_cal file if needed, and set it + # as the sensor's sensor_calibration. + if env("CALIBRATE_ON_STARTUP") or sensor.sensor_calibration is None: + try: + cal_action = actions_by_name[env("STARTUP_CALIBRATION_ACTION")] + cal_action(sensor=sensor, schedule_entry=None, task_id=None) + except KeyError: + logger.exception( + f"Specified startup calibration action does not exist." + ) + else: + logger.debug( + "Skipping startup calibration since sensor_calibration exists and" + + "CALIBRATE_ON_STARTUP environment variable is False" + ) + # Now load the differential calibration, if it exists differential_cal = get_calibration( settings.DIFFERENTIAL_CALIBRATION_FILE, From 5c343d227ec548f71c7cdfdff22c37a4ffbad473 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 6 Mar 2024 12:06:19 -0500 Subject: [PATCH 10/56] avoid setting environment vars to None --- src/sensor/settings.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/sensor/settings.py b/src/sensor/settings.py index e220e000..476dd72a 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -72,14 +72,15 @@ ########### Calibration Files ################## # Onboard calibration file; should only be autogenerated by actions. -# If present and parseable, this will be used in place of the SENSOR_CALIBRATION_FILE. +# If present and parseable, gain and noise figure values will be used in +# place of any present in the SENSOR_CALIBRATION_FILE. if path.exists( onboard_cal_path := path.join(CONFIG_DIR, "onboard_sensor_calibration.json") ): ONBOARD_SENSOR_CALIBRATION_FILE = onboard_cal_path else: ONBOARD_SENSOR_CALIBRATION_FILE = None -os.environ["ONBOARD_SENSOR_CALIBRATION_FILE"] = ONBOARD_SENSOR_CALIBRATION_FILE +os.environ["ONBOARD_SENSOR_CALIBRATION_FILE"] = onboard_cal_path # Sensor calibration file; should be provided manually. Will not be # overwritten when calibration actions. @@ -87,7 +88,7 @@ SENSOR_CALIBRATION_FILE = sensor_cal_path else: SENSOR_CALIBRATION_FILE = None -os.environ["SENSOR_CALIBRATION_FILE"] = SENSOR_CALIBRATION_FILE +os.environ["SENSOR_CALIBRATION_FILE"] = sensor_cal_path # Differential calibration file; should be provided manually. If present, # this will be used to apply additional scaling on top of the sensor or onboard @@ -97,7 +98,7 @@ DIFFERENTIAL_CALIBRATION_FILE = diff_cal_path else: DIFFERENTIAL_CALIBRATION_FILE = None -os.environ["DIFFERENTIAL_CALIBRATION_FILE"] = DIFFERENTIAL_CALIBRATION_FILE +os.environ["DIFFERENTIAL_CALIBRATION_FILE"] = diff_cal_path # Sensor Definition File if path.exists(sensor_def_path := path.join(CONFIG_DIR, "sensor_definition.json")): From 325b8bcc0185be9da74a19246a402936b7bd89fa Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 6 Mar 2024 12:19:21 -0500 Subject: [PATCH 11/56] remove unused import --- src/initialization/sensor_loader.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index d1281033..e3f1a1ef 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -13,7 +13,6 @@ from scos_actions.calibration.differential_calibration import DifferentialCalibration from scos_actions.calibration.sensor_calibration import SensorCalibration from scos_actions.hardware.sensor import Sensor -from scos_actions.hardware.sigan_iface import SIGAN_SETTINGS_KEYS from scos_actions.metadata.utils import construct_geojson_point from utils.signals import register_component_with_status From bc5969c8215fcb7bf3a463ca82c319845a342e27 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 6 Mar 2024 12:30:42 -0500 Subject: [PATCH 12/56] fix sensor loader initialization and pass loaded actions dictionary to constructor --- src/initialization/__init__.py | 4 +++- src/initialization/sensor_loader.py | 14 ++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index e97def1e..633a51d0 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -50,7 +50,9 @@ def set_container_unhealthy(): logger.debug(f"Actions ActionLoader has {len(action_loader.actions)} actions") capabilities_loader = CapabilitiesLoader() logger.debug("Calling sensor loader.") - sensor_loader = SensorLoader(capabilities_loader.capabilities) + sensor_loader = SensorLoader( + capabilities_loader.capabilities, action_loader.actions + ) if ( not settings.RUNNING_MIGRATIONS and not sensor_loader.sensor.signal_analyzer.healthy() diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index e3f1a1ef..f938ae7b 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -17,8 +17,6 @@ from utils.signals import register_component_with_status -from ..capabilities import actions_by_name - logger = logging.getLogger(__name__) env = Env() @@ -26,14 +24,14 @@ class SensorLoader: _instance = None - def __init__(self, sensor_capabilities: dict): + def __init__(self, sensor_capabilities: dict, sensor_actions: dict): if not hasattr(self, "sensor"): logger.debug("Sensor has not been loaded. Loading...") - self._sensor = load_sensor(sensor_capabilities) + self._sensor = load_sensor(sensor_capabilities, sensor_actions) else: logger.debug("Already loaded sensor. ") - def __new__(cls, sensor_capabilities): + def __new__(cls, sensor_capabilities: dict, sensor_actions: dict): if cls._instance is None: logger.debug("Creating the SensorLoader") cls._instance = super().__new__(cls) @@ -44,7 +42,7 @@ def sensor(self) -> Sensor: return self._sensor -def load_sensor(sensor_capabilities: dict) -> Sensor: +def load_sensor(sensor_capabilities: dict, sensor_actions: dict) -> Sensor: switches = {} sensor_cal = None differential_cal = None @@ -92,7 +90,7 @@ def load_sensor(sensor_capabilities: dict) -> Sensor: preselector=preselector, switches=switches, location=location, - sensor_calibration=None, + sensor_cal=None, differential_cal=None, ) @@ -113,7 +111,7 @@ def load_sensor(sensor_capabilities: dict) -> Sensor: # as the sensor's sensor_calibration. if env("CALIBRATE_ON_STARTUP") or sensor.sensor_calibration is None: try: - cal_action = actions_by_name[env("STARTUP_CALIBRATION_ACTION")] + cal_action = sensor_actions[env("STARTUP_CALIBRATION_ACTION")] cal_action(sensor=sensor, schedule_entry=None, task_id=None) except KeyError: logger.exception( From b5bb922ce23413cb59c0d105dd29305145c310d2 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 6 Mar 2024 12:48:00 -0500 Subject: [PATCH 13/56] get sensor cal from sensor instead of sigan --- src/status/views.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/status/views.py b/src/status/views.py index db3abfdb..059f3a3e 100644 --- a/src/status/views.py +++ b/src/status/views.py @@ -62,14 +62,11 @@ def status(request, version, format=None): } if ( sensor_loader.sensor is not None - and sensor_loader.sensor.signal_analyzer is not None - and sensor_loader.sensor.signal_analyzer.sensor_calibration is not None + and sensor_loader.sensor.sensor_calibration is not None ): status_json[ "last_calibration_datetime" - ] = ( - sensor_loader.sensor.signal_analyzer.sensor_calibration.last_calibration_datetime - ) + ] = sensor_loader.sensor.sensor_calibration.last_calibration_datetime for component in status_monitor.status_components: component_status = component.get_status() if isinstance(component, WebRelay): From e139738db1dc510f8174db14f005f1ca44833e03 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 14 Mar 2024 08:00:23 -0600 Subject: [PATCH 14/56] Fix ONBOARD_CALIBRATION_FILE setting. --- src/sensor/settings.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 476dd72a..3da4a24f 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -77,10 +77,10 @@ if path.exists( onboard_cal_path := path.join(CONFIG_DIR, "onboard_sensor_calibration.json") ): - ONBOARD_SENSOR_CALIBRATION_FILE = onboard_cal_path + ONBOARD_CALIBRATION_FILE = onboard_cal_path else: - ONBOARD_SENSOR_CALIBRATION_FILE = None -os.environ["ONBOARD_SENSOR_CALIBRATION_FILE"] = onboard_cal_path + ONBOARD_CALIBRATION_FILE = None +os.environ["ONBOARD_CALIBRATION_FILE"] = onboard_cal_path # Sensor calibration file; should be provided manually. Will not be # overwritten when calibration actions. From 9576cbebf7608bba4deaf94c59688b5345adec8d Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 14 Mar 2024 08:35:32 -0600 Subject: [PATCH 15/56] Add CALIBRATE_ON_STARTUP setting defaulting to false. --- src/initialization/sensor_loader.py | 2 +- src/sensor/settings.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index f938ae7b..c8064476 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -109,7 +109,7 @@ def load_sensor(sensor_capabilities: dict, sensor_actions: dict) -> Sensor: # Now run the calibration action defined in the environment # This will create an onboard_cal file if needed, and set it # as the sensor's sensor_calibration. - if env("CALIBRATE_ON_STARTUP") or sensor.sensor_calibration is None: + if settings.CALIBRATE_ON_STARTUP or sensor.sensor_calibration is None: try: cal_action = sensor_actions[env("STARTUP_CALIBRATION_ACTION")] cal_action(sensor=sensor, schedule_entry=None, task_id=None) diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 3da4a24f..44da7d1a 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -455,3 +455,4 @@ MAX_FAILURES = env("MAX_FAILURES", default=2) os.environ["RUNNING_TESTS"] = str(RUNNING_TESTS) USB_DEVICE = env("USB_DEVICE", default=None) +CALIBRATE_ON_STARTUP = env.bool("CALIBRATE_ON_STARTUP", default=False) \ No newline at end of file From 9175e8861592ff0cad6abb2d79a70fac9b167bdd Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 14 Mar 2024 08:37:10 -0600 Subject: [PATCH 16/56] Add CALIBRATE_ON_STARTUP to docker-compose.yml. --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index db9f3a60..a08f8d47 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,6 +38,7 @@ services: - ADDITIONAL_USER_NAMES - ADDITIONAL_USER_PASSWORD - AUTHENTICATION + - CALIBRATE_ON_STARTUP - CALLBACK_AUTHENTICATION - CALLBACK_SSL_VERIFICATION - CALLBACK_TIMEOUT From 3829f7d1a048e705d4929ecce5c649c9a88a3e23 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 14 Mar 2024 09:18:33 -0600 Subject: [PATCH 17/56] Use setting for STARTUP_CALIBRATION_ACTION and don't raise exception if not set and no calibration provided. --- docker-compose.yml | 1 + src/initialization/sensor_loader.py | 10 ++++------ src/sensor/settings.py | 3 ++- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index a08f8d47..82f510f9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -64,6 +64,7 @@ services: - SIGAN_CLASS - SIGAN_POWER_SWITCH - SIGAN_POWER_CYCLE_STATES + - STARTUP_CALIBRATION_ACTION - RUNNING_MIGRATIONS - USB_DEVICE expose: diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index c8064476..4745595b 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -110,13 +110,11 @@ def load_sensor(sensor_capabilities: dict, sensor_actions: dict) -> Sensor: # This will create an onboard_cal file if needed, and set it # as the sensor's sensor_calibration. if settings.CALIBRATE_ON_STARTUP or sensor.sensor_calibration is None: - try: - cal_action = sensor_actions[env("STARTUP_CALIBRATION_ACTION")] + if settings.STARTUP_CALIBRATION_ACTION is None: + logger.error("No STARTUP_CALIBRATION_ACTION set.") + else: + cal_action = sensor_actions[settings.STARTUP_CALIBRATION_ACTION] cal_action(sensor=sensor, schedule_entry=None, task_id=None) - except KeyError: - logger.exception( - f"Specified startup calibration action does not exist." - ) else: logger.debug( "Skipping startup calibration since sensor_calibration exists and" diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 44da7d1a..31e75cf1 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -455,4 +455,5 @@ MAX_FAILURES = env("MAX_FAILURES", default=2) os.environ["RUNNING_TESTS"] = str(RUNNING_TESTS) USB_DEVICE = env("USB_DEVICE", default=None) -CALIBRATE_ON_STARTUP = env.bool("CALIBRATE_ON_STARTUP", default=False) \ No newline at end of file +CALIBRATE_ON_STARTUP = env.bool("CALIBRATE_ON_STARTUP", default=False) +STARTUP_CALIBRATION_ACTION = env("CALIBRATE_ON_STARTUP", default=None) From 75e64c8cb201b0fc6a0afc530a4edd5cdb364f68 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 14 Mar 2024 09:29:35 -0600 Subject: [PATCH 18/56] Set ONBOARD_CALIBRATION_FILE whether it exists or not. --- src/sensor/settings.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 31e75cf1..63e48a7b 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -74,13 +74,8 @@ # Onboard calibration file; should only be autogenerated by actions. # If present and parseable, gain and noise figure values will be used in # place of any present in the SENSOR_CALIBRATION_FILE. -if path.exists( - onboard_cal_path := path.join(CONFIG_DIR, "onboard_sensor_calibration.json") -): - ONBOARD_CALIBRATION_FILE = onboard_cal_path -else: - ONBOARD_CALIBRATION_FILE = None -os.environ["ONBOARD_CALIBRATION_FILE"] = onboard_cal_path +ONBOARD_CALIBRATION_FILE = path.join(CONFIG_DIR, "onboard_sensor_calibration.json") +os.environ["ONBOARD_CALIBRATION_FILE"] = ONBOARD_CALIBRATION_FILE # Sensor calibration file; should be provided manually. Will not be # overwritten when calibration actions. From 33f8200de77b21726e849fe52183cc51b8143492 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 14 Mar 2024 13:13:45 -0600 Subject: [PATCH 19/56] Cal on startup if calibration is expired. --- env.template | 2 +- src/initialization/sensor_loader.py | 2 +- src/sensor/settings.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/env.template b/env.template index 7bf49093..0b0b4fd9 100644 --- a/env.template +++ b/env.template @@ -18,7 +18,7 @@ ADMIN_PASSWORD=password # set to CERT to enable scos-sensor certificate authentication AUTHENTICATION=TOKEN - +CALIBRATION_EXPIRATION_LIMIT=360 # Default callback api/results # Set to CERT for certificate authentication CALLBACK_AUTHENTICATION=TOKEN diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index 4745595b..da21b5ea 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -109,7 +109,7 @@ def load_sensor(sensor_capabilities: dict, sensor_actions: dict) -> Sensor: # Now run the calibration action defined in the environment # This will create an onboard_cal file if needed, and set it # as the sensor's sensor_calibration. - if settings.CALIBRATE_ON_STARTUP or sensor.sensor_calibration is None: + if sensor.sensor_calibration is None or sensor.sensor_calibration.expired(): if settings.STARTUP_CALIBRATION_ACTION is None: logger.error("No STARTUP_CALIBRATION_ACTION set.") else: diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 63e48a7b..4ad9ae57 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -450,5 +450,5 @@ MAX_FAILURES = env("MAX_FAILURES", default=2) os.environ["RUNNING_TESTS"] = str(RUNNING_TESTS) USB_DEVICE = env("USB_DEVICE", default=None) -CALIBRATE_ON_STARTUP = env.bool("CALIBRATE_ON_STARTUP", default=False) +CALIBRATION_EXPIRATION_LIMIT = env("CALIBRATION_EXPIRATION_LIMIT", default=None) STARTUP_CALIBRATION_ACTION = env("CALIBRATE_ON_STARTUP", default=None) From bdf6b9f2912a787226233046f86eb3d5244e0dcd Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 14 Mar 2024 13:20:20 -0600 Subject: [PATCH 20/56] Add CALIBRATION_EXPIRATION_LIMIT to docker-compose.yml and remove from settings since it is used in scos-actions. --- docker-compose.yml | 2 +- src/sensor/settings.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 82f510f9..bc0c2264 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,7 +38,7 @@ services: - ADDITIONAL_USER_NAMES - ADDITIONAL_USER_PASSWORD - AUTHENTICATION - - CALIBRATE_ON_STARTUP + - CALIBRATION_EXPIRATION_LIMIT - CALLBACK_AUTHENTICATION - CALLBACK_SSL_VERIFICATION - CALLBACK_TIMEOUT diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 4ad9ae57..1688c188 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -450,5 +450,4 @@ MAX_FAILURES = env("MAX_FAILURES", default=2) os.environ["RUNNING_TESTS"] = str(RUNNING_TESTS) USB_DEVICE = env("USB_DEVICE", default=None) -CALIBRATION_EXPIRATION_LIMIT = env("CALIBRATION_EXPIRATION_LIMIT", default=None) STARTUP_CALIBRATION_ACTION = env("CALIBRATE_ON_STARTUP", default=None) From 19ac7a49a0ca118cf8753b850d1d4785352f915a Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 14 Mar 2024 13:25:46 -0600 Subject: [PATCH 21/56] Fix STARTUP_CALIBRATION_ACTION setting. --- src/sensor/settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 1688c188..da10b995 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -450,4 +450,5 @@ MAX_FAILURES = env("MAX_FAILURES", default=2) os.environ["RUNNING_TESTS"] = str(RUNNING_TESTS) USB_DEVICE = env("USB_DEVICE", default=None) -STARTUP_CALIBRATION_ACTION = env("CALIBRATE_ON_STARTUP", default=None) +CALIBRATION_EXPIRATION_LIMIT = env("CALIBRATION_EXPIRATION_LIMIT") +STARTUP_CALIBRATION_ACTION = env("STARTUP_CALIBRATION_ACTION", default=None) From a47087fcc091a61b549eaf4b46e4c42269b3de13 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 14 Mar 2024 13:29:03 -0600 Subject: [PATCH 22/56] Remove "CALIBRATION_EXPIRATION_LIMIT" from settings. --- src/sensor/settings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sensor/settings.py b/src/sensor/settings.py index da10b995..848aa81a 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -450,5 +450,4 @@ MAX_FAILURES = env("MAX_FAILURES", default=2) os.environ["RUNNING_TESTS"] = str(RUNNING_TESTS) USB_DEVICE = env("USB_DEVICE", default=None) -CALIBRATION_EXPIRATION_LIMIT = env("CALIBRATION_EXPIRATION_LIMIT") STARTUP_CALIBRATION_ACTION = env("STARTUP_CALIBRATION_ACTION", default=None) From 0bceffe6225ff2eeaf0698b482fde5b2a53f855d Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 14 Mar 2024 15:02:18 -0600 Subject: [PATCH 23/56] Only update cal when actually running. --- src/initialization/sensor_loader.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index da21b5ea..6da5e17f 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -109,17 +109,18 @@ def load_sensor(sensor_capabilities: dict, sensor_actions: dict) -> Sensor: # Now run the calibration action defined in the environment # This will create an onboard_cal file if needed, and set it # as the sensor's sensor_calibration. - if sensor.sensor_calibration is None or sensor.sensor_calibration.expired(): - if settings.STARTUP_CALIBRATION_ACTION is None: - logger.error("No STARTUP_CALIBRATION_ACTION set.") + if not settings.RUNNING_MIGRATIONS: + if sensor.sensor_calibration is None or sensor.sensor_calibration.expired(): + if settings.STARTUP_CALIBRATION_ACTION is None: + logger.error("No STARTUP_CALIBRATION_ACTION set.") + else: + cal_action = sensor_actions[settings.STARTUP_CALIBRATION_ACTION] + cal_action(sensor=sensor, schedule_entry=None, task_id=None) else: - cal_action = sensor_actions[settings.STARTUP_CALIBRATION_ACTION] - cal_action(sensor=sensor, schedule_entry=None, task_id=None) - else: - logger.debug( - "Skipping startup calibration since sensor_calibration exists and" - + "CALIBRATE_ON_STARTUP environment variable is False" - ) + logger.debug( + "Skipping startup calibration since sensor_calibration exists and" + + "CALIBRATE_ON_STARTUP environment variable is False" + ) # Now load the differential calibration, if it exists differential_cal = get_calibration( From d0c24cc7d78b7c4e444e6333823a8eace7e93951 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 14 Mar 2024 15:51:41 -0600 Subject: [PATCH 24/56] Merge in autohealing fixes. Move cal on startup check to initialization/__init__.py to ensure healthy signal analyzer. --- src/initialization/__init__.py | 222 +++++++++++++++++++++++----- src/initialization/action_loader.py | 2 +- src/initialization/sensor_loader.py | 181 +++-------------------- src/initialization/utils.py | 24 +++ 4 files changed, 231 insertions(+), 198 deletions(-) create mode 100644 src/initialization/utils.py diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 633a51d0..05b118a7 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -1,31 +1,89 @@ +import importlib import logging -import sys -import types +import time +from os import path from pathlib import Path -from subprocess import check_output from django.conf import settings - +from its_preselector.configuration_exception import ConfigurationException +from its_preselector.controlbyweb_web_relay import ControlByWebWebRelay +from its_preselector.preselector import Preselector +from scos_actions.calibration.differential_calibration import DifferentialCalibration +from scos_actions.calibration.sensor_calibration import SensorCalibration +from scos_actions.hardware.utils import power_cycle_sigan +from scos_actions.utils import load_from_json +from typing import Optional, Union from utils.signals import register_component_with_status from .action_loader import ActionLoader from .capabilities_loader import CapabilitiesLoader from .sensor_loader import SensorLoader from .status_monitor import StatusMonitor +from .utils import get_usb_device_exists, set_container_unhealthy logger = logging.getLogger(__name__) status_monitor = StatusMonitor() -def get_usb_device_exists() -> bool: - logger.debug("Checking for USB...") - if not settings.RUNNING_TESTS and settings.USB_DEVICE is not None: - usb_devices = check_output("lsusb").decode(sys.stdout.encoding) - logger.debug("Checking for " + settings.USB_DEVICE) - logger.debug("Found " + usb_devices) - return settings.USB_DEVICE in usb_devices - return True +def load_preselector_from_file( + preselector_module, preselector_class, preselector_config_file: Path +): + if preselector_config_file is None: + return None + else: + try: + preselector_config = load_from_json(preselector_config_file) + return load_preselector( + preselector_config, preselector_module, preselector_class + ) + except ConfigurationException: + logger.exception( + f"Unable to create preselector defined in: {preselector_config_file}" + ) + return None + + +def load_preselector( + preselector_config: str, + module: str, + preselector_class_name: str, + sensor_definition: dict, +) -> Preselector: + logger.debug( + f"loading {preselector_class_name} from {module} with config: {preselector_config}" + ) + if module is not None and preselector_class_name is not None: + preselector_module = importlib.import_module(module) + preselector_constructor = getattr(preselector_module, preselector_class_name) + preselector_config = load_from_json(preselector_config) + ps = preselector_constructor(sensor_definition, preselector_config) + register_component_with_status.send(ps, component=ps) + else: + ps = None + return ps + + +def load_switches(switch_dir: Path) -> dict: + logger.debug(f"Loading switches in {switch_dir}") + switch_dict = {} + try: + if switch_dir is not None and switch_dir.is_dir(): + for f in switch_dir.iterdir(): + file_path = f.resolve() + logger.debug(f"loading switch config {file_path}") + conf = load_from_json(file_path) + try: + switch = ControlByWebWebRelay(conf) + logger.debug(f"Adding {switch.id}") + switch_dict[switch.id] = switch + logger.debug(f"Registering switch status for {switch.name}") + register_component_with_status.send(__name__, component=switch) + except ConfigurationException: + logger.error(f"Unable to configure switch defined in: {file_path}") + except Exception as ex: + logger.error(f"Unable to load switches {ex}") + return switch_dict def status_registration_handler(sender, **kwargs): @@ -42,34 +100,126 @@ def set_container_unhealthy(): Path(settings.SDR_HEALTHCHECK_FILE).touch() +def get_calibration( + cal_file_path: str, cal_type: str +) -> Optional[Union[DifferentialCalibration, SensorCalibration]]: + """ + Load calibration data from file. + + :param cal_file_path: Path to the JSON calibration file. + :param cal_type: Calibration type to load: "onboard", "sensor" or "differential" + :return: The ``Calibration`` object, if loaded, or ``None`` if loading failed. + """ + try: + cal = None + if cal_file_path is None or cal_file_path == "": + logger.error("No calibration file specified.") + raise ValueError + elif not path.exists(cal_file_path): + logger.error(f"{cal_file_path} does not exist.") + raise FileNotFoundError + else: + logger.debug(f"Loading calibration file: {cal_file_path}") + # Create calibration object + cal_file_path = Path(cal_file_path) + if cal_type.lower() in ["sensor", "onboard"]: + cal = SensorCalibration.from_json(cal_file_path) + elif cal_type.lower() == "differential": + cal = DifferentialCalibration.from_json(cal_file_path) + else: + logger.error(f"Unknown calibration type: {cal_type}") + raise ValueError + except Exception: + cal = None + logger.exception( + f"Unable to load {cal_type} calibration file, reverting to none" + ) + finally: + return cal + + try: + sensor_loader = None register_component_with_status.connect(status_registration_handler) - usb_device_exists = get_usb_device_exists() - if usb_device_exists: - action_loader = ActionLoader() - logger.debug(f"Actions ActionLoader has {len(action_loader.actions)} actions") - capabilities_loader = CapabilitiesLoader() - logger.debug("Calling sensor loader.") + action_loader = ActionLoader() + logger.debug(f"Actions ActionLoader has {len(action_loader.actions)} actions") + capabilities_loader = CapabilitiesLoader() + switches = load_switches(settings.SWITCH_CONFIGS_DIR) + preselector = load_preselector( + settings.PRESELECTOR_CONFIG, + settings.PRESELECTOR_MODULE, + settings.PRESELECTOR_CLASS, + capabilities_loader.capabilities["sensor"], + ) + + if get_usb_device_exists(): + logger.debug("Initializing Sensor...") sensor_loader = SensorLoader( - capabilities_loader.capabilities, action_loader.actions + capabilities_loader.capabilities, switches, preselector ) + + else: + logger.debug("Power cycling sigan") + try: + power_cycle_sigan(switches) + except Exception as power_cycle_exception: + logger.error(f"Unable to power cycle sigan: {power_cycle_exception}") + set_container_unhealthy() + time.sleep(60) + + if not settings.RUNNING_MIGRATIONS: if ( - not settings.RUNNING_MIGRATIONS - and not sensor_loader.sensor.signal_analyzer.healthy() + sensor_loader.sensor.signal_analyzer is None + or not sensor_loader.sensor.signal_analyzer.healthy() ): + try: + power_cycle_sigan(switches) + except Exception as power_cycle_exception: + logger.error(f"Unable to power cycle sigan: {power_cycle_exception}") set_container_unhealthy() - else: - action_loader = types.SimpleNamespace() - action_loader.actions = {} - capabilities_loader = types.SimpleNamespace() - capabilities_loader.capabilities = {} - sensor_loader = types.SimpleNamespace() - sensor_loader.sensor = types.SimpleNamespace() - sensor_loader.sensor.signal_analyzer = None - sensor_loader.preselector = None - sensor_loader.switches = {} - sensor_loader.capabilities = {} - logger.warning("Usb is not ready. Marking container as unhealthy") - set_container_unhealthy() -except: - logger.exception("Error during initialization") + time.sleep(60) + + # Calibration loading + if not settings.RUNNING_TESTS: + # Load the onboard cal file as the sensor calibration, if it exists + onboard_cal = get_calibration(settings.ONBOARD_CALIBRATION_FILE, "onboard") + if onboard_cal is not None: + sensor_loader.sensor.sensor_calibration = onboard_cal + else: + # Otherwise, try using the sensor calibration file + sensor_cal = get_calibration(settings.SENSOR_CALIBRATION_FILE, "sensor") + if sensor_cal is not None: + sensor_loader.sensor.sensor_calibration = sensor_cal + + # Now run the calibration action defined in the environment + # This will create an onboard_cal file if needed, and set it + # as the sensor's sensor_calibration. + if not settings.RUNNING_MIGRATIONS: + if sensor_loader.sensor.sensor_calibration is None or sensor_loader.sensor.sensor_calibration.expired(): + if settings.STARTUP_CALIBRATION_ACTION is None: + logger.error("No STARTUP_CALIBRATION_ACTION set.") + else: + cal_action = action_loader.actions[settings.STARTUP_CALIBRATION_ACTION] + cal_action(sensor=sensor_loader.sensor, schedule_entry=None, task_id=None) + else: + logger.debug( + "Skipping startup calibration since sensor_calibration exists and" + + "CALIBRATE_ON_STARTUP environment variable is False" + ) + + # Now load the differential calibration, if it exists + differential_cal = get_calibration( + settings.DIFFERENTIAL_CALIBRATION_FILE, + "differential", + ) + sensor_loader.sensor.differential_calibration = differential_cal + + import ray + if settings.RAY_INIT and not ray.is_initialized(): + # Dashboard is only enabled if ray[default] is installed + logger.debug("Initializing ray.") + ray.init() +except BaseException as error: + logger.exception(f"Error during initialization: {error}") + set_container_unhealthy() + diff --git a/src/initialization/action_loader.py b/src/initialization/action_loader.py index f94fb5a1..8b98b3f8 100644 --- a/src/initialization/action_loader.py +++ b/src/initialization/action_loader.py @@ -4,12 +4,12 @@ import os import pkgutil import shutil -from typing import Dict from django.conf import settings from scos_actions.actions import action_classes from scos_actions.actions.interfaces.action import Action from scos_actions.discover import init, test_actions +from typing import Dict logger = logging.getLogger(__name__) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index 6da5e17f..e579907e 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -1,22 +1,15 @@ import importlib import logging -from os import path -from pathlib import Path -from typing import Optional, Union from django.conf import settings from environs import Env -from its_preselector.configuration_exception import ConfigurationException -from its_preselector.controlbyweb_web_relay import ControlByWebWebRelay from its_preselector.preselector import Preselector -from scos_actions import utils -from scos_actions.calibration.differential_calibration import DifferentialCalibration -from scos_actions.calibration.sensor_calibration import SensorCalibration from scos_actions.hardware.sensor import Sensor from scos_actions.metadata.utils import construct_geojson_point - from utils.signals import register_component_with_status +from .utils import get_usb_device_exists, set_container_unhealthy + logger = logging.getLogger(__name__) env = Env() @@ -24,14 +17,16 @@ class SensorLoader: _instance = None - def __init__(self, sensor_capabilities: dict, sensor_actions: dict): + def __init__( + self, sensor_capabilities: dict, switches: dict, preselector: Preselector + ): if not hasattr(self, "sensor"): logger.debug("Sensor has not been loaded. Loading...") - self._sensor = load_sensor(sensor_capabilities, sensor_actions) + self._sensor = load_sensor(sensor_capabilities,switches, preselector) else: logger.debug("Already loaded sensor. ") - def __new__(cls, sensor_capabilities: dict, sensor_actions: dict): + def __new__(cls, sensor_capabilities: dict,switches: dict, preselector: Preselector): if cls._instance is None: logger.debug("Creating the SensorLoader") cls._instance = super().__new__(cls) @@ -42,8 +37,7 @@ def sensor(self) -> Sensor: return self._sensor -def load_sensor(sensor_capabilities: dict, sensor_actions: dict) -> Sensor: - switches = {} +def load_sensor(sensor_capabilities: dict, switches: dict, preselector: Preselector, sensor_actions: dict) -> Sensor: sensor_cal = None differential_cal = None preselector = None @@ -59,28 +53,25 @@ def load_sensor(sensor_capabilities: dict, sensor_actions: dict) -> Sensor: sensor_loc["y"], sensor_loc["z"] if "z" in sensor_loc else None, ) - switches = load_switches(settings.SWITCH_CONFIGS_DIR) - preselector = load_preselector( - settings.PRESELECTOR_CONFIG, - settings.PRESELECTOR_MODULE, - settings.PRESELECTOR_CLASS, - sensor_capabilities["sensor"], - ) sigan = None try: if not settings.RUNNING_MIGRATIONS: - check_for_required_sigan_settings() - sigan_module_setting = settings.SIGAN_MODULE - sigan_module = importlib.import_module(sigan_module_setting) - logger.info(f"Creating {settings.SIGAN_CLASS} from {settings.SIGAN_MODULE}") - sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) - sigan = sigan_constructor(switches=switches) - register_component_with_status.send(sigan, component=sigan) + if get_usb_device_exists(): + check_for_required_sigan_settings() + sigan_module_setting = settings.SIGAN_MODULE + sigan_module = importlib.import_module(sigan_module_setting) + logger.info(f"Creating {settings.SIGAN_CLASS} from {settings.SIGAN_MODULE}") + sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) + sigan = sigan_constructor(switches=switches) + register_component_with_status.send(sigan, component=sigan) + else: + logger.warning("Required USB Device does not exist.") else: logger.info("Running migrations. Not loading signal analyzer.") - except Exception as ex: + except BaseException as ex: logger.warning(f"unable to create signal analyzer: {ex}") + set_container_unhealthy() # Create sensor before handling calibrations sensor = Sensor( @@ -93,42 +84,6 @@ def load_sensor(sensor_capabilities: dict, sensor_actions: dict) -> Sensor: sensor_cal=None, differential_cal=None, ) - - # Calibration loading - if not settings.RUNNING_TESTS: - # Load the onboard cal file as the sensor calibration, if it exists - onboard_cal = get_calibration(settings.ONBOARD_CALIBRATION_FILE, "onboard") - if onboard_cal is not None: - sensor.sensor_calibration = onboard_cal - else: - # Otherwise, try using the sensor calibration file - sensor_cal = get_calibration(settings.SENSOR_CALIBRATION_FILE, "sensor") - if sensor_cal is not None: - sensor.sensor_calibration = sensor_cal - - # Now run the calibration action defined in the environment - # This will create an onboard_cal file if needed, and set it - # as the sensor's sensor_calibration. - if not settings.RUNNING_MIGRATIONS: - if sensor.sensor_calibration is None or sensor.sensor_calibration.expired(): - if settings.STARTUP_CALIBRATION_ACTION is None: - logger.error("No STARTUP_CALIBRATION_ACTION set.") - else: - cal_action = sensor_actions[settings.STARTUP_CALIBRATION_ACTION] - cal_action(sensor=sensor, schedule_entry=None, task_id=None) - else: - logger.debug( - "Skipping startup calibration since sensor_calibration exists and" - + "CALIBRATE_ON_STARTUP environment variable is False" - ) - - # Now load the differential calibration, if it exists - differential_cal = get_calibration( - settings.DIFFERENTIAL_CALIBRATION_FILE, - "differential", - ) - sensor.differential_calibration = differential_cal - return sensor @@ -145,99 +100,3 @@ def check_for_required_sigan_settings(): raise Exception(error) -def load_switches(switch_dir: Path) -> dict: - logger.debug(f"Loading switches in {switch_dir}") - switch_dict = {} - try: - if switch_dir is not None and switch_dir.is_dir(): - for f in switch_dir.iterdir(): - file_path = f.resolve() - logger.debug(f"loading switch config {file_path}") - conf = utils.load_from_json(file_path) - try: - switch = ControlByWebWebRelay(conf) - logger.debug(f"Adding {switch.id}") - switch_dict[switch.id] = switch - logger.debug(f"Registering switch status for {switch.name}") - register_component_with_status.send(__name__, component=switch) - except ConfigurationException: - logger.error(f"Unable to configure switch defined in: {file_path}") - except Exception as ex: - logger.error(f"Unable to load switches {ex}") - return switch_dict - - -def load_preselector_from_file( - preselector_module, preselector_class, preselector_config_file: Path -): - if preselector_config_file is None: - return None - else: - try: - preselector_config = utils.load_from_json(preselector_config_file) - return load_preselector( - preselector_config, preselector_module, preselector_class - ) - except ConfigurationException: - logger.exception( - f"Unable to create preselector defined in: {preselector_config_file}" - ) - return None - - -def load_preselector( - preselector_config: str, - module: str, - preselector_class_name: str, - sensor_definition: dict, -) -> Preselector: - logger.debug( - f"loading {preselector_class_name} from {module} with config: {preselector_config}" - ) - if module is not None and preselector_class_name is not None: - preselector_module = importlib.import_module(module) - preselector_constructor = getattr(preselector_module, preselector_class_name) - preselector_config = utils.load_from_json(preselector_config) - ps = preselector_constructor(sensor_definition, preselector_config) - register_component_with_status.send(ps, component=ps) - else: - ps = None - return ps - - -def get_calibration( - cal_file_path: str, cal_type: str -) -> Optional[Union[DifferentialCalibration, SensorCalibration]]: - """ - Load calibration data from file. - - :param cal_file_path: Path to the JSON calibration file. - :param cal_type: Calibration type to load: "onboard", "sensor" or "differential" - :return: The ``Calibration`` object, if loaded, or ``None`` if loading failed. - """ - try: - cal = None - if cal_file_path is None or cal_file_path == "": - logger.error("No calibration file specified.") - raise ValueError - elif not path.exists(cal_file_path): - logger.error(f"{cal_file_path} does not exist.") - raise FileNotFoundError - else: - logger.debug(f"Loading calibration file: {cal_file_path}") - # Create calibration object - cal_file_path = Path(cal_file_path) - if cal_type.lower() in ["sensor", "onboard"]: - cal = SensorCalibration.from_json(cal_file_path) - elif cal_type.lower() == "differential": - cal = DifferentialCalibration.from_json(cal_file_path) - else: - logger.error(f"Unknown calibration type: {cal_type}") - raise ValueError - except Exception: - cal = None - logger.exception( - f"Unable to load {cal_type} calibration file, reverting to none" - ) - finally: - return cal diff --git a/src/initialization/utils.py b/src/initialization/utils.py new file mode 100644 index 00000000..c6d4022a --- /dev/null +++ b/src/initialization/utils.py @@ -0,0 +1,24 @@ +import logging +import sys +from pathlib import Path +from subprocess import check_output + +from django.conf import settings + +logger = logging.getLogger(__name__) + + +def set_container_unhealthy(): + if settings.IN_DOCKER: + logger.warning("Marking container for restart.") + Path(settings.SDR_HEALTHCHECK_FILE).touch() + + +def get_usb_device_exists() -> bool: + logger.debug("Checking for USB...") + if not settings.RUNNING_TESTS and settings.USB_DEVICE is not None: + usb_devices = check_output("lsusb").decode(sys.stdout.encoding) + logger.debug("Checking for " + settings.USB_DEVICE) + logger.debug("Found " + usb_devices) + return settings.USB_DEVICE in usb_devices + return True From ec4c26aa68f9e9f49883f175616d3350b263311c Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 14 Mar 2024 16:03:16 -0600 Subject: [PATCH 25/56] remove actions from laod_sensor --- src/initialization/sensor_loader.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index e579907e..0b640c2a 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -37,10 +37,9 @@ def sensor(self) -> Sensor: return self._sensor -def load_sensor(sensor_capabilities: dict, switches: dict, preselector: Preselector, sensor_actions: dict) -> Sensor: +def load_sensor(sensor_capabilities: dict, switches: dict, preselector: Preselector) -> Sensor: sensor_cal = None differential_cal = None - preselector = None location = None if not settings.RUNNING_TESTS: # Remove location from sensor definition and convert to geojson. From ee5aef6301f75032d0a06c10c83f39a4cd65eca5 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 14 Mar 2024 16:14:20 -0600 Subject: [PATCH 26/56] add RAY_INIT setting. --- docker-compose.yml | 1 + src/sensor/settings.py | 1 + 2 files changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index bc0c2264..061977aa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -65,6 +65,7 @@ services: - SIGAN_POWER_SWITCH - SIGAN_POWER_CYCLE_STATES - STARTUP_CALIBRATION_ACTION + - RAY_INIT - RUNNING_MIGRATIONS - USB_DEVICE expose: diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 848aa81a..9aee00ee 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -451,3 +451,4 @@ os.environ["RUNNING_TESTS"] = str(RUNNING_TESTS) USB_DEVICE = env("USB_DEVICE", default=None) STARTUP_CALIBRATION_ACTION = env("STARTUP_CALIBRATION_ACTION", default=None) +RAY_INIT = env.bool("RAY_INIT", default=False) \ No newline at end of file From 975d2404764d15c31c1ade8e0d15699c4f752a9d Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 14 Mar 2024 16:29:38 -0600 Subject: [PATCH 27/56] Log performing startup cal. --- src/initialization/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 05b118a7..521547fc 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -199,6 +199,7 @@ def get_calibration( if settings.STARTUP_CALIBRATION_ACTION is None: logger.error("No STARTUP_CALIBRATION_ACTION set.") else: + logger.debug("Performing startup calibration...") cal_action = action_loader.actions[settings.STARTUP_CALIBRATION_ACTION] cal_action(sensor=sensor_loader.sensor, schedule_entry=None, task_id=None) else: From 74bc22dc0749e48f826f15ff50f84e79832ca3f9 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 15 Mar 2024 09:05:26 -0600 Subject: [PATCH 28/56] Correct log message. --- src/initialization/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 521547fc..228e7279 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -204,8 +204,7 @@ def get_calibration( cal_action(sensor=sensor_loader.sensor, schedule_entry=None, task_id=None) else: logger.debug( - "Skipping startup calibration since sensor_calibration exists and" - + "CALIBRATE_ON_STARTUP environment variable is False" + "Skipping startup calibration since sensor_calibration exists and has not expired." ) # Now load the differential calibration, if it exists From 103253ad5db4ca7fcd17af82b71ffd7ea9edd4b1 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 15 Mar 2024 12:03:37 -0600 Subject: [PATCH 29/56] temporarily catch BaseException from startup cal. --- src/initialization/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 228e7279..8046b72d 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -200,8 +200,11 @@ def get_calibration( logger.error("No STARTUP_CALIBRATION_ACTION set.") else: logger.debug("Performing startup calibration...") - cal_action = action_loader.actions[settings.STARTUP_CALIBRATION_ACTION] - cal_action(sensor=sensor_loader.sensor, schedule_entry=None, task_id=None) + try: + cal_action = action_loader.actions[settings.STARTUP_CALIBRATION_ACTION] + cal_action(sensor=sensor_loader.sensor, schedule_entry=None, task_id=None) + except BaseException as cal_error: + logger.error(f"Error during startup calibration: {cal_error}") else: logger.debug( "Skipping startup calibration since sensor_calibration exists and has not expired." From 68a2aa56a66c730d69632464a9b28c63402a517e Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 15 Mar 2024 12:47:35 -0600 Subject: [PATCH 30/56] create method for startup cal. --- src/initialization/__init__.py | 39 ++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 8046b72d..972a9aff 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -137,6 +137,25 @@ def get_calibration( finally: return cal +def calibrate_if_needed(): + # Now run the calibration action defined in the environment + # This will create an onboard_cal file if needed, and set it + # as the sensor's sensor_calibration. + if not settings.RUNNING_MIGRATIONS: + if sensor_loader.sensor.sensor_calibration is None or sensor_loader.sensor.sensor_calibration.expired(): + if settings.STARTUP_CALIBRATION_ACTION is None: + logger.error("No STARTUP_CALIBRATION_ACTION set.") + else: + logger.debug("Performing startup calibration...") + try: + cal_action = action_loader.actions[settings.STARTUP_CALIBRATION_ACTION] + cal_action(sensor=sensor_loader.sensor, schedule_entry=None, task_id=None) + except BaseException as cal_error: + logger.error(f"Error during startup calibration: {cal_error}") + else: + logger.debug( + "Skipping startup calibration since sensor_calibration exists and has not expired." + ) try: sensor_loader = None @@ -191,24 +210,6 @@ def get_calibration( if sensor_cal is not None: sensor_loader.sensor.sensor_calibration = sensor_cal - # Now run the calibration action defined in the environment - # This will create an onboard_cal file if needed, and set it - # as the sensor's sensor_calibration. - if not settings.RUNNING_MIGRATIONS: - if sensor_loader.sensor.sensor_calibration is None or sensor_loader.sensor.sensor_calibration.expired(): - if settings.STARTUP_CALIBRATION_ACTION is None: - logger.error("No STARTUP_CALIBRATION_ACTION set.") - else: - logger.debug("Performing startup calibration...") - try: - cal_action = action_loader.actions[settings.STARTUP_CALIBRATION_ACTION] - cal_action(sensor=sensor_loader.sensor, schedule_entry=None, task_id=None) - except BaseException as cal_error: - logger.error(f"Error during startup calibration: {cal_error}") - else: - logger.debug( - "Skipping startup calibration since sensor_calibration exists and has not expired." - ) # Now load the differential calibration, if it exists differential_cal = get_calibration( @@ -217,6 +218,8 @@ def get_calibration( ) sensor_loader.sensor.differential_calibration = differential_cal + calibrate_if_needed() + import ray if settings.RAY_INIT and not ray.is_initialized(): # Dashboard is only enabled if ray[default] is installed From 104ea451375ca0300704b1d6471508cce131df24 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 15 Mar 2024 13:05:24 -0600 Subject: [PATCH 31/56] move startup calibration into scheduler. --- src/initialization/__init__.py | 20 +------------------- src/scheduler/scheduler.py | 24 +++++++++++++++++++++++- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 972a9aff..0e35c130 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -137,25 +137,7 @@ def get_calibration( finally: return cal -def calibrate_if_needed(): - # Now run the calibration action defined in the environment - # This will create an onboard_cal file if needed, and set it - # as the sensor's sensor_calibration. - if not settings.RUNNING_MIGRATIONS: - if sensor_loader.sensor.sensor_calibration is None or sensor_loader.sensor.sensor_calibration.expired(): - if settings.STARTUP_CALIBRATION_ACTION is None: - logger.error("No STARTUP_CALIBRATION_ACTION set.") - else: - logger.debug("Performing startup calibration...") - try: - cal_action = action_loader.actions[settings.STARTUP_CALIBRATION_ACTION] - cal_action(sensor=sensor_loader.sensor, schedule_entry=None, task_id=None) - except BaseException as cal_error: - logger.error(f"Error during startup calibration: {cal_error}") - else: - logger.debug( - "Skipping startup calibration since sensor_calibration exists and has not expired." - ) + try: sensor_loader = None diff --git a/src/scheduler/scheduler.py b/src/scheduler/scheduler.py index a264dd90..8f4aadd2 100644 --- a/src/scheduler/scheduler.py +++ b/src/scheduler/scheduler.py @@ -11,7 +11,8 @@ from scos_actions.hardware.sensor import Sensor from scos_actions.signals import trigger_api_restart -from initialization import sensor_loader +from initialization import sensor_loader, action_loader + from schedule.models import ScheduleEntry from tasks.consts import MAX_DETAIL_LEN from tasks.models import TaskResult @@ -91,6 +92,7 @@ def run(self, blocking=True): pass try: + self.calibrate_if_needed() while True: with minimum_duration(blocking): self._consume_schedule(blocking) @@ -335,6 +337,26 @@ def __repr__(self): s = "running" if self.running else "stopped" return f"<{self.__class__.__name__} status={s}>" + def calibrate_if_needed(self): + # Now run the calibration action defined in the environment + # This will create an onboard_cal file if needed, and set it + # as the sensor's sensor_calibration. + if not settings.RUNNING_MIGRATIONS: + if sensor_loader.sensor.sensor_calibration is None or sensor_loader.sensor.sensor_calibration.expired(): + if settings.STARTUP_CALIBRATION_ACTION is None: + logger.error("No STARTUP_CALIBRATION_ACTION set.") + else: + logger.debug("Performing startup calibration...") + try: + cal_action = action_loader.actions[settings.STARTUP_CALIBRATION_ACTION] + cal_action(sensor=sensor_loader.sensor, schedule_entry=None, task_id=None) + except BaseException as cal_error: + logger.error(f"Error during startup calibration: {cal_error}") + else: + logger.debug( + "Skipping startup calibration since sensor_calibration exists and has not expired." + ) + @contextmanager def minimum_duration(blocking): From 6d13cff527e4a04b48681dd2a1725b45a660e048 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 15 Mar 2024 13:19:19 -0600 Subject: [PATCH 32/56] remove dead code. --- src/initialization/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 0e35c130..9478470c 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -200,8 +200,6 @@ def get_calibration( ) sensor_loader.sensor.differential_calibration = differential_cal - calibrate_if_needed() - import ray if settings.RAY_INIT and not ray.is_initialized(): # Dashboard is only enabled if ray[default] is installed From c94399b451c52af9a758dc02d0c55047b6793c95 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sun, 17 Mar 2024 13:09:16 -0600 Subject: [PATCH 33/56] Don't raise exceptions (that will be caught anyway) in loading cals. --- src/initialization/__init__.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 9478470c..32215d93 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -113,11 +113,9 @@ def get_calibration( try: cal = None if cal_file_path is None or cal_file_path == "": - logger.error("No calibration file specified.") - raise ValueError + logger.error("No calibration file specified, reverting to none.") elif not path.exists(cal_file_path): - logger.error(f"{cal_file_path} does not exist.") - raise FileNotFoundError + logger.error(f"{cal_file_path} does not exist, reverting to none.") else: logger.debug(f"Loading calibration file: {cal_file_path}") # Create calibration object From 0b95fe9beb6d804ac61cad9dc9eaa1e14058be31 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 19 Mar 2024 09:13:56 -0600 Subject: [PATCH 34/56] pre-commit --- gunicorn/config.py | 1 - src/handlers/apps.py | 5 +---- src/initialization/__init__.py | 7 +++---- src/initialization/action_loader.py | 2 +- src/initialization/sensor_loader.py | 19 ++++++++++++------- src/scheduler/scheduler.py | 20 ++++++++++++++------ src/sensor/settings.py | 2 +- src/sensor/utils.py | 2 -- src/sensor/wsgi.py | 2 +- src/tasks/__init__.py | 2 -- src/utils/signals.py | 4 ++-- 11 files changed, 35 insertions(+), 31 deletions(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index 5efc64fb..0b68ad33 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -2,7 +2,6 @@ import sys from multiprocessing import cpu_count - bind = ":8000" workers = 1 worker_class = "gthread" diff --git a/src/handlers/apps.py b/src/handlers/apps.py index 8afad0e2..3dffb135 100644 --- a/src/handlers/apps.py +++ b/src/handlers/apps.py @@ -5,7 +5,7 @@ from scos_actions.signals import ( location_action_completed, measurement_action_completed, - trigger_api_restart + trigger_api_restart, ) logger = logging.getLogger(__name__) @@ -20,7 +20,6 @@ def ready(self): db_location_deleted, db_location_updated, location_action_completed_callback, - ) from handlers.measurement_handler import measurement_action_completed_callback @@ -40,5 +39,3 @@ def ready(self): trigger_api_restart.connect(trigger_api_restart_callback) logger.debug("trigger_api_restart_callback registered to trigger_api_restart") - - diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 32215d93..ae53ff7d 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -3,6 +3,7 @@ import time from os import path from pathlib import Path +from typing import Optional, Union from django.conf import settings from its_preselector.configuration_exception import ConfigurationException @@ -12,7 +13,7 @@ from scos_actions.calibration.sensor_calibration import SensorCalibration from scos_actions.hardware.utils import power_cycle_sigan from scos_actions.utils import load_from_json -from typing import Optional, Union + from utils.signals import register_component_with_status from .action_loader import ActionLoader @@ -136,7 +137,6 @@ def get_calibration( return cal - try: sensor_loader = None register_component_with_status.connect(status_registration_handler) @@ -190,7 +190,6 @@ def get_calibration( if sensor_cal is not None: sensor_loader.sensor.sensor_calibration = sensor_cal - # Now load the differential calibration, if it exists differential_cal = get_calibration( settings.DIFFERENTIAL_CALIBRATION_FILE, @@ -199,6 +198,7 @@ def get_calibration( sensor_loader.sensor.differential_calibration = differential_cal import ray + if settings.RAY_INIT and not ray.is_initialized(): # Dashboard is only enabled if ray[default] is installed logger.debug("Initializing ray.") @@ -206,4 +206,3 @@ def get_calibration( except BaseException as error: logger.exception(f"Error during initialization: {error}") set_container_unhealthy() - diff --git a/src/initialization/action_loader.py b/src/initialization/action_loader.py index 8b98b3f8..f94fb5a1 100644 --- a/src/initialization/action_loader.py +++ b/src/initialization/action_loader.py @@ -4,12 +4,12 @@ import os import pkgutil import shutil +from typing import Dict from django.conf import settings from scos_actions.actions import action_classes from scos_actions.actions.interfaces.action import Action from scos_actions.discover import init, test_actions -from typing import Dict logger = logging.getLogger(__name__) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index 0b640c2a..3c94ee64 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -6,6 +6,7 @@ from its_preselector.preselector import Preselector from scos_actions.hardware.sensor import Sensor from scos_actions.metadata.utils import construct_geojson_point + from utils.signals import register_component_with_status from .utils import get_usb_device_exists, set_container_unhealthy @@ -18,15 +19,17 @@ class SensorLoader: _instance = None def __init__( - self, sensor_capabilities: dict, switches: dict, preselector: Preselector + self, sensor_capabilities: dict, switches: dict, preselector: Preselector ): if not hasattr(self, "sensor"): logger.debug("Sensor has not been loaded. Loading...") - self._sensor = load_sensor(sensor_capabilities,switches, preselector) + self._sensor = load_sensor(sensor_capabilities, switches, preselector) else: logger.debug("Already loaded sensor. ") - def __new__(cls, sensor_capabilities: dict,switches: dict, preselector: Preselector): + def __new__( + cls, sensor_capabilities: dict, switches: dict, preselector: Preselector + ): if cls._instance is None: logger.debug("Creating the SensorLoader") cls._instance = super().__new__(cls) @@ -37,7 +40,9 @@ def sensor(self) -> Sensor: return self._sensor -def load_sensor(sensor_capabilities: dict, switches: dict, preselector: Preselector) -> Sensor: +def load_sensor( + sensor_capabilities: dict, switches: dict, preselector: Preselector +) -> Sensor: sensor_cal = None differential_cal = None location = None @@ -60,7 +65,9 @@ def load_sensor(sensor_capabilities: dict, switches: dict, preselector: Preselec check_for_required_sigan_settings() sigan_module_setting = settings.SIGAN_MODULE sigan_module = importlib.import_module(sigan_module_setting) - logger.info(f"Creating {settings.SIGAN_CLASS} from {settings.SIGAN_MODULE}") + logger.info( + f"Creating {settings.SIGAN_CLASS} from {settings.SIGAN_MODULE}" + ) sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) sigan = sigan_constructor(switches=switches) register_component_with_status.send(sigan, component=sigan) @@ -97,5 +104,3 @@ def check_for_required_sigan_settings(): error += "SIGAN_CLASS environment variable. " if raise_exception: raise Exception(error) - - diff --git a/src/scheduler/scheduler.py b/src/scheduler/scheduler.py index 8f4aadd2..33d1e119 100644 --- a/src/scheduler/scheduler.py +++ b/src/scheduler/scheduler.py @@ -11,8 +11,7 @@ from scos_actions.hardware.sensor import Sensor from scos_actions.signals import trigger_api_restart -from initialization import sensor_loader, action_loader - +from initialization import action_loader, sensor_loader from schedule.models import ScheduleEntry from tasks.consts import MAX_DETAIL_LEN from tasks.models import TaskResult @@ -342,14 +341,23 @@ def calibrate_if_needed(self): # This will create an onboard_cal file if needed, and set it # as the sensor's sensor_calibration. if not settings.RUNNING_MIGRATIONS: - if sensor_loader.sensor.sensor_calibration is None or sensor_loader.sensor.sensor_calibration.expired(): + if ( + sensor_loader.sensor.sensor_calibration is None + or sensor_loader.sensor.sensor_calibration.expired() + ): if settings.STARTUP_CALIBRATION_ACTION is None: logger.error("No STARTUP_CALIBRATION_ACTION set.") else: - logger.debug("Performing startup calibration...") + logger.info("Performing startup calibration...") try: - cal_action = action_loader.actions[settings.STARTUP_CALIBRATION_ACTION] - cal_action(sensor=sensor_loader.sensor, schedule_entry=None, task_id=None) + cal_action = action_loader.actions[ + settings.STARTUP_CALIBRATION_ACTION + ] + cal_action( + sensor=sensor_loader.sensor, + schedule_entry=None, + task_id=None, + ) except BaseException as cal_error: logger.error(f"Error during startup calibration: {cal_error}") else: diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 9aee00ee..6adea50f 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -451,4 +451,4 @@ os.environ["RUNNING_TESTS"] = str(RUNNING_TESTS) USB_DEVICE = env("USB_DEVICE", default=None) STARTUP_CALIBRATION_ACTION = env("STARTUP_CALIBRATION_ACTION", default=None) -RAY_INIT = env.bool("RAY_INIT", default=False) \ No newline at end of file +RAY_INIT = env.bool("RAY_INIT", default=False) diff --git a/src/sensor/utils.py b/src/sensor/utils.py index 3b2cd92d..b71ad05b 100644 --- a/src/sensor/utils.py +++ b/src/sensor/utils.py @@ -17,5 +17,3 @@ def get_timestamp_from_datetime(dt: datetime) -> int: def parse_datetime_str(d: str) -> datetime: return datetime.strptime(d, settings.DATETIME_FORMAT) - - diff --git a/src/sensor/wsgi.py b/src/sensor/wsgi.py index 610456e7..703f9628 100644 --- a/src/sensor/wsgi.py +++ b/src/sensor/wsgi.py @@ -30,4 +30,4 @@ if not settings.IN_DOCKER: # Normally scheduler is started by gunicorn worker process - scheduler.thread.start() \ No newline at end of file + scheduler.thread.start() diff --git a/src/tasks/__init__.py b/src/tasks/__init__.py index 8302e1a6..725c1d12 100644 --- a/src/tasks/__init__.py +++ b/src/tasks/__init__.py @@ -1,6 +1,4 @@ import logging - logger = logging.getLogger(__name__) logger.debug("********** Initializing tasks **********") - diff --git a/src/utils/signals.py b/src/utils/signals.py index 225298b5..c6753f6a 100644 --- a/src/utils/signals.py +++ b/src/utils/signals.py @@ -1,4 +1,4 @@ from django.dispatch import Signal -#provides component -register_component_with_status = Signal() \ No newline at end of file +# provides component +register_component_with_status = Signal() From c096d368fa78a775b42d6d49a860f222f0fc72b5 Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Wed, 27 Mar 2024 13:28:53 -0600 Subject: [PATCH 35/56] update scos-tekrsa and scos-actions version --- src/requirements-dev.txt | 8 +++----- src/requirements.in | 2 +- src/requirements.txt | 4 ++-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index adf1e6cc..7d955bd4 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -59,9 +59,7 @@ colorama==0.4.6 colorful==0.5.5 # via ray coverage[toml]==7.3.2 - # via - # coverage - # pytest-cov + # via pytest-cov cryptography==42.0.4 # via -r requirements.txt defusedxml==0.7.1 @@ -315,11 +313,11 @@ scipy==1.10.1 # via # -r requirements.txt # scos-actions -scos-actions @ git+https://github.com/NTIA/scos-actions@calibrate_to_antenna +scos-actions @ git+https://github.com/NTIA/scos-actions@9.0.0 # via # -r requirements.txt # scos-tekrsa -scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@calibrate_to_antenna +scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@6.0.0 # via -r requirements.txt sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive # via diff --git a/src/requirements.in b/src/requirements.in index e29e304d..01179100 100644 --- a/src/requirements.in +++ b/src/requirements.in @@ -11,7 +11,7 @@ packaging>=23.0, <24.0 psycopg2-binary>=2.0, <3.0 requests-mock>=1.0, <2.0 requests_oauthlib>=1.0, <2.0 -scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@calibrate_to_antenna +scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@6.0.0 # The following are sub-dependencies for which SCOS Sensor enforces a # higher minimum patch version than the dependencies which require them. diff --git a/src/requirements.txt b/src/requirements.txt index eb80ff8a..363648b3 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -149,9 +149,9 @@ ruamel-yaml-clib==0.2.8 # via ruamel-yaml scipy==1.10.1 # via scos-actions -scos-actions @ git+https://github.com/NTIA/scos-actions@calibrate_to_antenna +scos-actions @ git+https://github.com/NTIA/scos-actions@9.0.0 # via scos-tekrsa -scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@calibrate_to_antenna +scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@6.0.0 # via -r requirements.in sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive # via scos-actions From 3b060626ff34b2dd7690270cfbd7ef467fdf8f58 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 27 Mar 2024 15:43:59 -0400 Subject: [PATCH 36/56] fix import --- src/initialization/tests/test_initialization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/initialization/tests/test_initialization.py b/src/initialization/tests/test_initialization.py index 706a2e15..bc2b92f8 100644 --- a/src/initialization/tests/test_initialization.py +++ b/src/initialization/tests/test_initialization.py @@ -1,7 +1,7 @@ import logging import os -from initialization.sensor_loader import load_preselector +from initialization import load_preselector logger = logging.getLogger(__name__) From c71267d1d25914c80e26a1e069585fd7172732ad Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 27 Mar 2024 19:58:10 +0000 Subject: [PATCH 37/56] Remove obsolete 'version' field and rename to compose.yaml --- docker-compose.yml => compose.yaml | 2 -- 1 file changed, 2 deletions(-) rename docker-compose.yml => compose.yaml (99%) diff --git a/docker-compose.yml b/compose.yaml similarity index 99% rename from docker-compose.yml rename to compose.yaml index 435087c7..8bce4c9b 100644 --- a/docker-compose.yml +++ b/compose.yaml @@ -1,5 +1,3 @@ -version: '3' - services: db: image: postgres:15-alpine From bd4639723c7d2f494e15cf4357175eb153746e75 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 27 Mar 2024 16:41:18 -0400 Subject: [PATCH 38/56] GH Actions: wait longer for container to be healthy --- .github/workflows/github-actions-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github-actions-test.yml b/.github/workflows/github-actions-test.yml index ec289c2d..e8f871f4 100644 --- a/.github/workflows/github-actions-test.yml +++ b/.github/workflows/github-actions-test.yml @@ -79,7 +79,7 @@ jobs: docker-compose up -d - name: Wait for containers # wait for containers to finish starting - run: sleep 45 + run: sleep 55 - name: Check API container run: docker ps | grep api | grep -q healthy From c9bc7d16b49a5a74b8294823a746308ff266fbc6 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 27 Mar 2024 17:12:58 -0400 Subject: [PATCH 39/56] try longer sleep in GH actions testing --- .github/workflows/github-actions-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github-actions-test.yml b/.github/workflows/github-actions-test.yml index e8f871f4..b6559084 100644 --- a/.github/workflows/github-actions-test.yml +++ b/.github/workflows/github-actions-test.yml @@ -79,7 +79,7 @@ jobs: docker-compose up -d - name: Wait for containers # wait for containers to finish starting - run: sleep 55 + run: sleep 120 - name: Check API container run: docker ps | grep api | grep -q healthy From d07afdb98e29834d545ac2121ea68b424ee26df9 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 27 Mar 2024 17:41:36 -0400 Subject: [PATCH 40/56] use docker compose v2 in github action --- .github/workflows/github-actions-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/github-actions-test.yml b/.github/workflows/github-actions-test.yml index b6559084..362bdf04 100644 --- a/.github/workflows/github-actions-test.yml +++ b/.github/workflows/github-actions-test.yml @@ -75,8 +75,8 @@ jobs: source ./env.template export MOCK_SIGAN=1 export MOCK_SIGAN_RANDOM=1 - docker-compose build --no-cache - docker-compose up -d + docker compose build --no-cache + docker compose up -d - name: Wait for containers # wait for containers to finish starting run: sleep 120 From 2864b662003892ffc769a434c81282c12773022f Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 27 Mar 2024 17:42:11 -0400 Subject: [PATCH 41/56] return to shorter sleep time in GH action --- .github/workflows/github-actions-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github-actions-test.yml b/.github/workflows/github-actions-test.yml index 362bdf04..6aaa7561 100644 --- a/.github/workflows/github-actions-test.yml +++ b/.github/workflows/github-actions-test.yml @@ -79,7 +79,7 @@ jobs: docker compose up -d - name: Wait for containers # wait for containers to finish starting - run: sleep 120 + run: sleep 45 - name: Check API container run: docker ps | grep api | grep -q healthy From d4795de079f273504aa82925b82f862eb559735a Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 27 Mar 2024 17:51:35 -0400 Subject: [PATCH 42/56] skip startup calibration when running with mock sigan --- src/scheduler/scheduler.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/scheduler/scheduler.py b/src/scheduler/scheduler.py index 33d1e119..68157e6b 100644 --- a/src/scheduler/scheduler.py +++ b/src/scheduler/scheduler.py @@ -340,6 +340,8 @@ def calibrate_if_needed(self): # Now run the calibration action defined in the environment # This will create an onboard_cal file if needed, and set it # as the sensor's sensor_calibration. + if settings.MOCK_SIGAN: + logger.debug("Skipping startup calibration when using mock sigan") if not settings.RUNNING_MIGRATIONS: if ( sensor_loader.sensor.sensor_calibration is None From 6b2b92a28ce25f459bfcc980863c3f8dba7ad82c Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 27 Mar 2024 17:55:23 -0400 Subject: [PATCH 43/56] add missing return statement --- src/scheduler/scheduler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/scheduler/scheduler.py b/src/scheduler/scheduler.py index 68157e6b..82312049 100644 --- a/src/scheduler/scheduler.py +++ b/src/scheduler/scheduler.py @@ -342,6 +342,7 @@ def calibrate_if_needed(self): # as the sensor's sensor_calibration. if settings.MOCK_SIGAN: logger.debug("Skipping startup calibration when using mock sigan") + return if not settings.RUNNING_MIGRATIONS: if ( sensor_loader.sensor.sensor_calibration is None From a01a6ad341e156541f59334674fbf49afa3cdce2 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 27 Mar 2024 18:05:46 -0400 Subject: [PATCH 44/56] Do not check for USB_DEVICE when using mock sigan "running_tests" case is covered since MOCK_SIGAN is always true when running tests --- src/initialization/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/initialization/utils.py b/src/initialization/utils.py index c6d4022a..9eff0253 100644 --- a/src/initialization/utils.py +++ b/src/initialization/utils.py @@ -16,9 +16,11 @@ def set_container_unhealthy(): def get_usb_device_exists() -> bool: logger.debug("Checking for USB...") - if not settings.RUNNING_TESTS and settings.USB_DEVICE is not None: + if not settings.MOCK_SIGAN and settings.USB_DEVICE is not None: usb_devices = check_output("lsusb").decode(sys.stdout.encoding) logger.debug("Checking for " + settings.USB_DEVICE) logger.debug("Found " + usb_devices) return settings.USB_DEVICE in usb_devices + else: + logger.debug("Skipping check for USB device") return True From 560e347262ef1adb375b5936309d935e7eec06e9 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 28 Mar 2024 15:31:23 -0400 Subject: [PATCH 45/56] remove unused variables --- src/initialization/sensor_loader.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index 3c94ee64..5a911743 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -43,8 +43,6 @@ def sensor(self) -> Sensor: def load_sensor( sensor_capabilities: dict, switches: dict, preselector: Preselector ) -> Sensor: - sensor_cal = None - differential_cal = None location = None if not settings.RUNNING_TESTS: # Remove location from sensor definition and convert to geojson. From fbb8404d311d018e06dc8e59c21a8e825cf66297 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 28 Mar 2024 15:38:17 -0400 Subject: [PATCH 46/56] remove unused import, get SCOS_SENSOR_GIT_TAG from settings --- src/status/views.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/status/views.py b/src/status/views.py index 5e71f0f3..684aa563 100644 --- a/src/status/views.py +++ b/src/status/views.py @@ -4,22 +4,22 @@ import shutil import sys -from initialization import sensor_loader, status_monitor +from django.conf import settings from its_preselector import __version__ as PRESELECTOR_API_VERSION from its_preselector.preselector import Preselector from its_preselector.web_relay import WebRelay from rest_framework.decorators import api_view from rest_framework.response import Response -from scheduler import scheduler from scos_actions import __version__ as SCOS_ACTIONS_VERSION from scos_actions.hardware.sigan_iface import SignalAnalyzerInterface -from scos_actions.metadata.structs import ntia_diagnostics -from scos_actions.settings import SCOS_SENSOR_GIT_TAG from scos_actions.utils import ( convert_datetime_to_millisecond_iso_format, get_datetime_str_now, ) +from initialization import sensor_loader, status_monitor +from scheduler import scheduler + from . import start_time from .serializers import LocationSerializer from .utils import get_location @@ -58,18 +58,27 @@ def get_software_version(): software_version = { "system_platform": platform.platform(), "python_version": sys.version.split()[0], - "scos_sensor_version": SCOS_SENSOR_GIT_TAG, + "scos_sensor_version": settings.SCOS_SENSOR_GIT_TAG, "scos_actions_version": SCOS_ACTIONS_VERSION, "preselector_api_version": PRESELECTOR_API_VERSION, } - if sensor_loader.sensor is not None and sensor_loader.sensor.signal_analyzer is not None: + if ( + sensor_loader.sensor is not None + and sensor_loader.sensor.signal_analyzer is not None + ): if sensor_loader.sensor.signal_analyzer.firmware_version is not None: - software_version["sigan_firmware_version"] = sensor_loader.sensor.signal_analyzer.firmware_version + software_version[ + "sigan_firmware_version" + ] = sensor_loader.sensor.signal_analyzer.firmware_version if sensor_loader.sensor.signal_analyzer.api_version is not None: - software_version["sigan_api_version"] = sensor_loader.sensor.signal_analyzer.api_version + software_version[ + "sigan_api_version" + ] = sensor_loader.sensor.signal_analyzer.api_version if sensor_loader.sensor.signal_analyzer.plugin_version is not None: - software_version["scos_sigan_plugin"] = sensor_loader.sensor.signal_analyzer.plugin_version + software_version[ + "scos_sigan_plugin" + ] = sensor_loader.sensor.signal_analyzer.plugin_version logger.debug(software_version) return software_version From 5680801a6837a66ee28103b8b509b4eb42ac2602 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 28 Mar 2024 16:33:08 -0400 Subject: [PATCH 47/56] update readme for calibration changes --- README.md | 88 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 48 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index d4d26e43..8538a045 100644 --- a/README.md +++ b/README.md @@ -292,7 +292,9 @@ scos-sensor website from the same computer as where it is hosted. When running in a production environment or on a remote system, various settings will need to be configured. -## docker-compose.yml +### Compose File + +This section details configuration which takes place in `compose.yaml`: - shm_size: This setting is overriding the default setting of 64 mb. If using scos-sensor on a computer with lower memory, this may need to be decreased. This is @@ -321,6 +323,9 @@ settings in the environment file: By default, this is configured to use a version of `ghcr.io/ntia/scos-tekrsa/tekrsa_usb` to use a Tektronix signal analyzer. +- CALIBRATION_EXPIRATION_LIMIT: Number of seconds elapsed for a calibration result to + become expired. On startup, if existing calibration is expired, the action defined by + STARTUP_CALIBRATION_ACTION will be run to generate new calibration data. - CALLBACK_AUTHENTICATION: Sets how to authenticate to the callback URL. Supports `TOKEN` or `CERT`. - CALLBACK_SSL_VERIFICATION: Set to “true” in production environment. If false, the SSL @@ -353,6 +358,7 @@ settings in the environment file: certificate. - POSTGRES_PASSWORD: Sets password for the Postgres database for the “postgres” user. Change in production. The env.template file sets to a randomly generated value. +- RAY_INIT: If set to true, SCOS Sensor will ensure initializaiton of the Ray library. - REPO_ROOT: Root folder of the repository. Should be correctly set by default. - SCOS_SENSOR_GIT_TAG: The scos-sensor branch name. This value may be used in action metadata to capture the version of the software that produced the sigmf archive. @@ -379,6 +385,8 @@ settings in the environment file: scos-sensor repository with a valid certificate in production. - SSL_KEY_PATH: Path to server SSL private key. Use the private key for your valid certificate in production. +- STARTUP_CALIBRATION_ACTION: The name of an available action which will run on startup + if no unexpired calibration data is already present. - USB_DEVICE: Optional string used to search for available USB devices. By default, this is set to Tektronix to see if the Tektronix signal analyzer is available. If the specified value is not found in the output of lsusb, scos-sensor will attempt @@ -420,46 +428,46 @@ specific to the sensor you are using. } ``` -### Sensor Calibration File +### Calibration Files -By default, scos-sensor will use `configs/default_sensor_calibration.json` as the sensor -calibration file. However, if `configs/sensor_calibration.json` exists it will be used -instead of the default calibration file. Sensor calibration files allow SCOS Sensor to -scale data based on a laboratory or in-field calibration of the sensor and may also -contain other useful metadata that characterizes the sensor performance. For additional -information on the calibration data, see the -[NTIA-Sensor SigMF Calibration Object](https://github.com/NTIA/sigmf-ns-ntia/blob/master/ntia-sensor.sigmf-ext.md#08-the-calibration-object). -The default calibration file is shown below: +Calibration files allow SCOS Sensor to scale data based on a laboratory and/or +in-field calibration of the sensor, and may also contain other useful metadata that +characterizes the sensor performance. Two primary types of calibration files are used: +sensor calibration files and differential calibration files. -```json -{ - "calibration_data":{ - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 0, - "noise_figure": null, - "1db_compression_point": null, - "enbw": null, - "temperature": 26.85 - }, - "last_calibration_datetime": "1970-01-01T00:00:00.000000Z", - "calibration_parameters": [], - "clock_rate_lookup_by_sample_rate": [ - ], - "sensor_uid": "DEFAULT CALIBRATION", - "calibration_reference": "noise source output" -} -``` +Sensor calibration files may be provided upon sensor deployment or generated by onboard +calibration actions. If both exist, onboard calibration data takes priority. The +sensor will first attempt to load `configs/onboard_sensor_calibration.json`, and fall +back to `configs/sensor_calibration.json` if the first option fails. Next, SCOS determines +whether the loaded calibration data is expired, based on the threshold set by the +CALIBRATION_EXPIRATION_LIMIT threshold. If calibration data is expired, the sensor will +attempt to run the calibration action defined by STARTUP_CALIBRATION_ACTION. + +SCOS Sensor also supports an additional calibration file, called a differential +calibration. The differential calibration is used to provide additional scaling factors +to shift the reference point of data from the onboard calibration terminal to elsewhere +in the signal path. For instance, a differential calibration file can be provided with +scaling factors to shift the data reference point from the onboard calibration terminal +to the antenna port. The differential calibration is loaded separately, and used in +addition to, either the onboard or lab-provided sensor calibration file. + +#### Calibration File Contents + +In sensor calibration files, the unit of calibration data is defined by the +[NTIA-Sensor SigMF Calibration Object](https://github.com/NTIA/sigmf-ns-ntia/blob/master/ntia-sensor.sigmf-ext.md#08-the-calibration-object). +In differential calibration files, the only key in the calibration data is `loss`. The `calibration_parameters` key lists the parameters that will be used to obtain -the calibration data. In the case of the default calibration, there are no -`calibration_parameters` so the calibration data is found directly within the -`calibration_data` element and by default scos-sensor will not apply any additional -gain. Typically, a sensor would be calibrated at particular -sensing parameters. The calibration data for specific parameters should be listed -within the calibration_data object and accessed by the values of the settings -listed in the calibration_parameters element. For example, the calibration below -provides an example of a sensor calibrated at a sample rate of 14000000.0 samples -per second at several frequencies with a signal analyzer reference level setting of -25. +the calibration data. In the case of onboard calibration being generated from scratch, +the startup calibration action's signal analyzer settings are used as the `calibration_parameters`. +The calibration data is found directly within the `calibration_data` element and by +default SCOS Sensor will not apply any additional gain. Typically, a sensor would be +calibrated at particular sensing parameters. The calibration data for specific parameters +should be listed within the `calibration_data` object and accessed by the values of the +settings listed in the calibration_parameters element. For example, the sensor +calibration below provides an example of a sensor calibrated at a sample rate of +140000000 samples per second at several frequencies with a signal analyzer reference +level setting of -25. ```json { @@ -517,7 +525,7 @@ per second at several frequencies with a signal analyzer reference level setting When an action is run with the above calibration, SCOS will expect the action to have a sample_rate, frequency, and reference_level specified in the action config. The values -specified for these parameters will then be used to retrieve the calibration data. +specified for these parameters will then be used to retrieve the calibration data entry. ## Security @@ -843,7 +851,7 @@ for additional information. Be sure to re-source the environment file, update th requirements files, and prune any existing containers before rebuilding scos-sensor. -## Preselector Support +### Preselector Support Scos-sensor can be configured to support [preselectors](http://www.github.com/ntia/Preselector). @@ -857,7 +865,7 @@ in docker-compose.yaml to the python module that contains the preselector implementation you specify in PRESELECTOR_CLASS in docker-compose.yaml. -## Relay Support +### Relay Support Scos-sensor can be configured with zero or more [network controlled relays](https://www.controlbyweb.com/webrelay/). The default relay configuration directory is configs/switches. From fc8335b77ac1f6d1a9ba480ef33dba3642a75f40 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 28 Mar 2024 16:35:26 -0400 Subject: [PATCH 48/56] remove unused variable CALIBRATE_ON_STARTUP --- env.template | 1 - 1 file changed, 1 deletion(-) diff --git a/env.template b/env.template index 0b0b4fd9..8619c4f8 100644 --- a/env.template +++ b/env.template @@ -91,7 +91,6 @@ SSL_KEY_PATH=sensor01.pem # The action specified here will be used to attempt an onboard # sensor calibration on startup, if no onboard calibration data # is available on startup. The specified action must be available. -CALIBRATE_ON_STARTUP=true STARTUP_CALIBRATION_ACTION=SEA_CBRS_Calibrate_Baseline # Debug dependant settings From e2bd0083202445f772440b30837228703952bb70 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 28 Mar 2024 16:36:06 -0400 Subject: [PATCH 49/56] switch BASE_IMAGE version to latest --- env.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/env.template b/env.template index 8619c4f8..35dacf17 100644 --- a/env.template +++ b/env.template @@ -77,7 +77,7 @@ SIGAN_CLASS=TekRSASigan SIGAN_MODULE=scos_tekrsa.hardware.tekrsa_sigan USB_DEVICE=Tektronix DEVICE_MODEL=RSA507A -BASE_IMAGE=ghcr.io/ntia/scos-tekrsa/tekrsa_usb:0.2.3 +BASE_IMAGE=ghcr.io/ntia/scos-tekrsa/tekrsa_usb:latest # SECURITY WARNING: You should be using certs from a trusted authority. # If you don't have any, try letsencrypt or a similar service. From b0af7724a65eee9063fee1ead06370ff64252841 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 28 Mar 2024 16:46:47 -0400 Subject: [PATCH 50/56] update all pre-commit hooks --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ab6aed0a..09e02539 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,24 +19,24 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/asottile/pyupgrade - rev: v3.15.0 + rev: v3.15.2 hooks: - id: pyupgrade args: ["--py38-plus"] - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort name: isort (python) types: [file, python] args: ["--profile", "black", "--filter-files", "--gitignore"] - repo: https://github.com/psf/black - rev: 23.10.1 + rev: 24.3.0 hooks: - id: black types: [file, python] - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.37.0 + rev: v0.39.0 hooks: - id: markdownlint types: [file, markdown] From c3f02ad8ea8c792d142e264da276c70616f4bd8d Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 28 Mar 2024 16:47:16 -0400 Subject: [PATCH 51/56] Run updated pre-commit on all files --- src/scheduler/scheduler.py | 1 + src/sensor/settings.py | 1 + src/status/views.py | 24 ++++++++++++------------ 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/scheduler/scheduler.py b/src/scheduler/scheduler.py index 82312049..9f392079 100644 --- a/src/scheduler/scheduler.py +++ b/src/scheduler/scheduler.py @@ -1,4 +1,5 @@ """Queue and run tasks.""" + import json import logging import threading diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 6adea50f..f7714d38 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -9,6 +9,7 @@ https://docs.djangoproject.com/en/1.11/ref/settings/ """ + import logging import os import sys diff --git a/src/status/views.py b/src/status/views.py index 684aa563..fdb635d4 100644 --- a/src/status/views.py +++ b/src/status/views.py @@ -68,17 +68,17 @@ def get_software_version(): and sensor_loader.sensor.signal_analyzer is not None ): if sensor_loader.sensor.signal_analyzer.firmware_version is not None: - software_version[ - "sigan_firmware_version" - ] = sensor_loader.sensor.signal_analyzer.firmware_version + software_version["sigan_firmware_version"] = ( + sensor_loader.sensor.signal_analyzer.firmware_version + ) if sensor_loader.sensor.signal_analyzer.api_version is not None: - software_version[ - "sigan_api_version" - ] = sensor_loader.sensor.signal_analyzer.api_version + software_version["sigan_api_version"] = ( + sensor_loader.sensor.signal_analyzer.api_version + ) if sensor_loader.sensor.signal_analyzer.plugin_version is not None: - software_version[ - "scos_sigan_plugin" - ] = sensor_loader.sensor.signal_analyzer.plugin_version + software_version["scos_sigan_plugin"] = ( + sensor_loader.sensor.signal_analyzer.plugin_version + ) logger.debug(software_version) return software_version @@ -101,9 +101,9 @@ def status(request, version, format=None): sensor_loader.sensor is not None and sensor_loader.sensor.sensor_calibration is not None ): - status_json[ - "last_calibration_datetime" - ] = sensor_loader.sensor.sensor_calibration.last_calibration_datetime + status_json["last_calibration_datetime"] = ( + sensor_loader.sensor.sensor_calibration.last_calibration_datetime + ) for component in status_monitor.status_components: component_status = component.get_status() if isinstance(component, WebRelay): From ec3cb11bacdee19195a759550e1f2df3d251d2c1 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 28 Mar 2024 16:49:14 -0400 Subject: [PATCH 52/56] Use get_disk_usage from SCOS Actions in status view --- src/status/views.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/status/views.py b/src/status/views.py index fdb635d4..ecf51544 100644 --- a/src/status/views.py +++ b/src/status/views.py @@ -15,6 +15,7 @@ from scos_actions.utils import ( convert_datetime_to_millisecond_iso_format, get_datetime_str_now, + get_disk_usage, ) from initialization import sensor_loader, status_monitor @@ -37,14 +38,6 @@ def serialize_location(): return None -def disk_usage(): - """Return the total disk usage as a percentage.""" - usage = shutil.disk_usage("/") - percent_used = round(100 * usage.used / usage.total) - logger.debug(str(percent_used) + " disk used") - return round(percent_used, 2) - - def get_days_up(): """Return the number of days SCOS has been running.""" elapsed = datetime.datetime.utcnow() - start_time @@ -93,7 +86,7 @@ def status(request, version, format=None): "location": serialize_location(), "system_time": get_datetime_str_now(), "start_time": convert_datetime_to_millisecond_iso_format(start_time), - "disk_usage": disk_usage(), + "disk_usage": get_disk_usage(), "days_up": get_days_up(), "software": get_software_version(), } From beef855f3c030b8743349cf3cbe3dff964ab499a Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 28 Mar 2024 16:49:34 -0400 Subject: [PATCH 53/56] remove unused import --- src/status/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/status/views.py b/src/status/views.py index ecf51544..c7d070d1 100644 --- a/src/status/views.py +++ b/src/status/views.py @@ -1,7 +1,6 @@ import datetime import logging import platform -import shutil import sys from django.conf import settings From 6da59dbb891734563859d4f422341503c06be28a Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 28 Mar 2024 16:52:58 -0400 Subject: [PATCH 54/56] fix wording in cal file comment --- src/sensor/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sensor/settings.py b/src/sensor/settings.py index f7714d38..d1a46a9f 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -79,7 +79,7 @@ os.environ["ONBOARD_CALIBRATION_FILE"] = ONBOARD_CALIBRATION_FILE # Sensor calibration file; should be provided manually. Will not be -# overwritten when calibration actions. +# overwritten by calibration actions. if path.exists(sensor_cal_path := path.join(CONFIG_DIR, "sensor_calibration.json")): SENSOR_CALIBRATION_FILE = sensor_cal_path else: From 24199981338a4a4856574e935c8a4070d3c93c6a Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 28 Mar 2024 17:25:12 -0400 Subject: [PATCH 55/56] simplify parameterization of get_calibration No longer make this function arbitrarily distinguish between "onboard" and "sensor" calibrations --- src/initialization/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index ae53ff7d..82822e38 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -108,7 +108,7 @@ def get_calibration( Load calibration data from file. :param cal_file_path: Path to the JSON calibration file. - :param cal_type: Calibration type to load: "onboard", "sensor" or "differential" + :param cal_type: Calibration type to load: "sensor" or "differential" :return: The ``Calibration`` object, if loaded, or ``None`` if loading failed. """ try: @@ -131,7 +131,8 @@ def get_calibration( except Exception: cal = None logger.exception( - f"Unable to load {cal_type} calibration file, reverting to none" + f"Unable to load {cal_type} calibration file from {cal_file_path}." + + " Reverting to none" ) finally: return cal @@ -181,7 +182,7 @@ def get_calibration( # Calibration loading if not settings.RUNNING_TESTS: # Load the onboard cal file as the sensor calibration, if it exists - onboard_cal = get_calibration(settings.ONBOARD_CALIBRATION_FILE, "onboard") + onboard_cal = get_calibration(settings.ONBOARD_CALIBRATION_FILE, "sensor") if onboard_cal is not None: sensor_loader.sensor.sensor_calibration = onboard_cal else: From 2eb839f4eec5d249aa2661b16b5e5a4103382b2f Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 28 Mar 2024 17:27:35 -0400 Subject: [PATCH 56/56] simplify calibration file example remove preamp_enable and attenuation cal parameters from example JSON --- README.md | 40 +++++++++++++--------------------------- 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 8538a045..484caf8a 100644 --- a/README.md +++ b/README.md @@ -475,47 +475,33 @@ level setting of -25. "calibration_parameters": [ "sample_rate", "frequency", - "reference_level", - "preamp_enable", - "attenuation" + "reference_level" ], "clock_rate_lookup_by_sample_rate": [], "calibration_data": { "14000000.0": { "3545000000.0": { "-25": { - "true": { - "0": { - "datetime": "2023-10-23T14:38:02.882Z", - "gain": 30.09194805857024, - "noise_figure": 4.741521295220736, - "temperature": 15.6 - } - } + "datetime": "2023-10-23T14:38:02.882Z", + "gain": 30.09194805857024, + "noise_figure": 4.741521295220736, + "temperature": 15.6 } }, "3555000000.0": { "-25": { - "true": { - "0": { - "datetime": "2023-10-23T14:38:08.022Z", - "gain": 30.401008416406599, - "noise_figure": 4.394893979804061, - "temperature": 15.6 - } - } + "datetime": "2023-10-23T14:38:08.022Z", + "gain": 30.401008416406599, + "noise_figure": 4.394893979804061, + "temperature": 15.6 } }, "3565000000.0": { "-25": { - "true": { - "0": { - "datetime": "2023-10-23T14:38:11.922Z", - "gain": 30.848049817892105, - "noise_figure": 4.0751785215495819, - "temperature": 15.6 - } - } + "datetime": "2023-10-23T14:38:11.922Z", + "gain": 30.848049817892105, + "noise_figure": 4.0751785215495819, + "temperature": 15.6 } } }