From 5a010ac7a464a380b8278b49c1cdcc1fb7c47ee3 Mon Sep 17 00:00:00 2001 From: "dboulware@ntia.doc.gov" Date: Tue, 15 Oct 2024 13:59:21 -0600 Subject: [PATCH 1/9] Don't update calibration if NAN for gain or noise figure. --- scos_actions/actions/calibrate_y_factor.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index c8ca666b..83871127 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -346,11 +346,13 @@ def calibrate(self, params: dict): pwr_on_watts, pwr_off_watts, enr_linear, enbw_hz, temp_k ) - # Update sensor calibration with results - self.sensor.sensor_calibration.update( - sigan_params, utils.get_datetime_str_now(), gain, noise_figure, temp_c - ) - + if not np.isnan(gain) and not np.isnan(noise_figure): + # Update sensor calibration with results + self.sensor.sensor_calibration.update( + sigan_params, utils.get_datetime_str_now(), gain, noise_figure, temp_c + ) + else: + logger.warn(f"Not updating calibration. NF: {noise_figure}, Gain: {gain}") # Debugging noise_floor_dBm = convert_watts_to_dBm(Boltzmann * temp_k * enbw_hz) logger.debug(f"Noise floor: {noise_floor_dBm:.2f} dBm") From 4b9af01371f03817128602b356914b2238d19cee Mon Sep 17 00:00:00 2001 From: Anthony Romaniello <66272872+aromanielloNTIA@users.noreply.github.com> Date: Wed, 16 Oct 2024 13:31:48 -0400 Subject: [PATCH 2/9] Update logger and result outputs --- scos_actions/actions/calibrate_y_factor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 83871127..8f4838d4 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -352,15 +352,15 @@ def calibrate(self, params: dict): sigan_params, utils.get_datetime_str_now(), gain, noise_figure, temp_c ) else: - logger.warn(f"Not updating calibration. NF: {noise_figure}, Gain: {gain}") + logger.warning(f"Calibration result is NaN at {params[FREQUENCY]}:") + logger.warning(f"\tNF: {noise_figure}, Gain: {gain}") # Debugging noise_floor_dBm = convert_watts_to_dBm(Boltzmann * temp_k * enbw_hz) logger.debug(f"Noise floor: {noise_floor_dBm:.2f} dBm") logger.debug(f"Noise figure: {noise_figure:.2f} dB") logger.debug(f"Gain: {gain:.2f} dB") - # Detail results contain only FFT version of result for now - return f"Noise Figure: {noise_figure}, Gain: {gain}" + return f"Noise Figure: {noise_figure:.2f}, Gain: {gain:.2f}" @property def description(self): From 3b183afccdc8f6b732270560c563bf22f7a566c1 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello <66272872+aromanielloNTIA@users.noreply.github.com> Date: Thu, 7 Nov 2024 12:44:26 -0500 Subject: [PATCH 3/9] Update all pre-commit hooks --- .pre-commit-config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 90d2f9e4..fb8d7042 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,8 +1,8 @@ default_language_version: - python: python3.8 + python: python3.12 repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-ast types: [file, python] @@ -18,10 +18,10 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/asottile/pyupgrade - rev: v3.16.0 + rev: v3.19.0 hooks: - id: pyupgrade - args: ["--py38-plus"] + args: ["--py39-plus"] - repo: https://github.com/pycqa/isort rev: 5.13.2 hooks: @@ -30,12 +30,12 @@ repos: types: [file, python] args: ["--profile", "black", "--filter-files", "--gitignore"] - repo: https://github.com/psf/black - rev: 24.4.2 + rev: 24.10.0 hooks: - id: black types: [file, python] - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.41.0 + rev: v0.42.0 hooks: - id: markdownlint types: [file, markdown] From 68f6199ba631d742df47aa8a167cb2ff7ed9516f Mon Sep 17 00:00:00 2001 From: Anthony Romaniello <66272872+aromanielloNTIA@users.noreply.github.com> Date: Thu, 7 Nov 2024 13:06:54 -0500 Subject: [PATCH 4/9] revert to py38-plus --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fb8d7042..224405be 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,7 +21,7 @@ repos: rev: v3.19.0 hooks: - id: pyupgrade - args: ["--py39-plus"] + args: ["--py38-plus"] - repo: https://github.com/pycqa/isort rev: 5.13.2 hooks: From 252caa255d9af8578a0f07a3bb7cb1d1ceb98ad9 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello <66272872+aromanielloNTIA@users.noreply.github.com> Date: Thu, 7 Nov 2024 13:37:43 -0500 Subject: [PATCH 5/9] Add support and testing for Py 3.12 and 3.13 --- .github/workflows/tox.yaml | 2 ++ pyproject.toml | 2 ++ tox.ini | 4 ++++ 3 files changed, 8 insertions(+) diff --git a/.github/workflows/tox.yaml b/.github/workflows/tox.yaml index 2cf3db71..cdc4a100 100644 --- a/.github/workflows/tox.yaml +++ b/.github/workflows/tox.yaml @@ -37,6 +37,8 @@ jobs: - "3.9" - "3.10" - "3.11" + - "3.12" + - "3.13" steps: - name: Set up Python ${{ matrix.py }} uses: actions/setup-python@v4 diff --git a/pyproject.toml b/pyproject.toml index a34afe30..a29bd800 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,8 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ] dependencies = [ diff --git a/tox.ini b/tox.ini index f04c3a61..04b6072d 100644 --- a/tox.ini +++ b/tox.ini @@ -4,6 +4,8 @@ env_list = py39 py310 py311 + py312 + py313 min_version = 4.5.1 skip_missing_interpreters = true no_package = false @@ -21,3 +23,5 @@ python = 3.9 = py39 3.10 = py310 3.11 = py311 + 3.12 = py312 + 3.13 = py313 From 7ffcf5b6b5b3360ac6c2641839e2ee8c5038a0f5 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello <66272872+aromanielloNTIA@users.noreply.github.com> Date: Thu, 7 Nov 2024 13:52:42 -0500 Subject: [PATCH 6/9] Add simple one-time retry on failed calibration --- scos_actions/actions/calibrate_y_factor.py | 28 ++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 8f4838d4..1dc77ee4 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -255,10 +255,19 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int): # Run calibration routine for i, p in enumerate(self.iteration_params): + cal_result = self.calibrate(p) + # Retry once if channel calibration failed + if cal_result == "FAILED": + logger.warning(f"Retrying calibration at {p[FREQUENCY]/1e6} MHz") + cal_result = self.calibrate(p) + if cal_result == "FAILED": + logger.warning( + f"Retry failed. Calibration data not updated for f={p[FREQUENCY]}" + ) if i == 0: - detail += self.calibrate(p) + detail += cal_result else: - detail += os.linesep + self.calibrate(p) + detail += os.linesep + cal_result return detail def calibrate(self, params: dict): @@ -346,14 +355,25 @@ def calibrate(self, params: dict): pwr_on_watts, pwr_off_watts, enr_linear, enbw_hz, temp_k ) - if not np.isnan(gain) and not np.isnan(noise_figure): + # TODO: For testing, manually trigger NaN behavior on certain channels + if params[FREQUENCY] == 3535e6: + gain = np.nan + if params[FREQUENCY] == 3545e6: + noise_figure = np.nan + if params[FREQUENCY] == 3555e6: + noise_figure = np.inf + + if np.isfinite(gain) and np.isfinite(noise_figure): # Update sensor calibration with results self.sensor.sensor_calibration.update( sigan_params, utils.get_datetime_str_now(), gain, noise_figure, temp_c ) else: - logger.warning(f"Calibration result is NaN at {params[FREQUENCY]}:") + # At least one of {noise figure, gain} is NaN or infinite. This triggers + # a single retry for this set of params. See __call__ above. + logger.warning(f"Calibration result is NaN at {params[FREQUENCY]/1e6} MHz:") logger.warning(f"\tNF: {noise_figure}, Gain: {gain}") + return "FAILED" # Debugging noise_floor_dBm = convert_watts_to_dBm(Boltzmann * temp_k * enbw_hz) logger.debug(f"Noise floor: {noise_floor_dBm:.2f} dBm") From b0d3f756e4beb792e2da411415043b89a439db14 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello <66272872+aromanielloNTIA@users.noreply.github.com> Date: Thu, 7 Nov 2024 14:39:08 -0500 Subject: [PATCH 7/9] Remove default_language_version --- .pre-commit-config.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 224405be..030a7469 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,3 @@ -default_language_version: - python: python3.12 repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 From fcd790f82c812553d71c4cc0d6db8807765e7c71 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello <66272872+aromanielloNTIA@users.noreply.github.com> Date: Wed, 13 Nov 2024 16:49:35 -0500 Subject: [PATCH 8/9] limit numpy<2 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6e69ef93..b9057541 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ dependencies = [ "its_preselector @ git+https://github.com/NTIA/Preselector@3.1.0", "msgspec>=0.16.0,<1.0.0", "numexpr>=2.8.3", - "numpy>=1.25.0", + "numpy>=1.25.0,<2", "psutil>=5.9.4", "python-dateutil>=2.0", "ray>=2.10.0", From d847e1fffd1f843a4f9d7e706b463c9d39cb1334 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello <66272872+aromanielloNTIA@users.noreply.github.com> Date: Wed, 13 Nov 2024 16:55:14 -0500 Subject: [PATCH 9/9] Remove testing code which intentionally failed calibrations --- scos_actions/actions/calibrate_y_factor.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 1dc77ee4..e425bad3 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -355,14 +355,6 @@ def calibrate(self, params: dict): pwr_on_watts, pwr_off_watts, enr_linear, enbw_hz, temp_k ) - # TODO: For testing, manually trigger NaN behavior on certain channels - if params[FREQUENCY] == 3535e6: - gain = np.nan - if params[FREQUENCY] == 3545e6: - noise_figure = np.nan - if params[FREQUENCY] == 3555e6: - noise_figure = np.inf - if np.isfinite(gain) and np.isfinite(noise_figure): # Update sensor calibration with results self.sensor.sensor_calibration.update(