From 78732f22981df914ee1ce98aa1a9d22c8ff3e04b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Jun 2023 15:53:03 -0400 Subject: [PATCH 1/3] Bump cryptography from 39.0.1 to 41.0.0 in /src (#243) * Bump cryptography from 39.0.1 to 41.0.0 in /src Bumps [cryptography](https://github.com/pyca/cryptography) from 39.0.1 to 41.0.0. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/39.0.1...41.0.0) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Update requirements.in and run pip-compile --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Anthony Romaniello --- src/requirements-dev.txt | 2 +- src/requirements.in | 2 +- src/requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index c577cad0..3ce198ff 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -58,7 +58,7 @@ coreschema==0.0.4 # drf-yasg coverage[toml]==7.2.1 # via pytest-cov -cryptography==39.0.1 +cryptography==41.0.0 # via # -r requirements.txt # paramiko diff --git a/src/requirements.in b/src/requirements.in index 349f9c0c..e1baf249 100644 --- a/src/requirements.in +++ b/src/requirements.in @@ -19,5 +19,5 @@ scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@3.0.5 # The following are sub-dependencies for which SCOS Sensor enforces a # higher minimum patch version than the dependencies which require them. # This is done to ensure the inclusion of specific security patches. -cryptography>=39.0.1 # CVE-2023-0286 +cryptography>=41.0.0 # GHSA-5cpq-8wj7-hf2v sqlparse>=0.4.4 # CVE-2023-30608 diff --git a/src/requirements.txt b/src/requirements.txt index a69e1f96..064767ec 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -30,7 +30,7 @@ coreschema==0.0.4 # via # coreapi # drf-yasg -cryptography==39.0.1 +cryptography==41.0.0 # via # -r requirements.in # paramiko From f2de43979a8cc1a0f0b7809c2cf22c585a078da3 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello <66272872+aromanielloNTIA@users.noreply.github.com> Date: Fri, 23 Jun 2023 10:07:04 -0600 Subject: [PATCH 2/3] Update SCOS Actions for SigMF-NS-NTIA updates (#244) * testing update-sigmf-ns-ntia * Update sensor_definition.json * remove pywin dev requirement for linux testing * update requirements (add msgspec) * update pre-commit version * Update measurement_handler.py * revert attribute check to key check * do not overwrite sensor ID with FQDN * Remove old, unused schemas * remove sample count check this is no longer possible from the metadata, and the sha512 check makes it redundant * update requirements tagged releases for scos-actions and scos-tekrsa * increase wait time for container healthy in CI tests --- .github/workflows/github-actions-test.yml | 2 +- README.md | 4 +- configs/sensor_definition.json | 1 - schemas/scale_factors_schema.json | 66 --------- schemas/scos_transfer_spec_schema.json | 170 ---------------------- src/requirements-dev.in | 2 +- src/requirements-dev.txt | 24 +-- src/requirements.in | 4 +- src/requirements.txt | 20 +-- src/tasks/tests/test_archive_download.py | 13 -- 10 files changed, 25 insertions(+), 281 deletions(-) delete mode 100644 schemas/scale_factors_schema.json delete mode 100644 schemas/scos_transfer_spec_schema.json diff --git a/.github/workflows/github-actions-test.yml b/.github/workflows/github-actions-test.yml index f01ff98e..4cad4229 100644 --- a/.github/workflows/github-actions-test.yml +++ b/.github/workflows/github-actions-test.yml @@ -77,7 +77,7 @@ jobs: docker-compose up -d - name: Wait for containers # wait for containers to finish starting - run: sleep 20 + run: sleep 30 - name: Check API container run: docker ps | grep api | grep -q healthy diff --git a/README.md b/README.md index d2b2f4f7..34c9228c 100644 --- a/README.md +++ b/README.md @@ -203,7 +203,6 @@ actions. - entrypoints: Docker entrypoint scripts which are executed when starting a container. - gunicorn: Gunicorn configuration file. - nginx: Nginx configuration template and SSL certificates. -- schemas: JSON schema files. - scripts: Various utility scripts. - src: Contains the scos-sensor source code. - actions: Code to discover actions in plugins and to perform a simple logger action. @@ -339,14 +338,13 @@ settings in the environment file: This file contains information on the sensor and components being used. It is used in the SigMF metadata to identify the hardware used for the measurement. It should follow the [sigmf-ns-ntia Sensor Object format]( - + ). See an example below. Overwrite the [example file in scos-sensor/configs](configs/sensor_definition.json) with the information specific to the sensor you are using. ```json { - "id": "", "sensor_spec": { "id": "", "model": "greyhound" diff --git a/configs/sensor_definition.json b/configs/sensor_definition.json index 302335da..8e2689e8 100644 --- a/configs/sensor_definition.json +++ b/configs/sensor_definition.json @@ -1,5 +1,4 @@ { - "id": "OVERWRITTEN BY FQDN ENVIRONMENT VARIABLE", "location": { "x": -105.26208, "y": 39.99473, diff --git a/schemas/scale_factors_schema.json b/schemas/scale_factors_schema.json deleted file mode 100644 index 26cb1498..00000000 --- a/schemas/scale_factors_schema.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "definitions": { - "division": { - "type": "object", - "required": ["lower_bound", "upper_bound"], - "properties": { - "lower_bound": { - "type": "number", - "title": "Lower frequency bound of the division in Hertz" - }, - "upper_bound": { - "type": "number", - "title": "Upper frequency bound of the division in Hertz" - } - } - } - }, - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "title": "NTIA SCOS Sensor Scale Factor JSON Schema", - "required": ["frequencies", "gains", "factors"], - "properties": { - "divisions": { - "$id": "#/properties/divisions", - "type": "array", - "items": { - "$id": "#/properties/divisions/division", - "$ref": "#/definitions/division" - } - }, - "frequencies": { - "$id": "#/properties/frequencies", - "type": "array", - "uniqueItems": true, - "minItems": 1, - "items": { - "$id": "#/properties/frequencies/items", - "type": "number" - } - }, - "gains": { - "$id": "#/properties/gains", - "type": "array", - "uniqueItems": true, - "minItems": 1, - "items": { - "$id": "#/properties/gains/items", - "type": "number" - } - }, - "factors": { - "$id": "#/properties/factors", - "type": "array", - "minItems": 1, - "items": { - "$id": "#/properties/factors/items", - "type": "array", - "minItems": 1, - "items": { - "$id": "#/properties/factors/items/items", - "type": "number" - } - } - } - } -} diff --git a/schemas/scos_transfer_spec_schema.json b/schemas/scos_transfer_spec_schema.json deleted file mode 100644 index 550a98b6..00000000 --- a/schemas/scos_transfer_spec_schema.json +++ /dev/null @@ -1,170 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "definitions": { - "antenna_object": { - "properties": { - "cable_loss": { - "type": "number" - }, - "cross_polar_discrimination": { - "type": "number" - }, - "gain": { - "type": "number" - }, - "high_frequency": { - "type": "number" - }, - "horizontal_beam_width": { - "type": "number" - }, - "horizontal_gain_pattern": { - "type": "array" - }, - "low_frequency": { - "type": "number" - }, - "mobile": { - "type": "boolean" - }, - "model": { - "type": "string" - }, - "steerable": { - "type": "boolean" - }, - "type": { - "type": "string" - }, - "vertical_beam_width": { - "type": "number" - }, - "vertical_gain_pattern": { - "type": "array" - }, - "voltage_standing_wave_ratio": { - "type": "number" - } - }, - "required": [ - "model" - ], - "type": "object" - }, - "receiver_object": { - "properties": { - "high_frequency": { - "type": "number" - }, - "low_frequency": { - "type": "number" - }, - "max_power": { - "type": "number" - }, - "model": { - "type": "string" - }, - "noise_figure": { - "type": "number" - } - }, - "required": [ - "model" - ], - "type": "object" - }, - "single_frequency_fft_detection_object": { - "properties": { - "detector": { - "type": "string" - }, - "equivalent_noise_bandwidth": { - "type": "number" - }, - "number_of_ffts": { - "type": "integer" - }, - "number_of_samples_in_fft": { - "type": "integer" - }, - "reference": { - "type": "string" - }, - "units": { - "type": "string" - }, - "window": { - "type": "string" - } - }, - "required": [ - "number_of_samples_in_fft", - "window", - "detector", - "number_of_ffts", - "units" - ], - "type": "object" - } - }, - "properties": { - "annotations": { - "items": { - "properties": { - "scos:measurement_type": { - "properties": { - "single_frequency_fft_detection": { - "$ref": "#/definitions/single_frequency_fft_detection_object" - } - }, - "required": [ - "single_frequency_fft_detection" - ], - "type": "object" - } - }, - "required": [ - "scos:measurement_type" - ], - "type": "object" - }, - "type": "array" - }, - "global": { - "properties": { - "scos:sensor_definition": { - "properties": { - "antenna": { - "$ref": "#/definitions/antenna_object" - }, - "data_extraction_unit": { - "$ref": "#/definitions/reciever_object" - } - }, - "required": [ - "antenna", - "receiver" - ], - "type": "object" - }, - "scos:sensor_id": { - "type": "string" - }, - "scos:version": { - "type": "string" - } - }, - "required": [ - "scos:sensor_id", - "scos:version" - ], - "type": "object" - } - }, - "required": [ - "global", - "annotations" - ], - "type": "object" -} diff --git a/src/requirements-dev.in b/src/requirements-dev.in index 7f6d00c8..406d7e65 100644 --- a/src/requirements-dev.in +++ b/src/requirements-dev.in @@ -1,6 +1,6 @@ -rrequirements.txt -pre-commit>=2.0, <3.0 +pre-commit>=3.0, <4.0 pytest-cov>=3.0, <4.0 pytest-django>=4.0, <5.0 tox>=4.0,<5.0 diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index 3ce198ff..19c7d3d5 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -67,9 +67,7 @@ defusedxml==0.7.1 # -r requirements.txt # its-preselector distlib==0.3.6 - # via - # -r requirements.txt - # virtualenv + # via virtualenv distro==1.8.0 # via # -r requirements.txt @@ -140,7 +138,7 @@ inflection==0.5.1 # drf-yasg iniconfig==2.0.0 # via pytest -its-preselector @ git+https://github.com/NTIA/Preselector@3.0.1 +its-preselector @ git+https://github.com/NTIA/Preselector@3.0.2 # via # -r requirements.txt # scos-actions @@ -171,6 +169,10 @@ msgpack==1.0.4 # via # -r requirements.txt # ray +msgspec==0.16.0 + # via + # -r requirements.txt + # scos-actions nodeenv==1.7.0 # via pre-commit numexpr==2.8.4 @@ -198,6 +200,7 @@ packaging==23.0 # marshmallow # pyproject-api # pytest + # ray # tox paramiko==3.0.0 # via @@ -205,14 +208,13 @@ paramiko==3.0.0 # docker platformdirs==3.0.0 # via - # -r requirements.txt # tox # virtualenv pluggy==1.0.0 # via # pytest # tox -pre-commit==2.21.0 +pre-commit==3.3.3 # via -r requirements-dev.in protobuf==4.22.0 # via @@ -269,11 +271,11 @@ pyyaml==5.4.1 # docker-compose # pre-commit # ray -ray==2.3.0 +ray==2.5.0 # via # -r requirements.txt # scos-actions -requests==2.28.2 +requests==2.31.0 # via # -r requirements.txt # coreapi @@ -300,11 +302,11 @@ scipy==1.10.1 # via # -r requirements.txt # scos-actions -scos_actions @ git+https://github.com/NTIA/scos-actions@6.2.3 +scos_actions @ git+https://github.com/NTIA/scos-actions@6.3.0 # via # -r requirements.txt # scos-tekrsa -scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@3.0.5 +scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@3.1.0 # via -r requirements.txt sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive # via @@ -352,9 +354,7 @@ urllib3==1.26.14 # requests virtualenv==20.20.0 # via - # -r requirements.txt # pre-commit - # ray # tox websocket-client==0.59.0 # via diff --git a/src/requirements.in b/src/requirements.in index e1baf249..8d69192f 100644 --- a/src/requirements.in +++ b/src/requirements.in @@ -13,8 +13,8 @@ psycopg2-binary>=2.0, <3.0 pyjwt>=2.4.0, <3.0 requests-mock>=1.0, <2.0 requests_oauthlib>=1.0, <2.0 -scos_actions @ git+https://github.com/NTIA/scos-actions@6.2.3 -scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@3.0.5 +scos_actions @ git+https://github.com/NTIA/scos-actions@6.3.0 +scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@3.1.0 # The following are sub-dependencies for which SCOS Sensor enforces a # higher minimum patch version than the dependencies which require them. diff --git a/src/requirements.txt b/src/requirements.txt index 064767ec..bd858f96 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -36,8 +36,6 @@ cryptography==41.0.0 # paramiko defusedxml==0.7.1 # via its-preselector -distlib==0.3.6 - # via virtualenv distro==1.8.0 # via docker-compose django==3.2.19 @@ -73,7 +71,6 @@ filelock==3.9.0 # via # -r requirements.in # ray - # virtualenv frozenlist==1.3.3 # via # aiosignal @@ -86,7 +83,7 @@ idna==3.4 # via requests inflection==0.5.1 # via drf-yasg -its-preselector @ git+https://github.com/NTIA/Preselector@3.0.1 +its-preselector @ git+https://github.com/NTIA/Preselector@3.0.2 # via scos-actions itypes==1.2.0 # via coreapi @@ -104,6 +101,8 @@ marshmallow==3.19.0 # via environs msgpack==1.0.4 # via ray +msgspec==0.16.0 + # via scos-actions numexpr==2.8.4 # via scos-actions numpy==1.24.2 @@ -124,10 +123,9 @@ packaging==23.0 # docker # drf-yasg # marshmallow + # ray paramiko==3.0.0 # via docker -platformdirs==3.0.0 - # via virtualenv protobuf==4.22.0 # via ray psutil==5.9.4 @@ -157,9 +155,9 @@ pyyaml==5.4.1 # via # docker-compose # ray -ray==2.3.0 +ray==2.5.0 # via scos-actions -requests==2.28.2 +requests==2.31.0 # via # coreapi # docker @@ -180,11 +178,11 @@ ruamel-yaml-clib==0.2.7 # via ruamel-yaml scipy==1.10.1 # via scos-actions -scos_actions @ git+https://github.com/NTIA/scos-actions@6.2.3 +scos_actions @ git+https://github.com/NTIA/scos-actions@6.3.0 # via # -r requirements.in # scos-tekrsa -scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@3.0.5 +scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@3.1.0 # via -r requirements.in sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive # via scos-actions @@ -213,8 +211,6 @@ urllib3==1.26.14 # via # docker # requests -virtualenv==20.20.0 - # via ray websocket-client==0.59.0 # via # docker diff --git a/src/tasks/tests/test_archive_download.py b/src/tasks/tests/test_archive_download.py index 97c5180b..3ee78fe7 100644 --- a/src/tasks/tests/test_archive_download.py +++ b/src/tasks/tests/test_archive_download.py @@ -35,22 +35,9 @@ def test_single_acquisition_archive_download(admin_client, test_scheduler): sigmf_archive_contents = sigmf.sigmffile.fromarchive(tf.name) md = sigmf_archive_contents._metadata datafile = sigmf_archive_contents.data_file - datafile_actual_size = os.stat(datafile).st_size claimed_sha512 = md["global"]["core:sha512"] - # number_of_sample_arrays = len(md["annotations"]) - number_of_sample_arrays = 1 - cal_annotation = list( - filter( - lambda a: a["ntia-core:annotation_type"] == "CalibrationAnnotation", - md["annotations"], - ) - )[0] - samples_per_array = cal_annotation["core:sample_count"] - sample_array_size = samples_per_array * np.float32(0.0).nbytes - datafile_expected_size = number_of_sample_arrays * sample_array_size actual_sha512 = sigmf.sigmf_hash.calculate_sha512(datafile) - assert datafile_actual_size == datafile_expected_size assert claimed_sha512 == actual_sha512 From eb4be4d406003cc3d458a054fd2436528d14ec7c Mon Sep 17 00:00:00 2001 From: Anthony Romaniello <66272872+aromanielloNTIA@users.noreply.github.com> Date: Fri, 30 Jun 2023 09:01:08 -0600 Subject: [PATCH 3/3] Improve debugging tools for `ray` and segmentation faults (#245) * testing branches for scos-actions and scos-tekrsa * record Ray reference creation sites in debug mode * start ray dashboard in debug mode * use faulthandler in debug mode * Compile deps * remove ray init from scheduler * decrease ray version * Update Ray to 2.4.0 * tagged versions of actions and tekrsa * increase sleep time before CI container health check * try 60s wait for API container test * do not use grep quiet option using "-q", grep always has an exit status of 0. Without this flag, exit status is only 0 if a line is selected by grep * update pre-commit hooks * run pre-commit hooks on all files * use sea-prototype-v0.4.1 tags * recompile requirements * update markdownlint hook * revert unnecessary GH actions workflow changes * Bump minimum ray debug version to match scos-actions * Increase wait for container healthy time in GH Action * update scos-actions branch for testing * update scos-tekrsa version for testing * update requirements tagged scos plugin versions, other minor updates --- .github/workflows/github-actions-test.yml | 2 +- .pre-commit-config.yaml | 8 +- env.template | 1 + .../management/commands/get_auth_token.py | 2 +- src/handlers/measurement_handler.py | 2 +- src/requirements-dev.in | 1 + src/requirements-dev.txt | 81 +++++++++++++++++-- src/requirements.in | 4 +- src/requirements.txt | 12 +-- src/schedule/__init__.py | 2 +- src/schedule/models/request.py | 2 +- src/schedule/serializers.py | 2 +- src/schedule/tests/utils.py | 2 +- src/scheduler/scheduler.py | 10 +-- src/sensor/tests/test_api_docs.py | 2 +- src/sensor/urls.py | 2 +- src/sensor/wsgi.py | 6 ++ src/tasks/task_queue.py | 2 +- src/test_utils/task_test_utils.py | 4 +- 19 files changed, 111 insertions(+), 36 deletions(-) diff --git a/.github/workflows/github-actions-test.yml b/.github/workflows/github-actions-test.yml index 4cad4229..9830bde1 100644 --- a/.github/workflows/github-actions-test.yml +++ b/.github/workflows/github-actions-test.yml @@ -77,7 +77,7 @@ jobs: docker-compose up -d - name: Wait for containers # wait for containers to finish starting - run: sleep 30 + run: sleep 45 - name: Check API container run: docker ps | grep api | grep -q healthy diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index df0ae22e..b33e7e19 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,10 +19,10 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/asottile/pyupgrade - rev: v3.3.1 + rev: v3.4.0 hooks: - id: pyupgrade - args: ["--py3-plus"] + args: ["--py38-plus"] - repo: https://github.com/pycqa/isort rev: 5.12.0 hooks: @@ -31,12 +31,12 @@ repos: types: [file, python] args: ["--profile", "black", "--filter-files", "--gitignore"] - repo: https://github.com/psf/black - rev: 23.1.0 + rev: 23.3.0 hooks: - id: black types: [file, python] - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.33.0 + rev: v0.35.0 hooks: - id: markdownlint types: [file, markdown] diff --git a/env.template b/env.template index ddf4bee0..ea47fc9c 100644 --- a/env.template +++ b/env.template @@ -48,6 +48,7 @@ SENTRY_DSN= if $DEBUG; then GUNICORN_LOG_LEVEL=debug + RAY_record_ref_creation_sites=1 else GUNICORN_LOG_LEVEL=info fi diff --git a/src/authentication/management/commands/get_auth_token.py b/src/authentication/management/commands/get_auth_token.py index a80b266e..8101f9a5 100644 --- a/src/authentication/management/commands/get_auth_token.py +++ b/src/authentication/management/commands/get_auth_token.py @@ -17,5 +17,5 @@ def handle(self, *args, **options): self.stdout.write(key) else: self.stdout.write( - "No user with username={} found in the database.".format(username) + f"No user with username={username} found in the database." ) diff --git a/src/handlers/measurement_handler.py b/src/handlers/measurement_handler.py index 8f2b686e..909d6947 100644 --- a/src/handlers/measurement_handler.py +++ b/src/handlers/measurement_handler.py @@ -56,4 +56,4 @@ def measurement_action_completed_callback(sender, **kwargs): acquisition.data_encrypted = False acquisition.save() - logger.debug("Saved new file at {}".format(acquisition.data.path)) + logger.debug(f"Saved new file at {acquisition.data.path}") diff --git a/src/requirements-dev.in b/src/requirements-dev.in index 406d7e65..fa6b2888 100644 --- a/src/requirements-dev.in +++ b/src/requirements-dev.in @@ -3,4 +3,5 @@ pre-commit>=3.0, <4.0 pytest-cov>=3.0, <4.0 pytest-django>=4.0, <5.0 +ray[default]>=2.5.0 tox>=4.0,<5.0 diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index 19c7d3d5..ef64d11a 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -4,17 +4,27 @@ # # pip-compile requirements-dev.in # +aiohttp==3.8.4 + # via + # aiohttp-cors + # ray +aiohttp-cors==0.7.0 + # via ray aiosignal==1.3.1 # via # -r requirements.txt + # aiohttp # ray asgiref==3.6.0 # via # -r requirements.txt # django +async-timeout==4.0.2 + # via aiohttp attrs==22.2.0 # via # -r requirements.txt + # aiohttp # jsonschema # pytest # ray @@ -22,8 +32,12 @@ bcrypt==4.0.1 # via # -r requirements.txt # paramiko +blessed==1.20.0 + # via gpustat cachetools==5.3.0 - # via tox + # via + # google-auth + # tox certifi==2022.12.7 # via # -r requirements.txt @@ -40,6 +54,7 @@ chardet==5.1.0 charset-normalizer==3.0.1 # via # -r requirements.txt + # aiohttp # requests click==8.1.3 # via @@ -47,6 +62,8 @@ click==8.1.3 # ray colorama==0.4.6 # via tox +colorful==0.5.5 + # via ray coreapi==2.3.3 # via # -r requirements.txt @@ -118,8 +135,17 @@ filelock==3.9.0 frozenlist==1.3.3 # via # -r requirements.txt + # aiohttp # aiosignal # ray +google-api-core==2.11.0 + # via opencensus +google-auth==2.17.3 + # via google-api-core +googleapis-common-protos==1.59.0 + # via google-api-core +gpustat==1.1 + # via ray grpcio==1.51.3 # via # -r requirements.txt @@ -132,6 +158,7 @@ idna==3.4 # via # -r requirements.txt # requests + # yarl inflection==0.5.1 # via # -r requirements.txt @@ -165,7 +192,7 @@ marshmallow==3.19.0 # via # -r requirements.txt # environs -msgpack==1.0.4 +msgpack==1.0.5 # via # -r requirements.txt # ray @@ -173,6 +200,10 @@ msgspec==0.16.0 # via # -r requirements.txt # scos-actions +multidict==6.0.4 + # via + # aiohttp + # yarl nodeenv==1.7.0 # via pre-commit numexpr==2.8.4 @@ -188,10 +219,16 @@ numpy==1.24.2 # scos-actions # sigmf # tekrsa-api-wrap +nvidia-ml-py==11.525.112 + # via gpustat oauthlib==3.2.2 # via # -r requirements.txt # requests-oauthlib +opencensus==0.11.2 + # via ray +opencensus-context==0.1.3 + # via opencensus packaging==23.0 # via # -r requirements.txt @@ -216,20 +253,35 @@ pluggy==1.0.0 # tox pre-commit==3.3.3 # via -r requirements-dev.in -protobuf==4.22.0 +prometheus-client==0.13.1 + # via ray +protobuf==4.23.3 # via # -r requirements.txt + # google-api-core + # googleapis-common-protos # ray -psutil==5.9.4 +psutil==5.9.5 # via # -r requirements.txt + # gpustat # scos-actions psycopg2-binary==2.9.5 # via -r requirements.txt +py-spy==0.3.14 + # via ray +pyasn1==0.5.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 + # via google-auth pycparser==2.21 # via # -r requirements.txt # cffi +pydantic==1.10.7 + # via ray pyjwt==2.6.0 # via -r requirements.txt pynacl==1.5.0 @@ -271,8 +323,9 @@ pyyaml==5.4.1 # docker-compose # pre-commit # ray -ray==2.5.0 +ray[default]==2.5.1 # via + # -r requirements-dev.in # -r requirements.txt # scos-actions requests==2.31.0 @@ -281,6 +334,7 @@ requests==2.31.0 # coreapi # docker # docker-compose + # google-api-core # its-preselector # ray # requests-mock @@ -289,6 +343,8 @@ requests-mock==1.10.0 # via -r requirements.txt requests-oauthlib==1.3.1 # via -r requirements.txt +rsa==4.9 + # via google-auth ruamel-yaml==0.17.21 # via # -r requirements.txt @@ -302,11 +358,11 @@ scipy==1.10.1 # via # -r requirements.txt # scos-actions -scos_actions @ git+https://github.com/NTIA/scos-actions@6.3.0 +scos_actions @ git+https://github.com/NTIA/scos-actions@6.3.1 # via # -r requirements.txt # scos-tekrsa -scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@3.1.0 +scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@3.1.1 # via -r requirements.txt sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive # via @@ -315,13 +371,17 @@ sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive six==1.16.0 # via # -r requirements.txt + # blessed # django-session-timeout # dockerpty + # google-auth # jsonschema # python-dateutil # requests-mock # sigmf # websocket-client +smart-open==6.3.0 + # via ray sqlparse==0.4.4 # via # -r requirements.txt @@ -342,6 +402,8 @@ tomli==2.0.1 # tox tox==4.4.6 # via -r requirements-dev.in +typing-extensions==4.5.0 + # via pydantic uritemplate==4.1.1 # via # -r requirements.txt @@ -355,12 +417,17 @@ urllib3==1.26.14 virtualenv==20.20.0 # via # pre-commit + # ray # tox +wcwidth==0.2.6 + # via blessed websocket-client==0.59.0 # via # -r requirements.txt # docker # docker-compose +yarl==1.9.2 + # via aiohttp # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/src/requirements.in b/src/requirements.in index 8d69192f..0971eb4f 100644 --- a/src/requirements.in +++ b/src/requirements.in @@ -13,8 +13,8 @@ psycopg2-binary>=2.0, <3.0 pyjwt>=2.4.0, <3.0 requests-mock>=1.0, <2.0 requests_oauthlib>=1.0, <2.0 -scos_actions @ git+https://github.com/NTIA/scos-actions@6.3.0 -scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@3.1.0 +scos_actions @ git+https://github.com/NTIA/scos-actions@6.3.1 +scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@3.1.1 # The following are sub-dependencies for which SCOS Sensor enforces a # higher minimum patch version than the dependencies which require them. diff --git a/src/requirements.txt b/src/requirements.txt index bd858f96..bb8ec781 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -99,7 +99,7 @@ markupsafe==2.1.2 # via jinja2 marshmallow==3.19.0 # via environs -msgpack==1.0.4 +msgpack==1.0.5 # via ray msgspec==0.16.0 # via scos-actions @@ -126,9 +126,9 @@ packaging==23.0 # ray paramiko==3.0.0 # via docker -protobuf==4.22.0 +protobuf==4.23.3 # via ray -psutil==5.9.4 +psutil==5.9.5 # via scos-actions psycopg2-binary==2.9.5 # via -r requirements.in @@ -155,7 +155,7 @@ pyyaml==5.4.1 # via # docker-compose # ray -ray==2.5.0 +ray==2.5.1 # via scos-actions requests==2.31.0 # via @@ -178,11 +178,11 @@ ruamel-yaml-clib==0.2.7 # via ruamel-yaml scipy==1.10.1 # via scos-actions -scos_actions @ git+https://github.com/NTIA/scos-actions@6.3.0 +scos_actions @ git+https://github.com/NTIA/scos-actions@6.3.1 # via # -r requirements.in # scos-tekrsa -scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@3.1.0 +scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@3.1.1 # via -r requirements.in sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive # via scos-actions diff --git a/src/schedule/__init__.py b/src/schedule/__init__.py index ab7450ba..da07553f 100644 --- a/src/schedule/__init__.py +++ b/src/schedule/__init__.py @@ -10,7 +10,7 @@ def get_action_with_summary(action): summary = get_summary(action_fn) action_with_summary = action if summary: - action_with_summary += " - {}".format(summary) + action_with_summary += f" - {summary}" return action_with_summary diff --git a/src/schedule/models/request.py b/src/schedule/models/request.py index 96cfdca8..9e919cd9 100644 --- a/src/schedule/models/request.py +++ b/src/schedule/models/request.py @@ -15,7 +15,7 @@ class Request(models.Model): def build_absolute_uri(self, location=None): """Called from within Django reverse.""" - scheme_host = "{}://{}".format(self.scheme, self.host) + scheme_host = f"{self.scheme}://{self.host}" return iri_to_uri(scheme_host + location) def from_drf_request(self, request, commit=True): diff --git a/src/schedule/serializers.py b/src/schedule/serializers.py index 3c6fda7b..27788b73 100644 --- a/src/schedule/serializers.py +++ b/src/schedule/serializers.py @@ -14,7 +14,7 @@ from .models import DEFAULT_PRIORITY, ScheduleEntry action_help = "[Required] The name of the action to be scheduled" -priority_help = "Lower number is higher priority (default={})".format(DEFAULT_PRIORITY) +priority_help = f"Lower number is higher priority (default={DEFAULT_PRIORITY})" CHOICES = [] actions = sorted(registered_actions.keys()) for action in actions: diff --git a/src/schedule/tests/utils.py b/src/schedule/tests/utils.py index 8ede195b..ac8304be 100644 --- a/src/schedule/tests/utils.py +++ b/src/schedule/tests/utils.py @@ -30,7 +30,7 @@ def post_schedule(client, entry, expected_status=status.HTTP_201_CREATED): url = reverse("schedule-list", kwargs=V1) r = client.post(url, **kwargs) - err = "Got status {}, expected {}".format(r.status_code, expected_status) + err = f"Got status {r.status_code}, expected {expected_status}" assert r.status_code == expected_status, err if r.status_code == status.HTTP_204_NO_CONTENT: diff --git a/src/scheduler/scheduler.py b/src/scheduler/scheduler.py index 011306cb..c81944a3 100644 --- a/src/scheduler/scheduler.py +++ b/src/scheduler/scheduler.py @@ -162,7 +162,7 @@ def _call_task_action(self): schedule_entry_json["id"] = entry_name try: - logger.debug("running task {}/{}".format(entry_name, task_id)) + logger.debug(f"running task {entry_name}/{task_id}") detail = self.task.action_caller(schedule_entry_json, task_id) self.delayfn(0) # let other threads run status = "success" @@ -170,7 +170,7 @@ def _call_task_action(self): detail = "" except Exception as err: detail = str(err) - logger.exception("action failed: {}".format(detail)) + logger.exception(f"action failed: {detail}") status = "failure" return status, detail[:MAX_DETAIL_LEN] @@ -225,7 +225,7 @@ def _finalize_task_result(self, started, finished, status, detail): @staticmethod def _callback_response_handler(resp, task_result): if resp.ok: - logger.info("POSTed to {}".format(resp.url)) + logger.info(f"POSTed to {resp.url}") else: msg = "Failed to POST to {}: {}" logger.warning(msg.format(resp.url, resp.reason)) @@ -297,7 +297,7 @@ def _get_min_interval(self, schedule_snapshot): def _cancel_if_completed(self, entry): if not entry.has_remaining_times(): - msg = "no times remaining in {}, removing".format(entry.name) + msg = f"no times remaining in {entry.name}, removing" logger.debug(msg) self.cancel(entry) @@ -309,7 +309,7 @@ def status(self): def __repr__(self): s = "running" if self.running else "stopped" - return "<{} status={}>".format(self.__class__.__name__, s) + return f"<{self.__class__.__name__} status={s}>" @contextmanager diff --git a/src/sensor/tests/test_api_docs.py b/src/sensor/tests/test_api_docs.py index 5f73b507..d5b4f150 100644 --- a/src/sensor/tests/test_api_docs.py +++ b/src/sensor/tests/test_api_docs.py @@ -12,7 +12,7 @@ def test_api_docs_up_to_date(admin_client): docs_dir = path.dirname(settings.OPENAPI_FILE) if not path.exists(docs_dir): # Probably running in Docker container for Jenkins... test should pass - print("{} doesn't exist, not in src tree.".format(docs_dir)) + print(f"{docs_dir} doesn't exist, not in src tree.") return True schema_url = reverse("api_schema", kwargs=V1) + "?format=openapi" diff --git a/src/sensor/urls.py b/src/sensor/urls.py index b04a8d00..2f469f8b 100644 --- a/src/sensor/urls.py +++ b/src/sensor/urls.py @@ -55,7 +55,7 @@ urlpatterns = [ path("", RedirectView.as_view(url="/api/")), path("admin/", admin.site.urls), - path("api/", RedirectView.as_view(url="/api/{}/".format(DEFAULT_API_VERSION))), + path("api/", RedirectView.as_view(url=f"/api/{DEFAULT_API_VERSION}/")), re_path(API_PREFIX, include(api_urlpatterns)), path("api/auth/", include("rest_framework.urls")), ] diff --git a/src/sensor/wsgi.py b/src/sensor/wsgi.py index a6e9bb7c..703f9628 100644 --- a/src/sensor/wsgi.py +++ b/src/sensor/wsgi.py @@ -20,6 +20,12 @@ from scheduler import scheduler # noqa from sensor import settings # noqa +if settings.DEBUG: + # Handle segmentation faults in DEBUG mode + import faulthandler + + faulthandler.enable() + application = get_wsgi_application() if not settings.IN_DOCKER: diff --git a/src/tasks/task_queue.py b/src/tasks/task_queue.py index ef3d126f..577b3ac9 100644 --- a/src/tasks/task_queue.py +++ b/src/tasks/task_queue.py @@ -42,4 +42,4 @@ def __getitem__(self, item): return self.to_list()[item] def __repr__(self): - return "<{} {!r}>".format(self.__class__.__name__, list(self)) + return f"<{self.__class__.__name__} {list(self)!r}>" diff --git a/src/test_utils/task_test_utils.py b/src/test_utils/task_test_utils.py index 2f8534ce..7528f722 100644 --- a/src/test_utils/task_test_utils.py +++ b/src/test_utils/task_test_utils.py @@ -143,7 +143,7 @@ def reverse_result_detail(schedule_entry_name, task_id): def reverse_archive(schedule_entry_name, task_id): rf = RequestFactory() entry_name = schedule_entry_name - url = "/tasks/completed/{}/{!s}/archive".format(entry_name, task_id) + url = f"/tasks/completed/{entry_name}/{task_id!s}/archive" request = rf.get(url, **HTTPS_KWARG) kws = {"schedule_entry_name": entry_name, "task_id": task_id} kws.update(V1) @@ -153,7 +153,7 @@ def reverse_archive(schedule_entry_name, task_id): def reverse_archive_all(schedule_entry_name): rf = RequestFactory() entry_name = schedule_entry_name - url = "/tasks/completed/{}/archive".format(entry_name) + url = f"/tasks/completed/{entry_name}/archive" request = rf.get(url, **HTTPS_KWARG) kws = {"schedule_entry_name": entry_name} kws.update(V1)