From 06f9965f165a29a1fa3506366edc9fb89e2cfe52 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 29 Jul 2022 09:28:19 -0600 Subject: [PATCH 001/255] Initial implementation of certificate rest api authentication --- nginx/conf.template | 2 +- scripts/create_superuser.py | 1 + src/authentication/auth.py | 80 +++++++++++-------------------------- src/scheduler/scheduler.py | 9 +++-- src/sensor/settings.py | 4 +- 5 files changed, 33 insertions(+), 63 deletions(-) diff --git a/nginx/conf.template b/nginx/conf.template index 30a4750a..ed438618 100644 --- a/nginx/conf.template +++ b/nginx/conf.template @@ -32,7 +32,7 @@ server { ssl_certificate_key /etc/ssl/private/ssl-cert.key; ssl_protocols TLSv1.2; ssl_client_certificate /etc/ssl/certs/ca.crt; - # ssl_verify_client on; + ssl_verify_client on; # ssl_ocsp on; # Enable OCSP validation ssl_verify_depth 4; # path for static files diff --git a/scripts/create_superuser.py b/scripts/create_superuser.py index 00a4a6a1..145e248f 100755 --- a/scripts/create_superuser.py +++ b/scripts/create_superuser.py @@ -27,6 +27,7 @@ UserModel = get_user_model() try: + username = os.environ["ADMIN_NAME"] admin_user = UserModel._default_manager.get(username="admin") admin_user.email = email admin_user.set_password(password) diff --git a/src/authentication/auth.py b/src/authentication/auth.py index a8a94069..294a5dc5 100644 --- a/src/authentication/auth.py +++ b/src/authentication/auth.py @@ -1,5 +1,5 @@ import logging - +import re import jwt from django.conf import settings from django.contrib.auth import get_user_model @@ -13,74 +13,42 @@ "rest_framework.authentication.TokenAuthentication" in settings.REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"] ) -oauth_jwt_authentication_enabled = ( - "authentication.auth.OAuthJWTAuthentication" +certificate_authentication_enabled = ( + "authentication.auth.CertificateAuthentication" in settings.REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"] ) -def jwt_request_has_required_role(request): - if request.auth: - if "authorities" in request.auth: - if request.auth["authorities"]: - authorities = request.auth["authorities"] - return settings.REQUIRED_ROLE.upper() in authorities - return False -class OAuthJWTAuthentication(authentication.BaseAuthentication): + +class CertificateAuthentication(authentication.BaseAuthentication): def authenticate(self, request): auth_header = get_authorization_header(request) if not auth_header: logger.debug("no auth header") return None - auth_header = auth_header.split() - if len(auth_header) != 2: - return None - if auth_header[0].decode().lower() != "bearer": - logger.debug("no JWT bearer token") - return None # attempt other configured authentication methods - token = auth_header[1] - # get JWT public key - public_key = "" - try: - with open(settings.PATH_TO_JWT_PUBLIC_KEY) as public_key_file: - public_key = public_key_file.read() - except Exception as e: - logger.error(e) - if not public_key: - error = exceptions.AuthenticationFailed( - "Unable to get public key to decode jwt" - ) - logger.error(error) - raise error - try: - # decode JWT token - # verifies jwt signature using RS256 algorithm and public key - # requires exp claim to verify token is not expired - # decodes and returns base64 encoded payload - decoded_key = jwt.decode( - token, - public_key, - verify=True, - algorithms="RS256", - options={"require": ["exp"], "verify_exp": True}, - ) - except ExpiredSignatureError as e: - logger.error(e) - raise exceptions.AuthenticationFailed("Token is expired!") - except InvalidSignatureError as e: - logger.error(e) - raise exceptions.AuthenticationFailed("Unable to verify token!") - except Exception as e: - logger.error(e) - raise exceptions.AuthenticationFailed(f"Unable to decode token! {e}") - jwt_username = decoded_key["user_name"] + cert_dn = request.headers.get("X-Ssl-Client-Dn") + logger.info("DN:" + cert_dn) + cn = get_cn_from_dn(cert_dn) + logger.info("Cert cn: " + cn) user_model = get_user_model() user = None try: - user = user_model.objects.get(username=jwt_username) + user = user_model.objects.get(username=cn) except user_model.DoesNotExist: - user = user_model.objects.create_user(username=jwt_username) + user = user_model.objects.create_user(username=cn) user.save() - return (user, decoded_key) + + return user + +def get_cn_from_dn(cert_dn): + p = re.compile("CN=(.*?)(?:,|\+|$)") + match = p.search(cert_dn) + if not match: + raise Exception("No CN found in certificate!") + uid_raw = match.group() + # logger.debug(f"uid_raw = {uid_raw}") + uid = uid_raw.split("=")[1].rstrip(",") + # logger.debug(f"uid = {uid}") + return uid diff --git a/src/scheduler/scheduler.py b/src/scheduler/scheduler.py index 2bfb1b18..63fab400 100644 --- a/src/scheduler/scheduler.py +++ b/src/scheduler/scheduler.py @@ -182,14 +182,15 @@ def _finalize_task_result(self, started, finished, status, detail): if settings.PATH_TO_VERIFY_CERT != "": verify_ssl = settings.PATH_TO_VERIFY_CERT logger.debug(settings.CALLBACK_AUTHENTICATION) - if settings.CALLBACK_AUTHENTICATION == "OAUTH": - client = oauth.get_oauth_client() + if settings.CALLBACK_AUTHENTICATION == "CERT": headers = {"Content-Type": "application/json"} - response = client.post( + + response = requests.post( self.entry.callback_url, data=json.dumps(result_json), headers=headers, - verify=verify_ssl + verify=verify_ssl, + cert=(settings.PATH_TO_CLIENT_CERT, settings.PATH_TO_CLIENT_KEY) ) self._callback_response_handler(response, tr) else: diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 2ead2102..7da628eb 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -244,9 +244,9 @@ } AUTHENTICATION = env("AUTHENTICATION", default="") -if AUTHENTICATION == "JWT": +if AUTHENTICATION == "CERT": REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"] = ( - "authentication.auth.OAuthJWTAuthentication", + "authentication.auth.CertificateAuthentication", "rest_framework.authentication.SessionAuthentication", ) else: From 5664774e5dd21dbf0d0706601a8ec5c3c5694ccd Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 29 Jul 2022 10:15:44 -0600 Subject: [PATCH 002/255] Use env variable to set admin user name. --- scripts/create_superuser.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/create_superuser.py b/scripts/create_superuser.py index 145e248f..dea05b30 100755 --- a/scripts/create_superuser.py +++ b/scripts/create_superuser.py @@ -25,13 +25,12 @@ sys.exit(0) UserModel = get_user_model() - +username = os.environ["ADMIN_NAME"] try: - username = os.environ["ADMIN_NAME"] - admin_user = UserModel._default_manager.get(username="admin") + admin_user = UserModel._default_manager.get(username=username) admin_user.email = email admin_user.set_password(password) print("Reset admin account password and email from environment") except UserModel.DoesNotExist: - UserModel._default_manager.create_superuser("admin", email, password) + UserModel._default_manager.create_superuser(username, email, password) print("Created admin account with password and email from environment") From c0e2574ce286cd476b89144c0fb121427ffe8c47 Mon Sep 17 00:00:00 2001 From: dboulware Date: Fri, 20 Jan 2023 09:59:32 -0700 Subject: [PATCH 003/255] Add ADMIN_NAME setting to be able to pass CN of admin user do db creation. Add setting for NGINX to set DN. --- docker-compose.yml | 1 + env.template | 3 ++- nginx/conf.template | 4 ++- src/authentication/auth.py | 41 ++++++++++++++----------------- src/authentication/permissions.py | 4 --- src/sensor/settings.py | 1 + 6 files changed, 25 insertions(+), 29 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4c1d7487..c885fe86 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,6 +31,7 @@ services: - DEBUG - DOCKER_GIT_CREDENTIALS environment: + - ADMIN_NAME - ADMIN_EMAIL - ADMIN_PASSWORD - AUTHENTICATION diff --git a/env.template b/env.template index 7ecd10fc..716fc42d 100644 --- a/env.template +++ b/env.template @@ -72,4 +72,5 @@ PATH_TO_VERIFY_CERT=scos_test_ca.crt # Path relative to configs/certs PATH_TO_JWT_PUBLIC_KEY=jwt_pubkey.pem # set to JWT to enable JWT authentication -AUTHENTICATION=TOKEN +AUTHENTICATION=CERT +ADMIN_NAME=Admin diff --git a/nginx/conf.template b/nginx/conf.template index ed438618..7b79bcac 100644 --- a/nginx/conf.template +++ b/nginx/conf.template @@ -32,7 +32,7 @@ server { ssl_certificate_key /etc/ssl/private/ssl-cert.key; ssl_protocols TLSv1.2; ssl_client_certificate /etc/ssl/certs/ca.crt; - ssl_verify_client on; + ssl_verify_client optional; # ssl_ocsp on; # Enable OCSP validation ssl_verify_depth 4; # path for static files @@ -50,8 +50,10 @@ server { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto https; proxy_set_header Host $http_host; + proxy_set_header X-SSL-CLIENT-DN $ssl_client_s_dn; proxy_redirect off; proxy_pass http://wsgi-server; + } } diff --git a/src/authentication/auth.py b/src/authentication/auth.py index 294a5dc5..0bf2ebf6 100644 --- a/src/authentication/auth.py +++ b/src/authentication/auth.py @@ -3,44 +3,39 @@ import jwt from django.conf import settings from django.contrib.auth import get_user_model -from jwt import ExpiredSignatureError, InvalidSignatureError from rest_framework import authentication, exceptions from rest_framework.authentication import get_authorization_header logger = logging.getLogger(__name__) token_auth_enabled = ( - "rest_framework.authentication.TokenAuthentication" - in settings.REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"] + "rest_framework.authentication.TokenAuthentication" + in settings.REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"] ) certificate_authentication_enabled = ( - "authentication.auth.CertificateAuthentication" - in settings.REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"] + "authentication.auth.CertificateAuthentication" + in settings.REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"] ) - - - class CertificateAuthentication(authentication.BaseAuthentication): def authenticate(self, request): - auth_header = get_authorization_header(request) - if not auth_header: - logger.debug("no auth header") - return None + logger.debug("Authenticating certificate.") cert_dn = request.headers.get("X-Ssl-Client-Dn") - logger.info("DN:" + cert_dn) - cn = get_cn_from_dn(cert_dn) - logger.info("Cert cn: " + cn) - user_model = get_user_model() - user = None - try: - user = user_model.objects.get(username=cn) - except user_model.DoesNotExist: - user = user_model.objects.create_user(username=cn) - user.save() + if cert_dn: + logger.info("DN:" + cert_dn) + cn = get_cn_from_dn(cert_dn) + logger.info("Cert cn: " + cn) + user_model = get_user_model() + user = None + try: + user = user_model.objects.get(username=cn) + except user_model.DoesNotExist: + user = user_model.objects.create_user(username=cn) + user.save() + return user, None + return None, None - return user def get_cn_from_dn(cert_dn): p = re.compile("CN=(.*?)(?:,|\+|$)") diff --git a/src/authentication/permissions.py b/src/authentication/permissions.py index 40c55ec6..4a2beafa 100644 --- a/src/authentication/permissions.py +++ b/src/authentication/permissions.py @@ -1,14 +1,10 @@ from rest_framework import permissions -from .auth import jwt_request_has_required_role, oauth_jwt_authentication_enabled - class RequiredJWTRolePermissionOrIsSuperuser(permissions.BasePermission): message = "User missing required role" def has_permission(self, request, view): - if oauth_jwt_authentication_enabled and jwt_request_has_required_role(request): - return True if request.user.is_superuser: return True return False diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 7da628eb..e3dfd1b5 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -246,6 +246,7 @@ AUTHENTICATION = env("AUTHENTICATION", default="") if AUTHENTICATION == "CERT": REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"] = ( + "rest_framework.authentication.TokenAuthentication", "authentication.auth.CertificateAuthentication", "rest_framework.authentication.SessionAuthentication", ) From 04c1f55a09176798f21568e2dcbec008fa857d5a Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Mon, 23 Jan 2023 11:21:59 -0700 Subject: [PATCH 004/255] raise exception if user not found, remove token auth and session auth --- src/authentication/auth.py | 3 +-- src/sensor/settings.py | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/authentication/auth.py b/src/authentication/auth.py index 0bf2ebf6..e51c88ad 100644 --- a/src/authentication/auth.py +++ b/src/authentication/auth.py @@ -31,8 +31,7 @@ def authenticate(self, request): try: user = user_model.objects.get(username=cn) except user_model.DoesNotExist: - user = user_model.objects.create_user(username=cn) - user.save() + raise exceptions.AuthenticationFailed("No matching username found!") return user, None return None, None diff --git a/src/sensor/settings.py b/src/sensor/settings.py index e3dfd1b5..a1d16628 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -246,9 +246,7 @@ AUTHENTICATION = env("AUTHENTICATION", default="") if AUTHENTICATION == "CERT": REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"] = ( - "rest_framework.authentication.TokenAuthentication", "authentication.auth.CertificateAuthentication", - "rest_framework.authentication.SessionAuthentication", ) else: REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"] = ( From 16b3f05376fa3e4a7b3873a95f5109a857ed5555 Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Mon, 23 Jan 2023 12:15:41 -0700 Subject: [PATCH 005/255] cert auth in migration and runtime settings --- src/sensor/migration_settings.py | 5 ++--- src/sensor/runtime_settings.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/sensor/migration_settings.py b/src/sensor/migration_settings.py index 614f058f..954b044f 100644 --- a/src/sensor/migration_settings.py +++ b/src/sensor/migration_settings.py @@ -239,10 +239,9 @@ } AUTHENTICATION = env("AUTHENTICATION", default="") -if AUTHENTICATION == "JWT": +if AUTHENTICATION == "CERT": REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"] = ( - "authentication.auth.OAuthJWTAuthentication", - "rest_framework.authentication.SessionAuthentication", + "authentication.auth.CertificateAuthentication", ) else: REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"] = ( diff --git a/src/sensor/runtime_settings.py b/src/sensor/runtime_settings.py index 8fd66452..09df46df 100644 --- a/src/sensor/runtime_settings.py +++ b/src/sensor/runtime_settings.py @@ -239,10 +239,9 @@ } AUTHENTICATION = env("AUTHENTICATION", default="") -if AUTHENTICATION == "JWT": +if AUTHENTICATION == "CERT": REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"] = ( - "authentication.auth.OAuthJWTAuthentication", - "rest_framework.authentication.SessionAuthentication", + "authentication.auth.CertificateAuthentication", ) else: REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"] = ( From 9cb4c80d9e863b0f317ae0ed261f9f5c88176195 Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Mon, 23 Jan 2023 12:35:36 -0700 Subject: [PATCH 006/255] update requirements --- src/requirements-dev.in | 2 +- src/requirements-dev.txt | 54 ++++++++++++++++------------------------ src/requirements.in | 2 -- src/requirements.txt | 39 +++++++++++------------------ 4 files changed, 38 insertions(+), 59 deletions(-) diff --git a/src/requirements-dev.in b/src/requirements-dev.in index 8a629ed0..82b5b2c5 100644 --- a/src/requirements-dev.in +++ b/src/requirements-dev.in @@ -1,6 +1,6 @@ -rrequirements.txt -cryptography>=36.0, <39.0 +cryptography>=38.0.3 numpy>=1.0, <2.0 pre-commit>=2.0, <3.0 pytest-cov>=3.0, <4.0 diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index fb7dbf2b..e11303cb 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -33,7 +33,7 @@ cffi==1.15.1 # pynacl cfgv==3.3.1 # via pre-commit -charset-normalizer==2.1.1 +charset-normalizer==3.0.1 # via # -r requirements.txt # requests @@ -50,9 +50,9 @@ coreschema==0.0.4 # -r requirements.txt # coreapi # drf-yasg -coverage[toml]==7.0.0 +coverage[toml]==7.0.5 # via pytest-cov -cryptography==38.0.1 +cryptography==39.0.0 # via # -r requirements-dev.in # -r requirements.txt @@ -104,9 +104,9 @@ environs==9.5.0 # -r requirements.txt # scos-actions # scos-tekrsa -exceptiongroup==1.0.4 +exceptiongroup==1.1.0 # via pytest -filelock==3.8.2 +filelock==3.9.0 # via # -r requirements.txt # ray @@ -123,7 +123,7 @@ grpcio==1.51.1 # ray gunicorn==20.1.0 # via -r requirements.txt -identify==2.5.11 +identify==2.5.15 # via pre-commit idna==3.4 # via @@ -133,7 +133,7 @@ inflection==0.5.1 # via # -r requirements.txt # drf-yasg -iniconfig==1.1.1 +iniconfig==2.0.0 # via pytest its-preselector @ git+https://github.com/NTIA/Preselector@3.0.0 # via @@ -154,7 +154,7 @@ jsonschema==3.2.0 # -r requirements.txt # docker-compose # ray -markupsafe==2.1.1 +markupsafe==2.1.2 # via # -r requirements.txt # jinja2 @@ -172,7 +172,7 @@ numexpr==2.8.4 # via # -r requirements.txt # scos-actions -numpy==1.24.0 +numpy==1.24.1 # via # -r requirements-dev.in # -r requirements.txt @@ -182,11 +182,7 @@ numpy==1.24.0 # scos-actions # sigmf # tekrsa-api-wrap -oauthlib==3.2.2 - # via - # -r requirements.txt - # requests-oauthlib -packaging==22.0 +packaging==23.0 # via # -r requirements.txt # docker @@ -194,11 +190,11 @@ packaging==22.0 # marshmallow # pytest # tox -paramiko==2.12.0 +paramiko==3.0.0 # via # -r requirements.txt # docker -platformdirs==2.6.0 +platformdirs==2.6.2 # via # -r requirements.txt # virtualenv @@ -206,7 +202,7 @@ pluggy==1.0.0 # via # pytest # tox -pre-commit==2.20.0 +pre-commit==2.21.0 # via -r requirements-dev.in protobuf==4.21.12 # via @@ -230,11 +226,11 @@ pynacl==1.5.0 # via # -r requirements.txt # paramiko -pyrsistent==0.19.2 +pyrsistent==0.19.3 # via # -r requirements.txt # jsonschema -pytest==7.2.0 +pytest==7.2.1 # via # pytest-cov # pytest-django @@ -246,12 +242,12 @@ python-dateutil==2.8.2 # via # -r requirements.txt # scos-actions -python-dotenv==0.21.0 +python-dotenv==0.21.1 # via # -r requirements.txt # docker-compose # environs -pytz==2022.7 +pytz==2022.7.1 # via # -r requirements.txt # django @@ -267,7 +263,7 @@ ray==2.2.0 # via # -r requirements.txt # scos-actions -requests==2.28.1 +requests==2.28.2 # via # -r requirements.txt # coreapi @@ -276,11 +272,8 @@ requests==2.28.1 # its-preselector # ray # requests-mock - # requests-oauthlib requests-mock==1.10.0 # via -r requirements.txt -requests-oauthlib==1.3.1 - # via -r requirements.txt ruamel-yaml==0.17.21 # via # -r requirements.txt @@ -288,9 +281,9 @@ ruamel-yaml==0.17.21 # scos-actions ruamel-yaml-clib==0.2.7 # via - # -r requirements.txt - # ruamel-yaml -scipy==1.9.3 + # -r requirements.txt + # ruamel-yaml +scipy==1.10.0 # via # -r requirements.txt # scos-actions @@ -310,7 +303,6 @@ six==1.16.0 # django-session-timeout # dockerpty # jsonschema - # paramiko # python-dateutil # requests-mock # sigmf @@ -328,8 +320,6 @@ texttable==1.6.7 # via # -r requirements.txt # docker-compose -toml==0.10.2 - # via pre-commit tomli==2.0.1 # via # coverage @@ -342,7 +332,7 @@ uritemplate==4.1.1 # -r requirements.txt # coreapi # drf-yasg -urllib3==1.26.13 +urllib3==1.26.14 # via # -r requirements.txt # docker diff --git a/src/requirements.in b/src/requirements.in index ca5e874a..c03ebab4 100644 --- a/src/requirements.in +++ b/src/requirements.in @@ -6,11 +6,9 @@ drf-yasg>=1.0, <2.0 environs>=9.0, <10.0 gunicorn>=20.0, <21.0 jsonfield>=3.0, <4.0 -oauthlib>=3.2.1, <4.0 psycopg2-binary>=2.0, <3.0 pyjwt>=2.4.0, <3.0 requests-mock>=1.0, <2.0 -requests_oauthlib>=1.0, <2.0 scos_actions @ git+https://github.com/NTIA/scos-actions@6.0.1 #scos_usrp @ git+https://github.com/NTIA/scos-usrp@3.0.0 scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@3.0.0 diff --git a/src/requirements.txt b/src/requirements.txt index c6bef082..812e4421 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -20,7 +20,7 @@ cffi==1.15.1 # via # cryptography # pynacl -charset-normalizer==2.1.1 +charset-normalizer==3.0.1 # via requests click==8.1.3 # via ray @@ -30,7 +30,7 @@ coreschema==0.0.4 # via # coreapi # drf-yasg -cryptography==38.0.1 +cryptography==39.0.0 # via paramiko defusedxml==0.7.1 # via its-preselector @@ -67,7 +67,7 @@ environs==9.5.0 # -r requirements.in # scos-actions # scos-tekrsa -filelock==3.8.2 +filelock==3.9.0 # via # ray # virtualenv @@ -95,7 +95,7 @@ jsonschema==3.2.0 # via # docker-compose # ray -markupsafe==2.1.1 +markupsafe==2.1.2 # via jinja2 marshmallow==3.19.0 # via environs @@ -103,7 +103,7 @@ msgpack==1.0.4 # via ray numexpr==2.8.4 # via scos-actions -numpy==1.24.0 +numpy==1.24.1 # via # numexpr # ray @@ -111,18 +111,14 @@ numpy==1.24.0 # scos-actions # sigmf # tekrsa-api-wrap -oauthlib==3.2.2 - # via - # -r requirements.in - # requests-oauthlib -packaging==22.0 +packaging==23.0 # via # docker # drf-yasg # marshmallow -paramiko==2.12.0 +paramiko==3.0.0 # via docker -platformdirs==2.6.0 +platformdirs==2.6.2 # via virtualenv protobuf==4.21.12 # via ray @@ -136,15 +132,15 @@ pyjwt==2.6.0 # via -r requirements.in pynacl==1.5.0 # via paramiko -pyrsistent==0.19.2 +pyrsistent==0.19.3 # via jsonschema python-dateutil==2.8.2 # via scos-actions -python-dotenv==0.21.0 +python-dotenv==0.21.1 # via # docker-compose # environs -pytz==2022.7 +pytz==2022.7.1 # via # django # djangorestframework @@ -155,7 +151,7 @@ pyyaml==5.4.1 # ray ray==2.2.0 # via scos-actions -requests==2.28.1 +requests==2.28.2 # via # coreapi # docker @@ -163,19 +159,15 @@ requests==2.28.1 # its-preselector # ray # requests-mock - # requests-oauthlib requests-mock==1.10.0 # via -r requirements.in -requests-oauthlib==1.3.1 - # via -r requirements.in ruamel-yaml==0.17.21 # via # drf-yasg # scos-actions ruamel-yaml-clib==0.2.7 - # via - # ruamel-yaml -scipy==1.9.3 + # via ruamel-yaml +scipy==1.10.0 # via scos-actions scos_actions @ git+https://github.com/NTIA/scos-actions@6.0.1 # via @@ -190,7 +182,6 @@ six==1.16.0 # django-session-timeout # dockerpty # jsonschema - # paramiko # python-dateutil # requests-mock # sigmf @@ -205,7 +196,7 @@ uritemplate==4.1.1 # via # coreapi # drf-yasg -urllib3==1.26.13 +urllib3==1.26.14 # via # docker # requests From ba037ab28b1e784b4ada8ca28c0b0c453ef7b5de Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Mon, 23 Jan 2023 13:13:14 -0700 Subject: [PATCH 007/255] remove pyjwt from requirements --- src/requirements-dev.txt | 2 -- src/requirements.in | 1 - src/requirements.txt | 2 -- 3 files changed, 5 deletions(-) diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index e11303cb..18724358 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -220,8 +220,6 @@ pycparser==2.21 # via # -r requirements.txt # cffi -pyjwt==2.6.0 - # via -r requirements.txt pynacl==1.5.0 # via # -r requirements.txt diff --git a/src/requirements.in b/src/requirements.in index c03ebab4..46d75fba 100644 --- a/src/requirements.in +++ b/src/requirements.in @@ -7,7 +7,6 @@ environs>=9.0, <10.0 gunicorn>=20.0, <21.0 jsonfield>=3.0, <4.0 psycopg2-binary>=2.0, <3.0 -pyjwt>=2.4.0, <3.0 requests-mock>=1.0, <2.0 scos_actions @ git+https://github.com/NTIA/scos-actions@6.0.1 #scos_usrp @ git+https://github.com/NTIA/scos-usrp@3.0.0 diff --git a/src/requirements.txt b/src/requirements.txt index 812e4421..8e3f6b58 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -128,8 +128,6 @@ psycopg2-binary==2.9.5 # via -r requirements.in pycparser==2.21 # via cffi -pyjwt==2.6.0 - # via -r requirements.in pynacl==1.5.0 # via paramiko pyrsistent==0.19.3 From a7d529155941fc36183c1565e495a35706578be3 Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Mon, 23 Jan 2023 13:28:57 -0700 Subject: [PATCH 008/255] update readme --- README.md | 102 ++++++++++------------------------------ configs/certs/README.md | 2 +- 2 files changed, 27 insertions(+), 77 deletions(-) diff --git a/README.md b/README.md index a8550edc..217e3d7f 100644 --- a/README.md +++ b/README.md @@ -371,38 +371,31 @@ authenticating when using a callback URL. ### Sensor Authentication And Permissions -The sensor can be configured to authenticate using OAuth JWT access tokens from an -external authorization server or using Django Rest Framework Token Authentication. +The sensor can be configured to authenticate using mutual TLS with client certificates +or using Django Rest Framework Token Authentication. #### Django Rest Framework Token Authentication This is the default authentication method. To enable Django Rest Framework Authentication, make sure `AUTHENTICATION` is set to `TOKEN` in the environment file (this will be enabled if `AUTHENTICATION` set to anything other -than `JWT`). +than `CERT`). A token is automatically created for each user. Django Rest Framework Token Authentication will check that the token in the Authorization header ("Token " + token) matches a user's token. -#### OAuth2 JWT Authentication +#### Certificate Authentication -To enable OAuth 2 JWT Authentication, set `AUTHENTICATION` to `JWT` in the environment -file. To authenticate, the client will need to send a JWT access token in the -authorization header (using "Bearer " + access token). The token signature will be -verified using the public key from the `PATH_TO_JWT_PUBLIC_KEY` setting. The expiration -time will be checked. Only users who have an authority matching the `REQUIRED_ROLE` -setting will be authorized. - -The token is expected to come from an OAuth2 authorization server. For more -information, see . +To enable Certificate Authentication, set `AUTHENTICATION` to `CERT` in the environment +file. To authenticate, the client will need to send a trusted client certificate. The +Common Name must match the username of a user in the database. #### Certificates Use this section to create self-signed certificates with customized organizational and host information. This section includes instructions for creating a self-signed -root CA, SSL server certificates for the sensor, optional client certificates, and test -JWT public/private key pair. +root CA, SSL server certificates for the sensor, and optional client certificates. As described below, a self-signed CA can be created for testing. **For production, make sure to use certificates from a trusted CA.** @@ -501,36 +494,11 @@ openssl pkcs12 -export -out client.pfx -inkey client.key -in client.pem -certfil Import client.pfx into web browser for use with the browsable API or use the client.pem or client.pfx when communicating with the API programmatically. -##### Generating JWT Public/Private Key - -The JWT public key must correspond to the private key of the JWT issuer (OAuth -authorization server). For manual testing, the instructions below could be used to -create a public/private key pair for creating JWTs without an authorization -server. - -###### Step 1: Create public/private key pair - -```bash -openssl genrsa -out jwt.pem 4096 -``` - -###### Step 2: Extract Public Key - -```bash -openssl rsa -in jwt.pem -outform PEM -pubout -out jwt_public_key.pem -``` - -###### Step 3: Extract Private Key - -```bash -openssl pkey -inform PEM -outform PEM -in jwt.pem -out jwt_private_key.pem -``` - ###### Configure scos-sensor The Nginx web server can be set to require client certificates (mutual TLS). This can -optionally be enabled. To require client certificates, uncomment -`ssl_verify_client on;` in the [Nginx configuration file](nginx/conf.template). If you +optionally be enabled. To require client certificates, make sure `ssl_verify_client` is +set to `on` in the [Nginx configuration file](nginx/conf.template). If you use OCSP, also uncomment `ssl_ocsp on;`. Additional configuration may be needed for Nginx to check certificate revocation lists (CRL). @@ -542,16 +510,10 @@ environment file) to the path of the sensor01_combined.pem relative to configs/c mutual TLS, also copy the CA certificate to the same directory. Then, set `SSL_CA_PATH` to the path of the CA certificate relative to `configs/certs`. -If you are using JWT authentication, set `PATH_TO_JWT_PUBLIC_KEY` to the path of the -JWT public key relative to configs/certs. This public key file should correspond to the -private key used to sign the JWT. Alternatively, the JWT private key -created above could be used to manually sign a JWT token for testing if -`PATH_TO_JWT_PUBLIC_KEY` is set to the JWT public key created above. - If you are using client certificates, use client.pfx to connect to the browsable API by importing this certificate into your browser. -For callback functionality with an OAuth authorized callback URL, set +For callback functionality with a server that uses certificate authentication, set `PATH_TO_CLIENT_CERT` and `PATH_TO_VERIFY_CERT`, both relative to configs/certs. Depending on the configuration of the callback URL server and the authorization server, the sensor server certificate could be used as a client certificate by setting @@ -562,18 +524,15 @@ as used for `SSL_CA_PATH` (scostestca.pem). #### Permissions and Users -The API requires the user to either have an authority in the JWT token matching the the -`REQUIRED_ROLE` setting or that the user be a superuser. New users created using the +The API requires the user to be a superuser. New users created using the API initially do not have superuser access. However, an admin can mark a user as a -superuser in the Sensor Configuration Portal. When using JWT tokens, the user does not -have to be pre-created using the sensor's API. The API will accept any user using a -JWT token if they have an authority matching the required role setting. +superuser in the Sensor Configuration Portal. ### Callback URL Authentication -OAuth and Token authentication are supported for authenticating against the server -pointed to by the callback URL. Callback SSL verification can be enabled -or disabled using `CALLBACK_SSL_VERIFICATION` in the environment file. +Certificate and token authentication are supported for authenticating against the +server pointed to by the callback URL. Callback SSL verification can be enabled or +disabled using `CALLBACK_SSL_VERIFICATION` in the environment file. #### Token @@ -590,29 +549,20 @@ verify the callback URL server SSL certificate. If this is unset and https://requests.readthedocs.io/en/master/user/advanced/#ca-certificates) will be used. -#### OAuth +#### Certificate -The OAuth 2 password flow is supported for callback URL authentication. The following -settings in the environment file are used to configure the OAuth 2 password flow -authentication. +Certificate authetnication (mutual TLS) is supported for callback URL authentication. +The following settings in the environment file are used to configure certificate +authentication for the callback URL. -- `CALLBACK_AUTHENTICATION` - set to `OAUTH`. -- `CLIENT_ID` - client ID used to authorize the client (the sensor) against the - authorization server. -- `CLIENT_SECRET` - client secret used to authorize the client (the sensor) against the - authorization server. -- `OAUTH_TOKEN_URL` - URL to get the access token. +- `CALLBACK_AUTHENTICATION` - set to `CERT`. - `PATH_TO_CLIENT_CERT` - client certificate used to authenticate against the - authorization server. -- `PATH_TO_VERIFY_CERT` - CA certificate to verify the authorization server and - callback URL server SSL certificate. If this is unset and `CALLBACK_SSL_VERIFICATION` - is set to true, [standard trusted CAs]( + callback URL server. +- `PATH_TO_VERIFY_CERT` - CA certificate to verify the callback URL server SSL + certificate. If this is unset and `CALLBACK_SSL_VERIFICATION` + is set to true, [standard trusted CAs]( https://requests.readthedocs.io/en/master/user/advanced/#ca-certificates) will be - used. - -In src/sensor/settings.py, the OAuth `USER_NAME` and `PASSWORD` are set to be the same -as `CLIENT_ID` and `CLIENT_SECRET`. This may need to change depending on your -authorization server. + used. ## Actions and Hardware Support diff --git a/configs/certs/README.md b/configs/certs/README.md index 2b9c6951..85b06b8f 100644 --- a/configs/certs/README.md +++ b/configs/certs/README.md @@ -1,3 +1,3 @@ # Certs -Add SSL certs and JWT public key here. +Add SSL certs here. From 8b9d2532564e0a00455fbc70b68b79e9194d86ae Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Tue, 24 Jan 2023 15:28:24 -0700 Subject: [PATCH 009/255] remove jwt/oauth, change permission to IsSuperuser --- docker-compose.yml | 2 -- env.template | 6 ++-- src/authentication/oauth.py | 54 ------------------------------- src/authentication/permissions.py | 4 +-- src/scheduler/scheduler.py | 1 - src/sensor/migration_settings.py | 9 ++---- src/sensor/runtime_settings.py | 8 +---- src/sensor/settings.py | 8 +---- 8 files changed, 8 insertions(+), 84 deletions(-) delete mode 100644 src/authentication/oauth.py diff --git a/docker-compose.yml b/docker-compose.yml index 0f0ff682..7b061cc3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -51,9 +51,7 @@ services: - MAX_DISK_USAGE - MOCK_SIGAN - MOCK_SIGAN_RANDOM - - OAUTH_TOKEN_URL - PATH_TO_CLIENT_CERT - - PATH_TO_JWT_PUBLIC_KEY - PATH_TO_VERIFY_CERT - POSTGRES_PASSWORD - SECRET_KEY diff --git a/env.template b/env.template index 3a34ca90..7550cc82 100644 --- a/env.template +++ b/env.template @@ -58,19 +58,17 @@ MANAGER_IP="$(hostname -I | cut -d' ' -f1)" BASE_IMAGE=ghcr.io/ntia/scos-tekrsa/tekrsa_usb:0.2.1 # Default callback api/results -# Set to OAUTH if using OAuth Password Flow Authentication, callback url needs to be api/v2/results +# Set to CERT for certificate authentication CALLBACK_AUTHENTICATION=TOKEN CLIENT_ID=sensor01.sms.internal CLIENT_SECRET=sensor-secret -OAUTH_TOKEN_URL=https://scosmgrqa01.sms.internal:443/authserver/oauth/token # Sensor certificate with private key used as client cert PATH_TO_CLIENT_CERT=sensor01.pem # Trusted Certificate Authority certificate to verify authserver and callback URL server certificate PATH_TO_VERIFY_CERT=scos_test_ca.crt # Path relative to configs/certs -PATH_TO_JWT_PUBLIC_KEY=jwt_pubkey.pem -# set to JWT to enable JWT authentication +# set to CERT to enable certificate authentication AUTHENTICATION=CERT ADMIN_NAME=Admin diff --git a/src/authentication/oauth.py b/src/authentication/oauth.py deleted file mode 100644 index 23f47f8f..00000000 --- a/src/authentication/oauth.py +++ /dev/null @@ -1,54 +0,0 @@ -import logging - -from django.conf import settings -from oauthlib.oauth2 import LegacyApplicationClient -from requests_oauthlib import OAuth2Session - -logger = logging.getLogger(__name__) - - -def get_oauth_token(): - """Returns OAuth access token.""" - try: - logger.debug(settings.CLIENT_ID) - logger.debug(settings.CLIENT_SECRET) - logger.debug(settings.USER_NAME) - logger.debug(settings.PASSWORD) - - logger.debug(settings.OAUTH_TOKEN_URL) - logger.debug(settings.PATH_TO_CLIENT_CERT) - logger.debug(settings.PATH_TO_VERIFY_CERT) - verify_ssl = settings.CALLBACK_SSL_VERIFICATION - if settings.CALLBACK_SSL_VERIFICATION: - if settings.PATH_TO_VERIFY_CERT != "": - verify_ssl = settings.PATH_TO_VERIFY_CERT - - logger.debug(verify_ssl) - oauth = OAuth2Session( - client=LegacyApplicationClient(client_id=settings.CLIENT_ID) - ) - oauth.cert = settings.PATH_TO_CLIENT_CERT - token = oauth.fetch_token( - token_url=settings.OAUTH_TOKEN_URL, - username=settings.USER_NAME, - password=settings.PASSWORD, - client_id=settings.CLIENT_ID, - client_secret=settings.CLIENT_SECRET, - verify=verify_ssl, - ) - oauth.close() - logger.debug("Response from oauth.fetch_token: " + str(token)) - return token - except Exception: - raise - - -def get_oauth_client(): - """Returns Authorized OAuth Client (with authentication header token).""" - try: - token = get_oauth_token() - client = OAuth2Session(settings.CLIENT_ID, token=token) - client.cert = settings.PATH_TO_CLIENT_CERT - return client - except Exception: - raise diff --git a/src/authentication/permissions.py b/src/authentication/permissions.py index 4a2beafa..42a3f14c 100644 --- a/src/authentication/permissions.py +++ b/src/authentication/permissions.py @@ -1,8 +1,8 @@ from rest_framework import permissions -class RequiredJWTRolePermissionOrIsSuperuser(permissions.BasePermission): - message = "User missing required role" +class IsSuperuser(permissions.BasePermission): + message = "User is not superuser" def has_permission(self, request, view): if request.user.is_superuser: diff --git a/src/scheduler/scheduler.py b/src/scheduler/scheduler.py index 6a72f935..346845f7 100644 --- a/src/scheduler/scheduler.py +++ b/src/scheduler/scheduler.py @@ -9,7 +9,6 @@ import requests from django.utils import timezone -from authentication import oauth from schedule.models import ScheduleEntry from sensor import settings from tasks.consts import MAX_DETAIL_LEN diff --git a/src/sensor/migration_settings.py b/src/sensor/migration_settings.py index 954b044f..d967b3aa 100644 --- a/src/sensor/migration_settings.py +++ b/src/sensor/migration_settings.py @@ -221,7 +221,7 @@ ), "DEFAULT_PERMISSION_CLASSES": ( "rest_framework.permissions.IsAuthenticated", - "authentication.permissions.RequiredJWTRolePermissionOrIsSuperuser", + "authentication.permissions.IsSuperuser", ), "DEFAULT_RENDERER_CLASSES": ( "rest_framework.renderers.JSONRenderer", @@ -389,12 +389,7 @@ PATH_TO_VERIFY_CERT = env("PATH_TO_VERIFY_CERT", default="") if PATH_TO_VERIFY_CERT != "": PATH_TO_VERIFY_CERT = path.join(CERTS_DIR, PATH_TO_VERIFY_CERT) -# Public key to verify JWT token -PATH_TO_JWT_PUBLIC_KEY = env.str("PATH_TO_JWT_PUBLIC_KEY", default="") -if PATH_TO_JWT_PUBLIC_KEY != "": - PATH_TO_JWT_PUBLIC_KEY = path.join(CERTS_DIR, PATH_TO_JWT_PUBLIC_KEY) -# Required role from JWT token to access API -REQUIRED_ROLE = "ROLE_MANAGER" + PRESELECTOR_CONFIG = env.str( "PRESELECTOR_CONFIG", default=path.join(CONFIG_DIR, "preselector_config.json") diff --git a/src/sensor/runtime_settings.py b/src/sensor/runtime_settings.py index 09df46df..dd5fe7a5 100644 --- a/src/sensor/runtime_settings.py +++ b/src/sensor/runtime_settings.py @@ -221,7 +221,7 @@ ), "DEFAULT_PERMISSION_CLASSES": ( "rest_framework.permissions.IsAuthenticated", - "authentication.permissions.RequiredJWTRolePermissionOrIsSuperuser", + "authentication.permissions.IsSuperuser", ), "DEFAULT_RENDERER_CLASSES": ( "rest_framework.renderers.JSONRenderer", @@ -389,12 +389,6 @@ PATH_TO_VERIFY_CERT = env("PATH_TO_VERIFY_CERT", default="") if PATH_TO_VERIFY_CERT != "": PATH_TO_VERIFY_CERT = path.join(CERTS_DIR, PATH_TO_VERIFY_CERT) -# Public key to verify JWT token -PATH_TO_JWT_PUBLIC_KEY = env.str("PATH_TO_JWT_PUBLIC_KEY", default="") -if PATH_TO_JWT_PUBLIC_KEY != "": - PATH_TO_JWT_PUBLIC_KEY = path.join(CERTS_DIR, PATH_TO_JWT_PUBLIC_KEY) -# Required role from JWT token to access API -REQUIRED_ROLE = "ROLE_MANAGER" PRESELECTOR_CONFIG = env.str( "PRESELECTOR_CONFIG", default=path.join(CONFIG_DIR, "preselector_config.json") diff --git a/src/sensor/settings.py b/src/sensor/settings.py index ee080e25..f0969beb 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -222,7 +222,7 @@ ), "DEFAULT_PERMISSION_CLASSES": ( "rest_framework.permissions.IsAuthenticated", - "authentication.permissions.RequiredJWTRolePermissionOrIsSuperuser", + "authentication.permissions.IsSuperuser", ), "DEFAULT_RENDERER_CLASSES": ( "rest_framework.renderers.JSONRenderer", @@ -390,12 +390,6 @@ PATH_TO_VERIFY_CERT = env("PATH_TO_VERIFY_CERT", default="") if PATH_TO_VERIFY_CERT != "": PATH_TO_VERIFY_CERT = path.join(CERTS_DIR, PATH_TO_VERIFY_CERT) -# Public key to verify JWT token -PATH_TO_JWT_PUBLIC_KEY = env.str("PATH_TO_JWT_PUBLIC_KEY", default="") -if PATH_TO_JWT_PUBLIC_KEY != "": - PATH_TO_JWT_PUBLIC_KEY = path.join(CERTS_DIR, PATH_TO_JWT_PUBLIC_KEY) -# Required role from JWT token to access API -REQUIRED_ROLE = "ROLE_MANAGER" PRESELECTOR_CONFIG = env.str( "PRESELECTOR_CONFIG", default=path.join(CONFIG_DIR, "preselector_config.json") From dfc673d315b38c2f1020ea4d3658f033c3c5033e Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Tue, 24 Jan 2023 15:59:24 -0700 Subject: [PATCH 010/255] fix tests for cert auth, remove oauth variables from settings, fix scheduler cert auth callback --- src/authentication/auth.py | 2 - .../tests/jwt_content_example.json | 35 -- src/authentication/tests/test_cert_auth.py | 342 ++++++++++++++ src/authentication/tests/test_jwt_auth.py | 432 ------------------ src/authentication/tests/utils.py | 21 - src/conftest.py | 30 +- src/scheduler/scheduler.py | 2 +- src/scheduler/tests/test_scheduler.py | 41 +- src/sensor/migration_settings.py | 5 - src/sensor/runtime_settings.py | 5 - src/sensor/settings.py | 5 - src/sensor/tests/certificate_auth_client.py | 52 +++ src/sensor/tests/utils.py | 10 + src/tasks/tests/test_list_view.py | 2 +- src/tox.ini | 9 +- 15 files changed, 434 insertions(+), 559 deletions(-) delete mode 100644 src/authentication/tests/jwt_content_example.json create mode 100644 src/authentication/tests/test_cert_auth.py delete mode 100644 src/authentication/tests/test_jwt_auth.py delete mode 100644 src/authentication/tests/utils.py create mode 100644 src/sensor/tests/certificate_auth_client.py diff --git a/src/authentication/auth.py b/src/authentication/auth.py index e51c88ad..1cdcd0b7 100644 --- a/src/authentication/auth.py +++ b/src/authentication/auth.py @@ -1,10 +1,8 @@ import logging import re -import jwt from django.conf import settings from django.contrib.auth import get_user_model from rest_framework import authentication, exceptions -from rest_framework.authentication import get_authorization_header logger = logging.getLogger(__name__) diff --git a/src/authentication/tests/jwt_content_example.json b/src/authentication/tests/jwt_content_example.json deleted file mode 100644 index 5e5f2b4f..00000000 --- a/src/authentication/tests/jwt_content_example.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "user_name": "sensor01", - "scope": [ - "read", - "write" - ], - "exp": 1601494344, - "userDetails": { - "id": null, - "uid": "", - "altSecurityIdenties": null, - "email": "sensor01", - "firstname": "sensor01", - "lastname": "", - "cn": "sensor01", - "lastlogin": 1583360759668, - "enabled": true, - "authorities": [ - { - "authority": "ROLE_MANAGER" - } - ], - "password": null, - "username": "sensor01", - "dn": "cn=sensor01,ou=OU1,ou=OU2,ou=OU3,dc=DC1,dc=DC2", - "accountNonLocked": true, - "accountnullxpired": true, - "credentialsnullxpired": true - }, - "authorities": [ - "ROLE_MANAGER" - ], - "jti": "e4271916-bfe0-4028-b372-dc05c4882c88", - "client_id": "sensor01" -} diff --git a/src/authentication/tests/test_cert_auth.py b/src/authentication/tests/test_cert_auth.py new file mode 100644 index 00000000..e8b489ec --- /dev/null +++ b/src/authentication/tests/test_cert_auth.py @@ -0,0 +1,342 @@ +import base64 +import json +import os +import secrets +from datetime import datetime, timedelta +from tempfile import NamedTemporaryFile + +import pytest +from rest_framework.reverse import reverse +from rest_framework.test import RequestsClient + +from authentication.auth import certificate_authentication_enabled +from authentication.models import User +from sensor import V1 +from sensor.tests.utils import get_requests_ssl_dn_header + +pytestmark = pytest.mark.skipif( + not certificate_authentication_enabled, + reason="Certificate authentication is not enabled!", +) + + + + +one_min = timedelta(minutes=1) +one_day = timedelta(days=1) + + + + + +@pytest.mark.django_db +def test_no_client_cert_unauthorized(live_server): + client = RequestsClient() + response = client.get(f"{live_server.url}") + assert response.status_code == 403 + + + +@pytest.mark.django_db +def test_client_cert_accepted(live_server, admin_user): + client = RequestsClient() + response = client.get(f"{live_server.url}", headers=get_requests_ssl_dn_header("admin")) + assert response.status_code == 200 + + +def test_bad_client_cert_forbidden(live_server, admin_user): + client = RequestsClient() + response = client.get(f"{live_server.url}", headers=get_requests_ssl_dn_header("user")) + assert response.status_code == 403 + assert "No matching username found!" in response.json()["detail"] + + +# @pytest.mark.django_db +# def test_certificate_expired_1_day_forbidden(live_server): +# current_datetime = datetime.now() +# token_payload = get_token_payload(exp=(current_datetime - one_day).timestamp()) +# encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") +# client = RequestsClient() +# response = client.get(f"{live_server.url}", headers=get_headers(encoded)) +# assert response.status_code == 403 +# assert response.json()["detail"] == "Token is expired!" + + +# @pytest.mark.django_db +# def test_bad_private_key_forbidden(live_server): +# token_payload = get_token_payload() +# encoded = jwt.encode( +# token_payload, str(BAD_PRIVATE_KEY.decode("utf-8")), algorithm="RS256" +# ) +# client = RequestsClient() +# response = client.get(f"{live_server.url}", headers=get_headers(encoded)) +# assert response.status_code == 403 +# assert response.json()["detail"] == "Unable to verify token!" + + +# @pytest.mark.django_db +# def test_bad_public_key_forbidden(settings, live_server): +# with NamedTemporaryFile() as jwt_public_key_file: +# jwt_public_key_file.write(BAD_PUBLIC_KEY) +# jwt_public_key_file.flush() +# settings.PATH_TO_JWT_PUBLIC_KEY = jwt_public_key_file.name +# token_payload = get_token_payload() +# encoded = jwt.encode( +# token_payload, str(jwt_keys.private_key), algorithm="RS256" +# ) +# client = RequestsClient() +# response = client.get(f"{live_server.url}", headers=get_headers(encoded)) +# assert response.status_code == 403 +# assert response.json()["detail"] == "Unable to verify token!" + + +# @pytest.mark.django_db +# def test_certificate_expired_1_min_forbidden(live_server): +# current_datetime = datetime.now() +# token_payload = get_token_payload(exp=(current_datetime - one_min).timestamp()) +# encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") +# client = RequestsClient() +# response = client.get(f"{live_server.url}", headers=get_headers(encoded)) +# assert response.status_code == 403 +# assert response.json()["detail"] == "Token is expired!" + + +# @pytest.mark.django_db +# def test_certificate_expires_in_1_min_accepted(live_server): +# current_datetime = datetime.now() +# token_payload = get_token_payload(exp=(current_datetime + one_min).timestamp()) +# encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") +# client = RequestsClient() +# response = client.get(f"{live_server.url}", headers=get_headers(encoded)) +# assert response.status_code == 200 + + +# @pytest.mark.django_db +# def test_urls_unauthorized(live_server): +# token_payload = get_token_payload(authorities=["ROLE_USER"]) +# encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") +# client = RequestsClient() +# headers = get_headers(encoded) + +# capabilities = reverse("capabilities", kwargs=V1) +# response = client.get(f"{live_server.url}{capabilities}", headers=headers) +# assert response.status_code == 403 + +# schedule_list = reverse("schedule-list", kwargs=V1) +# response = client.get(f"{live_server.url}{schedule_list}", headers=headers) +# assert response.status_code == 403 + +# status = reverse("status", kwargs=V1) +# response = client.get(f"{live_server.url}{status}", headers=headers) +# assert response.status_code == 403 + +# task_root = reverse("task-root", kwargs=V1) +# response = client.get(f"{live_server.url}{task_root}", headers=headers) +# assert response.status_code == 403 + +# task_results_overview = reverse("task-results-overview", kwargs=V1) +# response = client.get(f"{live_server.url}{task_results_overview}", headers=headers) +# assert response.status_code == 403 + +# upcoming_tasks = reverse("upcoming-tasks", kwargs=V1) +# response = client.get(f"{live_server.url}{upcoming_tasks}", headers=headers) +# assert response.status_code == 403 + +# user_list = reverse("user-list", kwargs=V1) +# response = client.get(f"{live_server.url}{user_list}", headers=headers) +# assert response.status_code == 403 + + +# @pytest.mark.django_db +# def test_urls_authorized(live_server): +# token_payload = get_token_payload(authorities=["ROLE_MANAGER"]) +# encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") +# client = RequestsClient() +# headers = get_headers(encoded) + +# capabilities = reverse("capabilities", kwargs=V1) +# response = client.get(f"{live_server.url}{capabilities}", headers=headers) +# assert response.status_code == 200 + +# schedule_list = reverse("schedule-list", kwargs=V1) +# response = client.get(f"{live_server.url}{schedule_list}", headers=headers) +# assert response.status_code == 200 + +# status = reverse("status", kwargs=V1) +# response = client.get(f"{live_server.url}{status}", headers=headers) +# assert response.status_code == 200 + +# task_root = reverse("task-root", kwargs=V1) +# response = client.get(f"{live_server.url}{task_root}", headers=headers) +# assert response.status_code == 200 + +# task_results_overview = reverse("task-results-overview", kwargs=V1) +# response = client.get(f"{live_server.url}{task_results_overview}", headers=headers) +# assert response.status_code == 200 + +# upcoming_tasks = reverse("upcoming-tasks", kwargs=V1) +# response = client.get(f"{live_server.url}{upcoming_tasks}", headers=headers) +# assert response.status_code == 200 + +# user_list = reverse("user-list", kwargs=V1) +# response = client.get(f"{live_server.url}{user_list}", headers=headers) +# assert response.status_code == 200 + + +# @pytest.mark.django_db +# def test_user_cannot_view_user_detail(live_server): +# sensor01_token_payload = get_token_payload(authorities=["ROLE_MANAGER"]) +# encoded = jwt.encode( +# sensor01_token_payload, str(jwt_keys.private_key), algorithm="RS256" +# ) +# client = RequestsClient() +# response = client.get(f"{live_server.url}", headers=get_headers(encoded)) +# assert response.status_code == 200 + +# sensor02_token_payload = get_token_payload(authorities=["ROLE_USER"]) +# sensor02_token_payload["user_name"] = "sensor02" +# encoded = jwt.encode( +# sensor02_token_payload, str(jwt_keys.private_key), algorithm="RS256" +# ) +# client = RequestsClient() + +# sensor01_user = User.objects.get(username=sensor01_token_payload["user_name"]) +# kws = {"pk": sensor01_user.pk} +# kws.update(V1) +# user_detail = reverse("user-detail", kwargs=kws) +# response = client.get( +# f"{live_server.url}{user_detail}", headers=get_headers(encoded) +# ) +# assert response.status_code == 403 + + +# @pytest.mark.django_db +# def test_user_cannot_view_user_detail_role_change(live_server): +# sensor01_token_payload = get_token_payload(authorities=["ROLE_MANAGER"]) +# encoded = jwt.encode( +# sensor01_token_payload, str(jwt_keys.private_key), algorithm="RS256" +# ) +# client = RequestsClient() +# response = client.get(f"{live_server.url}", headers=get_headers(encoded)) +# assert response.status_code == 200 + +# token_payload = get_token_payload(authorities=["ROLE_USER"]) +# encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") +# client = RequestsClient() + +# sensor01_user = User.objects.get(username=sensor01_token_payload["user_name"]) +# kws = {"pk": sensor01_user.pk} +# kws.update(V1) +# user_detail = reverse("user-detail", kwargs=kws) +# response = client.get( +# f"{live_server.url}{user_detail}", headers=get_headers(encoded) +# ) +# assert response.status_code == 403 + + +# @pytest.mark.django_db +# def test_admin_can_view_user_detail(live_server): +# token_payload = get_token_payload(authorities=["ROLE_MANAGER"]) +# encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") +# client = RequestsClient() +# headers = get_headers(encoded) +# response = client.get(f"{live_server.url}", headers=headers) +# assert response.status_code == 200 + +# sensor01_user = User.objects.get(username=token_payload["user_name"]) +# kws = {"pk": sensor01_user.pk} +# kws.update(V1) +# user_detail = reverse("user-detail", kwargs=kws) +# response = client.get(f"{live_server.url}{user_detail}", headers=headers) +# assert response.status_code == 200 + + +# @pytest.mark.django_db +# def test_admin_can_view_other_user_detail(live_server): +# sensor01_token_payload = get_token_payload(authorities=["ROLE_MANAGER"]) +# encoded = jwt.encode( +# sensor01_token_payload, str(jwt_keys.private_key), algorithm="RS256" +# ) +# client = RequestsClient() +# response = client.get(f"{live_server.url}", headers=get_headers(encoded)) +# assert response.status_code == 200 + +# sensor02_token_payload = get_token_payload(authorities=["ROLE_MANAGER"]) +# sensor02_token_payload["user_name"] = "sensor02" +# encoded = jwt.encode( +# sensor02_token_payload, str(jwt_keys.private_key), algorithm="RS256" +# ) +# client = RequestsClient() + +# sensor01_user = User.objects.get(username=sensor01_token_payload["user_name"]) +# kws = {"pk": sensor01_user.pk} +# kws.update(V1) +# user_detail = reverse("user-detail", kwargs=kws) +# response = client.get( +# f"{live_server.url}{user_detail}", headers=get_headers(encoded) +# ) +# assert response.status_code == 200 + + +# @pytest.mark.django_db +# def test_token_hidden(live_server): +# token_payload = get_token_payload(authorities=["ROLE_MANAGER"]) +# encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") +# client = RequestsClient() +# headers = get_headers(encoded) +# response = client.get(f"{live_server.url}", headers=headers) +# assert response.status_code == 200 + +# sensor01_user = User.objects.get(username=token_payload["user_name"]) +# kws = {"pk": sensor01_user.pk} +# kws.update(V1) +# user_detail = reverse("user-detail", kwargs=kws) +# client = RequestsClient() +# response = client.get(f"{live_server.url}{user_detail}", headers=headers) +# assert response.status_code == 200 +# assert ( +# response.json()["auth_token"] +# == "rest_framework.authentication.TokenAuthentication is not enabled" +# ) + + +# @pytest.mark.django_db +# def test_change_token_role_bad_signature(live_server): +# """Make sure token modified after it was signed is rejected""" +# token_payload = get_token_payload(authorities=["ROLE_USER"]) +# encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") +# first_period = encoded.find(".") +# second_period = encoded.find(".", first_period + 1) +# payload = encoded[first_period + 1 : second_period] +# payload_bytes = payload.encode("utf-8") +# # must be multiple of 4 for b64decode +# for i in range(len(payload_bytes) % 4): +# payload_bytes = payload_bytes + b"=" +# decoded = base64.b64decode(payload_bytes) +# payload_str = decoded.decode("utf-8") +# payload_data = json.loads(payload_str) +# payload_data["user_name"] = "sensor013" +# payload_data["authorities"] = ["ROLE_MANAGER"] +# payload_data["userDetails"]["authorities"] = [{"authority": "ROLE_MANAGER"}] +# payload_str = json.dumps(payload_data) +# modified_payload = base64.b64encode(payload_str.encode("utf-8")) +# modified_payload = modified_payload.decode("utf-8") +# # remove padding +# if modified_payload.endswith("="): +# last_padded_index = len(modified_payload) - 1 +# for i in range(len(modified_payload) - 1, -1, -1): +# if modified_payload[i] != "=": +# last_padded_index = i +# break +# modified_payload = modified_payload[: last_padded_index + 1] +# modified_token = ( +# encoded[:first_period] +# + "." +# + modified_payload +# + "." +# + encoded[second_period + 1 :] +# ) +# client = RequestsClient() +# response = client.get(f"{live_server.url}", headers=get_headers(modified_token)) +# assert response.status_code == 403 +# assert response.json()["detail"] == "Unable to verify token!" diff --git a/src/authentication/tests/test_jwt_auth.py b/src/authentication/tests/test_jwt_auth.py deleted file mode 100644 index 313e6f1d..00000000 --- a/src/authentication/tests/test_jwt_auth.py +++ /dev/null @@ -1,432 +0,0 @@ -import base64 -import json -import os -import secrets -from datetime import datetime, timedelta -from tempfile import NamedTemporaryFile - -import jwt -import pytest -from rest_framework.reverse import reverse -from rest_framework.test import RequestsClient - -from authentication.auth import oauth_jwt_authentication_enabled -from authentication.models import User -from authentication.tests.utils import get_test_public_private_key -from sensor import V1 - -pytestmark = pytest.mark.skipif( - not oauth_jwt_authentication_enabled, - reason="OAuth JWT authentication is not enabled!", -) - - -jwt_content_file = os.path.join( - os.path.dirname(os.path.abspath(__file__)), "jwt_content_example.json" -) -with open(jwt_content_file) as token_file: - TOKEN_CONTENT = json.load(token_file) - -BAD_PRIVATE_KEY, BAD_PUBLIC_KEY = get_test_public_private_key() - -one_min = timedelta(minutes=1) -one_day = timedelta(days=1) - - -def get_token_payload(authorities=["ROLE_MANAGER"], exp=None, client_id=None): - token_payload = TOKEN_CONTENT.copy() - current_datetime = datetime.now() - if not exp: - token_payload["exp"] = (current_datetime + one_day).timestamp() - else: - token_payload["exp"] = exp - token_payload["userDetails"]["lastlogin"] = (current_datetime - one_day).timestamp() - token_payload["userDetails"]["authorities"] = [] - for authority in authorities: - token_payload["userDetails"]["authorities"].append({"authority": authority}) - token_payload["userDetails"]["enabled"] = True - token_payload["authorities"] = authorities - if client_id: - token_payload["client_id"] = client_id - return token_payload - - -def get_headers(token): - return { - "Authorization": f"Bearer {token}", - "X-Ssl-Client-Dn": f"CN=Test,OU=Test,O=Test,L=Test,ST=Test,C=Test", - } - - -@pytest.mark.django_db -def test_no_token_unauthorized(live_server): - client = RequestsClient() - response = client.get(f"{live_server.url}") - assert response.status_code == 403 - - -@pytest.mark.django_db -def test_token_no_roles_unauthorized(live_server, jwt_keys): - token_payload = get_token_payload(authorities=[]) - encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") - client = RequestsClient() - response = client.get(f"{live_server.url}", headers=get_headers(encoded)) - assert response.status_code == 403 - assert response.json()["detail"] == "User missing required role" - - -@pytest.mark.django_db -def test_token_role_manager_accepted(live_server, jwt_keys): - token_payload = get_token_payload() - encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") - client = RequestsClient() - response = client.get(f"{live_server.url}", headers=get_headers(encoded)) - assert response.status_code == 200 - - -def test_bad_token_forbidden(live_server): - client = RequestsClient() - token = ( - secrets.token_urlsafe(28) - + "." - + secrets.token_urlsafe(679) - + "." - + secrets.token_urlsafe(525) - ) - response = client.get(f"{live_server.url}", headers=get_headers(token)) - print(f"headers: {response.request.headers}") - assert response.status_code == 403 - assert "Unable to decode token!" in response.json()["detail"] - - -@pytest.mark.django_db -def test_token_expired_1_day_forbidden(live_server, jwt_keys): - current_datetime = datetime.now() - token_payload = get_token_payload(exp=(current_datetime - one_day).timestamp()) - encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") - client = RequestsClient() - response = client.get(f"{live_server.url}", headers=get_headers(encoded)) - assert response.status_code == 403 - assert response.json()["detail"] == "Token is expired!" - - -@pytest.mark.django_db -def test_bad_private_key_forbidden(live_server): - token_payload = get_token_payload() - encoded = jwt.encode( - token_payload, str(BAD_PRIVATE_KEY.decode("utf-8")), algorithm="RS256" - ) - client = RequestsClient() - response = client.get(f"{live_server.url}", headers=get_headers(encoded)) - assert response.status_code == 403 - assert response.json()["detail"] == "Unable to verify token!" - - -@pytest.mark.django_db -def test_bad_public_key_forbidden(settings, live_server, jwt_keys): - with NamedTemporaryFile() as jwt_public_key_file: - jwt_public_key_file.write(BAD_PUBLIC_KEY) - jwt_public_key_file.flush() - settings.PATH_TO_JWT_PUBLIC_KEY = jwt_public_key_file.name - token_payload = get_token_payload() - encoded = jwt.encode( - token_payload, str(jwt_keys.private_key), algorithm="RS256" - ) - client = RequestsClient() - response = client.get(f"{live_server.url}", headers=get_headers(encoded)) - assert response.status_code == 403 - assert response.json()["detail"] == "Unable to verify token!" - - -@pytest.mark.django_db -def test_token_expired_1_min_forbidden(live_server, jwt_keys): - current_datetime = datetime.now() - token_payload = get_token_payload(exp=(current_datetime - one_min).timestamp()) - encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") - client = RequestsClient() - response = client.get(f"{live_server.url}", headers=get_headers(encoded)) - assert response.status_code == 403 - assert response.json()["detail"] == "Token is expired!" - - -@pytest.mark.django_db -def test_token_expires_in_1_min_accepted(live_server, jwt_keys): - current_datetime = datetime.now() - token_payload = get_token_payload(exp=(current_datetime + one_min).timestamp()) - encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") - client = RequestsClient() - response = client.get(f"{live_server.url}", headers=get_headers(encoded)) - assert response.status_code == 200 - - -@pytest.mark.django_db -def test_token_role_user_forbidden(live_server, jwt_keys): - token_payload = get_token_payload(authorities=["ROLE_USER"]) - encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") - client = RequestsClient() - response = client.get(f"{live_server.url}", headers=get_headers(encoded)) - assert response.status_code == 403 - assert response.json()["detail"] == "User missing required role" - - -@pytest.mark.django_db -def test_token_role_user_required_role_accepted(settings, live_server, jwt_keys): - settings.REQUIRED_ROLE = "ROLE_USER" - token_payload = get_token_payload(authorities=["ROLE_USER"]) - encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") - client = RequestsClient() - response = client.get(f"{live_server.url}", headers=get_headers(encoded)) - assert response.status_code == 200 - - -@pytest.mark.django_db -def test_token_multiple_roles_accepted(live_server, jwt_keys): - token_payload = get_token_payload( - authorities=["ROLE_MANAGER", "ROLE_USER", "ROLE_ITS"] - ) - encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") - client = RequestsClient() - response = client.get(f"{live_server.url}", headers=get_headers(encoded)) - assert response.status_code == 200 - - -@pytest.mark.django_db -def test_token_mulitple_roles_forbidden(live_server, jwt_keys): - token_payload = get_token_payload( - authorities=["ROLE_SENSOR", "ROLE_USER", "ROLE_ITS"] - ) - encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") - client = RequestsClient() - response = client.get(f"{live_server.url}", headers=get_headers(encoded)) - assert response.status_code == 403 - - -@pytest.mark.django_db -def test_urls_unauthorized(live_server, jwt_keys): - token_payload = get_token_payload(authorities=["ROLE_USER"]) - encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") - client = RequestsClient() - headers = get_headers(encoded) - - capabilities = reverse("capabilities", kwargs=V1) - response = client.get(f"{live_server.url}{capabilities}", headers=headers) - assert response.status_code == 403 - - schedule_list = reverse("schedule-list", kwargs=V1) - response = client.get(f"{live_server.url}{schedule_list}", headers=headers) - assert response.status_code == 403 - - status = reverse("status", kwargs=V1) - response = client.get(f"{live_server.url}{status}", headers=headers) - assert response.status_code == 403 - - task_root = reverse("task-root", kwargs=V1) - response = client.get(f"{live_server.url}{task_root}", headers=headers) - assert response.status_code == 403 - - task_results_overview = reverse("task-results-overview", kwargs=V1) - response = client.get(f"{live_server.url}{task_results_overview}", headers=headers) - assert response.status_code == 403 - - upcoming_tasks = reverse("upcoming-tasks", kwargs=V1) - response = client.get(f"{live_server.url}{upcoming_tasks}", headers=headers) - assert response.status_code == 403 - - user_list = reverse("user-list", kwargs=V1) - response = client.get(f"{live_server.url}{user_list}", headers=headers) - assert response.status_code == 403 - - -@pytest.mark.django_db -def test_urls_authorized(live_server, jwt_keys): - token_payload = get_token_payload(authorities=["ROLE_MANAGER"]) - encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") - client = RequestsClient() - headers = get_headers(encoded) - - capabilities = reverse("capabilities", kwargs=V1) - response = client.get(f"{live_server.url}{capabilities}", headers=headers) - assert response.status_code == 200 - - schedule_list = reverse("schedule-list", kwargs=V1) - response = client.get(f"{live_server.url}{schedule_list}", headers=headers) - assert response.status_code == 200 - - status = reverse("status", kwargs=V1) - response = client.get(f"{live_server.url}{status}", headers=headers) - assert response.status_code == 200 - - task_root = reverse("task-root", kwargs=V1) - response = client.get(f"{live_server.url}{task_root}", headers=headers) - assert response.status_code == 200 - - task_results_overview = reverse("task-results-overview", kwargs=V1) - response = client.get(f"{live_server.url}{task_results_overview}", headers=headers) - assert response.status_code == 200 - - upcoming_tasks = reverse("upcoming-tasks", kwargs=V1) - response = client.get(f"{live_server.url}{upcoming_tasks}", headers=headers) - assert response.status_code == 200 - - user_list = reverse("user-list", kwargs=V1) - response = client.get(f"{live_server.url}{user_list}", headers=headers) - assert response.status_code == 200 - - -@pytest.mark.django_db -def test_user_cannot_view_user_detail(live_server, jwt_keys): - sensor01_token_payload = get_token_payload(authorities=["ROLE_MANAGER"]) - encoded = jwt.encode( - sensor01_token_payload, str(jwt_keys.private_key), algorithm="RS256" - ) - client = RequestsClient() - response = client.get(f"{live_server.url}", headers=get_headers(encoded)) - assert response.status_code == 200 - - sensor02_token_payload = get_token_payload(authorities=["ROLE_USER"]) - sensor02_token_payload["user_name"] = "sensor02" - encoded = jwt.encode( - sensor02_token_payload, str(jwt_keys.private_key), algorithm="RS256" - ) - client = RequestsClient() - - sensor01_user = User.objects.get(username=sensor01_token_payload["user_name"]) - kws = {"pk": sensor01_user.pk} - kws.update(V1) - user_detail = reverse("user-detail", kwargs=kws) - response = client.get( - f"{live_server.url}{user_detail}", headers=get_headers(encoded) - ) - assert response.status_code == 403 - - -@pytest.mark.django_db -def test_user_cannot_view_user_detail_role_change(live_server, jwt_keys): - sensor01_token_payload = get_token_payload(authorities=["ROLE_MANAGER"]) - encoded = jwt.encode( - sensor01_token_payload, str(jwt_keys.private_key), algorithm="RS256" - ) - client = RequestsClient() - response = client.get(f"{live_server.url}", headers=get_headers(encoded)) - assert response.status_code == 200 - - token_payload = get_token_payload(authorities=["ROLE_USER"]) - encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") - client = RequestsClient() - - sensor01_user = User.objects.get(username=sensor01_token_payload["user_name"]) - kws = {"pk": sensor01_user.pk} - kws.update(V1) - user_detail = reverse("user-detail", kwargs=kws) - response = client.get( - f"{live_server.url}{user_detail}", headers=get_headers(encoded) - ) - assert response.status_code == 403 - - -@pytest.mark.django_db -def test_admin_can_view_user_detail(live_server, jwt_keys): - token_payload = get_token_payload(authorities=["ROLE_MANAGER"]) - encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") - client = RequestsClient() - headers = get_headers(encoded) - response = client.get(f"{live_server.url}", headers=headers) - assert response.status_code == 200 - - sensor01_user = User.objects.get(username=token_payload["user_name"]) - kws = {"pk": sensor01_user.pk} - kws.update(V1) - user_detail = reverse("user-detail", kwargs=kws) - response = client.get(f"{live_server.url}{user_detail}", headers=headers) - assert response.status_code == 200 - - -@pytest.mark.django_db -def test_admin_can_view_other_user_detail(live_server, jwt_keys): - sensor01_token_payload = get_token_payload(authorities=["ROLE_MANAGER"]) - encoded = jwt.encode( - sensor01_token_payload, str(jwt_keys.private_key), algorithm="RS256" - ) - client = RequestsClient() - response = client.get(f"{live_server.url}", headers=get_headers(encoded)) - assert response.status_code == 200 - - sensor02_token_payload = get_token_payload(authorities=["ROLE_MANAGER"]) - sensor02_token_payload["user_name"] = "sensor02" - encoded = jwt.encode( - sensor02_token_payload, str(jwt_keys.private_key), algorithm="RS256" - ) - client = RequestsClient() - - sensor01_user = User.objects.get(username=sensor01_token_payload["user_name"]) - kws = {"pk": sensor01_user.pk} - kws.update(V1) - user_detail = reverse("user-detail", kwargs=kws) - response = client.get( - f"{live_server.url}{user_detail}", headers=get_headers(encoded) - ) - assert response.status_code == 200 - - -@pytest.mark.django_db -def test_token_hidden(live_server, jwt_keys): - token_payload = get_token_payload(authorities=["ROLE_MANAGER"]) - encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") - client = RequestsClient() - headers = get_headers(encoded) - response = client.get(f"{live_server.url}", headers=headers) - assert response.status_code == 200 - - sensor01_user = User.objects.get(username=token_payload["user_name"]) - kws = {"pk": sensor01_user.pk} - kws.update(V1) - user_detail = reverse("user-detail", kwargs=kws) - client = RequestsClient() - response = client.get(f"{live_server.url}{user_detail}", headers=headers) - assert response.status_code == 200 - assert ( - response.json()["auth_token"] - == "rest_framework.authentication.TokenAuthentication is not enabled" - ) - - -@pytest.mark.django_db -def test_change_token_role_bad_signature(live_server, jwt_keys): - """Make sure token modified after it was signed is rejected""" - token_payload = get_token_payload(authorities=["ROLE_USER"]) - encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") - first_period = encoded.find(".") - second_period = encoded.find(".", first_period + 1) - payload = encoded[first_period + 1 : second_period] - payload_bytes = payload.encode("utf-8") - # must be multiple of 4 for b64decode - for i in range(len(payload_bytes) % 4): - payload_bytes = payload_bytes + b"=" - decoded = base64.b64decode(payload_bytes) - payload_str = decoded.decode("utf-8") - payload_data = json.loads(payload_str) - payload_data["user_name"] = "sensor013" - payload_data["authorities"] = ["ROLE_MANAGER"] - payload_data["userDetails"]["authorities"] = [{"authority": "ROLE_MANAGER"}] - payload_str = json.dumps(payload_data) - modified_payload = base64.b64encode(payload_str.encode("utf-8")) - modified_payload = modified_payload.decode("utf-8") - # remove padding - if modified_payload.endswith("="): - last_padded_index = len(modified_payload) - 1 - for i in range(len(modified_payload) - 1, -1, -1): - if modified_payload[i] != "=": - last_padded_index = i - break - modified_payload = modified_payload[: last_padded_index + 1] - modified_token = ( - encoded[:first_period] - + "." - + modified_payload - + "." - + encoded[second_period + 1 :] - ) - client = RequestsClient() - response = client.get(f"{live_server.url}", headers=get_headers(modified_token)) - assert response.status_code == 403 - assert response.json()["detail"] == "Unable to verify token!" diff --git a/src/authentication/tests/utils.py b/src/authentication/tests/utils.py deleted file mode 100644 index 3ee92b5d..00000000 --- a/src/authentication/tests/utils.py +++ /dev/null @@ -1,21 +0,0 @@ -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric import rsa - - -def get_test_public_private_key(): - """Creates public/private key pair for testing - https://stackoverflow.com/a/39126754 - """ - key = rsa.generate_private_key( - backend=default_backend(), public_exponent=65537, key_size=4096 - ) - private_key = key.private_bytes( - serialization.Encoding.PEM, - serialization.PrivateFormat.PKCS8, - serialization.NoEncryption(), - ) - public_key = key.public_key().public_bytes( - serialization.Encoding.PEM, serialization.PublicFormat.SubjectPublicKeyInfo - ) - return private_key, public_key diff --git a/src/conftest.py b/src/conftest.py index ec8667eb..a615cba8 100644 --- a/src/conftest.py +++ b/src/conftest.py @@ -8,13 +8,9 @@ import scheduler from authentication.models import User -from authentication.tests.utils import get_test_public_private_key +from sensor.tests.certificate_auth_client import CertificateAuthClient from tasks.models import TaskResult -PRIVATE_KEY, PUBLIC_KEY = get_test_public_private_key() -Keys = namedtuple("KEYS", ["private_key", "public_key"]) -keys = Keys(PRIVATE_KEY.decode("utf-8"), PUBLIC_KEY.decode("utf-8")) - @pytest.fixture(autouse=True) def cleanup_db(db): @@ -24,7 +20,7 @@ def cleanup_db(db): shutil.rmtree(settings.MEDIA_ROOT, ignore_errors=True) -@pytest.yield_fixture +@pytest.fixture def testclock(): """Replace scheduler's timefn with manually steppable test timefn.""" # Setup test clock @@ -66,11 +62,18 @@ def user(db): @pytest.fixture def user_client(db, user): """A Django test client logged in as a normal user""" - client = Client() + client = CertificateAuthClient() client.login(username=user.username, password=user.password) return client +@pytest.fixture +def admin_client(db, admin_user): + """A Django test client logged in as an admin user""" + client = CertificateAuthClient() + client.login(username=admin_user.username, password=admin_user.password) + + return client @pytest.fixture def alt_user(db): @@ -92,7 +95,7 @@ def alt_user(db): @pytest.fixture def alt_user_client(db, alt_user): """A Django test client logged in as a normal user""" - client = Client() + client = CertificateAuthClient() client.login(username=alt_user.username, password=alt_user.password) return client @@ -129,16 +132,7 @@ def alt_admin_client(db, alt_admin_user): """A Django test client logged in as an admin user.""" from django.test.client import Client - client = Client() + client = CertificateAuthClient() client.login(username=alt_admin_user.username, password="password") return client - - -@pytest.fixture(autouse=True) -def jwt_keys(settings): - with tempfile.NamedTemporaryFile() as jwt_public_key_file: - jwt_public_key_file.write(PUBLIC_KEY) - jwt_public_key_file.flush() - settings.PATH_TO_JWT_PUBLIC_KEY = jwt_public_key_file.name - yield keys diff --git a/src/scheduler/scheduler.py b/src/scheduler/scheduler.py index 346845f7..2b012397 100644 --- a/src/scheduler/scheduler.py +++ b/src/scheduler/scheduler.py @@ -188,7 +188,7 @@ def _finalize_task_result(self, started, finished, status, detail): data=json.dumps(result_json), headers=headers, verify=verify_ssl, - cert=(settings.PATH_TO_CLIENT_CERT, settings.PATH_TO_CLIENT_KEY) + cert=settings.PATH_TO_CLIENT_CERT ) self._callback_response_handler(response, tr) else: diff --git a/src/scheduler/tests/test_scheduler.py b/src/scheduler/tests/test_scheduler.py index 3810a5e6..10350ad4 100644 --- a/src/scheduler/tests/test_scheduler.py +++ b/src/scheduler/tests/test_scheduler.py @@ -317,24 +317,16 @@ def test_minimum_duration_non_blocking(): def verify_request(request_history, status="success", detail=None): request_json = None - if conf.settings.CALLBACK_AUTHENTICATION == "OAUTH": - oauth_history = request_history[0] - assert oauth_history.verify == conf.settings.PATH_TO_VERIFY_CERT - assert ( - oauth_history.text - == f"grant_type=password&username={conf.settings.USER_NAME}&password={conf.settings.PASSWORD}" - ) - assert oauth_history.cert == conf.settings.PATH_TO_CLIENT_CERT - auth_header = oauth_history.headers.get("Authorization") - auth_header = auth_header.replace("Basic ", "") - auth_header_decoded = base64.b64decode(auth_header).decode("utf-8") - assert ( - auth_header_decoded - == f"{conf.settings.CLIENT_ID}:{conf.settings.CLIENT_SECRET}" - ) - request_json = request_history[1].json() - else: - request_json = request_history[0].json() + history = request_history[0] + if conf.settings.CALLBACK_SSL_VERIFICATION: + if conf.settings.PATH_TO_VERIFY_CERT: + assert history.verify == conf.settings.PATH_TO_VERIFY_CERT + else: + assert history.verify == True + if conf.settings.CALLBACK_AUTHENTICATION == "CERT": + assert history.cert == conf.settings.PATH_TO_CLIENT_CERT + + request_json = request_history[0].json() assert request_json["status"] == status assert request_json["task_id"] == 1 assert request_json["self"] @@ -348,9 +340,7 @@ def verify_request(request_history, status="success", detail=None): @pytest.mark.django_db def test_failure_posted_to_callback_url(test_scheduler, settings): """If an entry has callback_url defined, scheduler should POST to it.""" - oauth_token_url = "https://auth/mock" callback_url = "https://results" - settings.OAUTH_TOKEN_URL = oauth_token_url cb_flag = threading.Event() def cb_request_handler(sess, resp): @@ -368,16 +358,14 @@ def cb_request_handler(sess, resp): request_history = None with requests_mock.Mocker() as m: # register mock url for posting - if settings.CALLBACK_AUTHENTICATION == "OAUTH": + if settings.CALLBACK_AUTHENTICATION == "CERT": m.post( callback_url, - request_headers={"Authorization": "Bearer " + "test_access_token"}, ) else: m.post( callback_url, request_headers={"Authorization": "Token " + str(token)} ) - m.post(oauth_token_url, json={"access_token": "test_access_token"}) s.run(blocking=False) time.sleep(0.1) # let requests thread run request_history = m.request_history @@ -389,9 +377,7 @@ def cb_request_handler(sess, resp): @pytest.mark.django_db def test_success_posted_to_callback_url(test_scheduler, settings): """If an entry has callback_url defined, scheduler should POST to it.""" - oauth_token_url = "https://auth/mock" callback_url = "https://results" - settings.OAUTH_TOKEN_URL = oauth_token_url cb_flag = threading.Event() def cb_request_handler(sess, resp): @@ -410,20 +396,17 @@ def cb_request_handler(sess, resp): request_history = None with requests_mock.Mocker() as m: # register mock url for posting - if settings.CALLBACK_AUTHENTICATION == "OAUTH": + if settings.CALLBACK_AUTHENTICATION == "CERT": m.post( callback_url, - request_headers={"Authorization": "Bearer " + "test_access_token"}, ) else: m.post( callback_url, request_headers={"Authorization": "Token " + str(token)} ) - m.post(oauth_token_url, json={"access_token": "test_access_token"}) s.run(blocking=False) time.sleep(0.1) # let requests thread run request_history = m.request_history - # request_json = m.request_history[0].json() assert cb_flag.is_set() assert action_flag.is_set() diff --git a/src/sensor/migration_settings.py b/src/sensor/migration_settings.py index d967b3aa..05dabed1 100644 --- a/src/sensor/migration_settings.py +++ b/src/sensor/migration_settings.py @@ -374,12 +374,7 @@ CALLBACK_SSL_VERIFICATION = env.bool("CALLBACK_SSL_VERIFICATION", default=True) # OAuth Password Flow Authentication CALLBACK_AUTHENTICATION = env("CALLBACK_AUTHENTICATION", default="") -CLIENT_ID = env("CLIENT_ID", default="") -CLIENT_SECRET = env("CLIENT_SECRET", default="") -USER_NAME = CLIENT_ID -PASSWORD = CLIENT_SECRET -OAUTH_TOKEN_URL = env("OAUTH_TOKEN_URL", default="") CERTS_DIR = path.join(CONFIG_DIR, "certs") # Sensor certificate with private key used as client cert PATH_TO_CLIENT_CERT = env("PATH_TO_CLIENT_CERT", default="") diff --git a/src/sensor/runtime_settings.py b/src/sensor/runtime_settings.py index dd5fe7a5..4cf5854e 100644 --- a/src/sensor/runtime_settings.py +++ b/src/sensor/runtime_settings.py @@ -374,12 +374,7 @@ CALLBACK_SSL_VERIFICATION = env.bool("CALLBACK_SSL_VERIFICATION", default=True) # OAuth Password Flow Authentication CALLBACK_AUTHENTICATION = env("CALLBACK_AUTHENTICATION", default="") -CLIENT_ID = env("CLIENT_ID", default="") -CLIENT_SECRET = env("CLIENT_SECRET", default="") -USER_NAME = CLIENT_ID -PASSWORD = CLIENT_SECRET -OAUTH_TOKEN_URL = env("OAUTH_TOKEN_URL", default="") CERTS_DIR = path.join(CONFIG_DIR, "certs") # Sensor certificate with private key used as client cert PATH_TO_CLIENT_CERT = env("PATH_TO_CLIENT_CERT", default="") diff --git a/src/sensor/settings.py b/src/sensor/settings.py index f0969beb..e799bf65 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -375,12 +375,7 @@ CALLBACK_SSL_VERIFICATION = env.bool("CALLBACK_SSL_VERIFICATION", default=True) # OAuth Password Flow Authentication CALLBACK_AUTHENTICATION = env("CALLBACK_AUTHENTICATION", default="") -CLIENT_ID = env("CLIENT_ID", default="") -CLIENT_SECRET = env("CLIENT_SECRET", default="") -USER_NAME = CLIENT_ID -PASSWORD = CLIENT_SECRET -OAUTH_TOKEN_URL = env("OAUTH_TOKEN_URL", default="") CERTS_DIR = path.join(CONFIG_DIR, "certs") # Sensor certificate with private key used as client cert PATH_TO_CLIENT_CERT = env("PATH_TO_CLIENT_CERT", default="") diff --git a/src/sensor/tests/certificate_auth_client.py b/src/sensor/tests/certificate_auth_client.py new file mode 100644 index 00000000..e8fca0a5 --- /dev/null +++ b/src/sensor/tests/certificate_auth_client.py @@ -0,0 +1,52 @@ +from django.test.client import Client, MULTIPART_CONTENT +from django.conf import settings +from django.core.serializers.json import DjangoJSONEncoder + +from sensor.tests.utils import get_http_request_ssl_dn_header + +cert_auth_enabled = settings.AUTHENTICATION == "CERT" + +class CertificateAuthClient(Client): + """Adds SSL DN header if certificate authentication is being used""" + + def __init__(self, enforce_csrf_checks=False, json_encoder=DjangoJSONEncoder, **defaults) -> None: + super().__init__(enforce_csrf_checks, json_encoder=json_encoder, **defaults) + self.username = "" + + def get_kwargs(self, extra): + kwargs = {} + kwargs.update(extra) + if cert_auth_enabled: + kwargs.update(get_http_request_ssl_dn_header(self.username)) + return kwargs + + def get(self, path, data=None, follow=False, secure=False, **extra): + return super().get(path, data, follow, secure, **self.get_kwargs(extra)) + + def post(self, path, data=None, content_type=MULTIPART_CONTENT, follow=False, secure=False, **extra): + return super().post(path, data, content_type, follow, secure, **self.get_kwargs(extra)) + + def head(self, path, data=None, follow=False, secure=False, **extra): + pass + + def options(self, path, data='', content_type='application/octet-stream', follow=False, secure=False, **extra): + pass + + def put(self, path, data='', content_type='application/octet-stream', follow=False, secure=False, **extra): + return super().put(path, data, content_type, follow, secure, **self.get_kwargs(extra)) + + def patch(self, path, data='', content_type='application/octet-stream', follow=False, secure=False, **extra): + pass + + def delete(self, path, data='', content_type='application/octet-stream', follow=False, secure=False, **extra): + return super().delete(path, data, content_type, follow, secure, **self.get_kwargs(extra)) + + def trace(self, path, follow=False, secure=False, **extra): + pass + + def login(self, **credentials): + if cert_auth_enabled: + assert "username" in credentials + self.username = credentials["username"] + else: + super().login(**credentials) diff --git a/src/sensor/tests/utils.py b/src/sensor/tests/utils.py index cf327955..b2de0c51 100644 --- a/src/sensor/tests/utils.py +++ b/src/sensor/tests/utils.py @@ -14,3 +14,13 @@ def validate_response(response, expected_code=None): if actual_code not in (status.HTTP_204_NO_CONTENT,): rjson = response.json() return rjson + +def get_requests_ssl_dn_header(common_name): + return { + "X-Ssl-Client-Dn": f"C=TC,ST=test_state,L=test_locality,O=test_org,OU=test_ou,CN={common_name}", + } + +def get_http_request_ssl_dn_header(common_name): + return { + "HTTP_X-SSL-CLIENT-DN": f"C=TC,ST=test_state,L=test_locality,O=test_org,OU=test_ou,CN={common_name}", + } diff --git a/src/tasks/tests/test_list_view.py b/src/tasks/tests/test_list_view.py index 163516bd..a86fc5f3 100644 --- a/src/tasks/tests/test_list_view.py +++ b/src/tasks/tests/test_list_view.py @@ -35,7 +35,7 @@ def test_user_cannot_view_result_list(admin_client, user_client): url = reverse_result_list(entry_name) response = user_client.get(url, **HTTPS_KWARG) rjson = validate_response(response, status.HTTP_403_FORBIDDEN) - return "results" not in rjson + assert "results" not in rjson @pytest.mark.django_db diff --git a/src/tox.ini b/src/tox.ini index 8ddd3f88..3ee4d73b 100644 --- a/src/tox.ini +++ b/src/tox.ini @@ -14,16 +14,15 @@ setenv = CALLBACK_AUTHENTICATION=TOKEN SWITCH_CONFIGS_DIR=../configs/switches -[testenv:oauth] +[testenv:cert] envlist = py38,py39,py310 setenv = - AUTHENTICATION=JWT - CALLBACK_AUTHENTICATION=OAUTH - CLIENT_ID=sensor01.sms.internal - CLIENT_SECRET=sensor-secret + AUTHENTICATION=CERT + CALLBACK_AUTHENTICATION=CERT PATH_TO_CLIENT_CERT=test/sensor01.pem PATH_TO_VERIFY_CERT=test/scos_test_ca.crt SWITCH_CONFIGS_DIR=../configs/switches + [testenv:coverage] basepython = python3 deps = From d36ce6bcf408fc74355104e0162f15c19bd6189b Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Fri, 27 Jan 2023 12:51:47 -0700 Subject: [PATCH 011/255] update README, remove unused files, fix tests, cleanup, exception handling --- README.md | 6 +- nginx/conf.template | 4 +- scripts/restore_fixture.sh | 66 --- src/authentication/auth.py | 9 +- src/authentication/tests/test_cert_auth.py | 443 ++++++-------------- src/authentication/tests/test_token_auth.py | 2 + src/conftest.py | 8 +- src/schedule/tests/test_user_views.py | 2 +- src/scheduler/tests/test_scheduler.py | 23 +- src/sensor/migration_settings.py | 1 - src/sensor/runtime_settings.py | 1 - src/sensor/settings.py | 1 - src/sensor/tests/certificate_auth_client.py | 19 +- src/sensor/tests/test_api_docs.py | 3 +- src/sensor/tests/utils.py | 2 +- src/status/fixtures/greyhound.json | 12 - src/tasks/tests/test_overview_view.py | 6 +- src/test_utils/task_test_utils.py | 1 + src/tox.ini | 2 +- 19 files changed, 190 insertions(+), 421 deletions(-) delete mode 100755 scripts/restore_fixture.sh delete mode 100644 src/status/fixtures/greyhound.json diff --git a/README.md b/README.md index 217e3d7f..f3b4b53d 100644 --- a/README.md +++ b/README.md @@ -460,8 +460,8 @@ cat sensor01_decrypted.key sensor01.pem > sensor01_combined.pem ##### Client Certificate -This certificate is required for using the sensor with mutual TLS which is required if -OAuth authentication is enabled. +This certificate is required for using the sensor with mutual TLS certificate +authentication. Replace the brackets with the information specific to your user and organization. @@ -542,7 +542,7 @@ will send the user's (user who created the schedule) token in the authorization the token against what it originally sent to the sensor when creating the schedule. This method of authentication for the callback URL is enabled by default. To verify it is enabled, set `CALLBACK_AUTHENTICATION` to `TOKEN` in the environment file (this will -be enabled if `CALLBACK_AUTHENTICATION` set to anything other than `OAUTH`). +be enabled if `CALLBACK_AUTHENTICATION` set to anything other than `CERT`). `PATH_TO_VERIFY_CERT`, in the environment file, can used to set a CA certificate to verify the callback URL server SSL certificate. If this is unset and `CALLBACK_SSL_VERIFICATION` is set to true, [standard trusted CAs]( diff --git a/nginx/conf.template b/nginx/conf.template index 7b79bcac..f39db8c7 100644 --- a/nginx/conf.template +++ b/nginx/conf.template @@ -30,9 +30,9 @@ server { ssl_session_tickets off; ssl_certificate /etc/ssl/certs/ssl-cert.pem; ssl_certificate_key /etc/ssl/private/ssl-cert.key; - ssl_protocols TLSv1.2; + ssl_protocols TLSv1.2 TLSv1.3; ssl_client_certificate /etc/ssl/certs/ca.crt; - ssl_verify_client optional; + ssl_verify_client on; # ssl_ocsp on; # Enable OCSP validation ssl_verify_depth 4; # path for static files diff --git a/scripts/restore_fixture.sh b/scripts/restore_fixture.sh deleted file mode 100755 index 5224e3e6..00000000 --- a/scripts/restore_fixture.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash - -# Exit on error -set -e - -REPO_ROOT=${REPO_ROOT:=$(git rev-parse --show-toplevel)} -PROGRAM_NAME=${0##*/} -INPUT="$1" - - -echo_usage() { - cat << EOF -Restore a fixture file. - -Usage: $PROGRAM_NAME filename - -Example: - $PROGRAM_NAME ./src/capabilities/fixtures/greyhound-2018-02-22.json - -EOF - - exit 0 -} - - -if [[ ! "$INPUT" || "$INPUT" == "-h" || "$INPUT" == "--help" ]]; then - echo_usage - exit 0 -fi - -HOST_PATH=$(readlink -e "$INPUT") -FILENAME=$(basename "$INPUT") -CONTAINER_PATH="/tmp/$FILENAME" - -if [[ ! -e "$HOST_PATH" ]]; then - echo "Fixture file \"$HOST_PATH\" doesn't exist." - exit 1 -fi - -set +e # this command may "fail" -DB_RUNNING=$(docker-compose -f ${REPO_ROOT}/docker-compose.yml ps db |grep Up) -API_RUNNING=$(docker-compose -f ${REPO_ROOT}/docker-compose.yml ps api |grep Up) -set -e - -# Ensure database container is running -docker-compose -f ${REPO_ROOT}/docker-compose.yml up -d db - -# Load given fixture file into database -if [[ "$API_RUNNING" ]]; then - API_CONTAINER=$(docker-compose -f ${REPO_ROOT}/docker-compose.yml ps -q api) - docker cp "$HOST_PATH" ${API_CONTAINER}:/tmp - docker-compose -f ${REPO_ROOT}/docker-compose.yml exec api \ - /src/manage.py loaddata "$CONTAINER_PATH" -else - docker-compose -f ${REPO_ROOT}/docker-compose.yml run \ - -v "$HOST_PATH":"$CONTAINER_PATH" \ - --rm api /src/manage.py loaddata "$CONTAINER_PATH" -fi - -# If the DB was already running, leave it up -if [[ ! "$DB_RUNNING" ]]; then - # Stop database container - docker-compose -f ${REPO_ROOT}/docker-compose.yml stop db -fi - -echo "Restored fixture from $INPUT." diff --git a/src/authentication/auth.py b/src/authentication/auth.py index 1cdcd0b7..9d13e833 100644 --- a/src/authentication/auth.py +++ b/src/authentication/auth.py @@ -21,15 +21,14 @@ def authenticate(self, request): logger.debug("Authenticating certificate.") cert_dn = request.headers.get("X-Ssl-Client-Dn") if cert_dn: - logger.info("DN:" + cert_dn) - cn = get_cn_from_dn(cert_dn) - logger.info("Cert cn: " + cn) user_model = get_user_model() - user = None try: + cn = get_cn_from_dn(cert_dn) user = user_model.objects.get(username=cn) except user_model.DoesNotExist: raise exceptions.AuthenticationFailed("No matching username found!") + except Exception: + raise exceptions.AuthenticationFailed("Error occurred during certificate authentication!") return user, None return None, None @@ -40,7 +39,5 @@ def get_cn_from_dn(cert_dn): if not match: raise Exception("No CN found in certificate!") uid_raw = match.group() - # logger.debug(f"uid_raw = {uid_raw}") uid = uid_raw.split("=")[1].rstrip(",") - # logger.debug(f"uid = {uid}") return uid diff --git a/src/authentication/tests/test_cert_auth.py b/src/authentication/tests/test_cert_auth.py index e8b489ec..36312fa2 100644 --- a/src/authentication/tests/test_cert_auth.py +++ b/src/authentication/tests/test_cert_auth.py @@ -1,10 +1,3 @@ -import base64 -import json -import os -import secrets -from datetime import datetime, timedelta -from tempfile import NamedTemporaryFile - import pytest from rest_framework.reverse import reverse from rest_framework.test import RequestsClient @@ -20,21 +13,39 @@ ) +@pytest.mark.django_db +def test_no_client_cert_unauthorized_no_dn(live_server): + client = RequestsClient() + response = client.get(f"{live_server.url}") + assert response.status_code == 403 + capabilities = reverse("capabilities", kwargs=V1) + response = client.get(f"{live_server.url}{capabilities}") + assert response.status_code == 403 -one_min = timedelta(minutes=1) -one_day = timedelta(days=1) - + schedule_list = reverse("schedule-list", kwargs=V1) + response = client.get(f"{live_server.url}{schedule_list}") + assert response.status_code == 403 + status = reverse("status", kwargs=V1) + response = client.get(f"{live_server.url}{status}") + assert response.status_code == 403 + task_root = reverse("task-root", kwargs=V1) + response = client.get(f"{live_server.url}{task_root}") + assert response.status_code == 403 + task_results_overview = reverse("task-results-overview", kwargs=V1) + response = client.get(f"{live_server.url}{task_results_overview}") + assert response.status_code == 403 -@pytest.mark.django_db -def test_no_client_cert_unauthorized(live_server): - client = RequestsClient() - response = client.get(f"{live_server.url}") + upcoming_tasks = reverse("upcoming-tasks", kwargs=V1) + response = client.get(f"{live_server.url}{upcoming_tasks}") assert response.status_code == 403 + user_list = reverse("user-list", kwargs=V1) + response = client.get(f"{live_server.url}{user_list}") + assert response.status_code == 403 @pytest.mark.django_db @@ -44,299 +55,123 @@ def test_client_cert_accepted(live_server, admin_user): assert response.status_code == 200 -def test_bad_client_cert_forbidden(live_server, admin_user): +def test_mismatching_user_forbidden(live_server, admin_user): client = RequestsClient() response = client.get(f"{live_server.url}", headers=get_requests_ssl_dn_header("user")) assert response.status_code == 403 assert "No matching username found!" in response.json()["detail"] -# @pytest.mark.django_db -# def test_certificate_expired_1_day_forbidden(live_server): -# current_datetime = datetime.now() -# token_payload = get_token_payload(exp=(current_datetime - one_day).timestamp()) -# encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") -# client = RequestsClient() -# response = client.get(f"{live_server.url}", headers=get_headers(encoded)) -# assert response.status_code == 403 -# assert response.json()["detail"] == "Token is expired!" - - -# @pytest.mark.django_db -# def test_bad_private_key_forbidden(live_server): -# token_payload = get_token_payload() -# encoded = jwt.encode( -# token_payload, str(BAD_PRIVATE_KEY.decode("utf-8")), algorithm="RS256" -# ) -# client = RequestsClient() -# response = client.get(f"{live_server.url}", headers=get_headers(encoded)) -# assert response.status_code == 403 -# assert response.json()["detail"] == "Unable to verify token!" - - -# @pytest.mark.django_db -# def test_bad_public_key_forbidden(settings, live_server): -# with NamedTemporaryFile() as jwt_public_key_file: -# jwt_public_key_file.write(BAD_PUBLIC_KEY) -# jwt_public_key_file.flush() -# settings.PATH_TO_JWT_PUBLIC_KEY = jwt_public_key_file.name -# token_payload = get_token_payload() -# encoded = jwt.encode( -# token_payload, str(jwt_keys.private_key), algorithm="RS256" -# ) -# client = RequestsClient() -# response = client.get(f"{live_server.url}", headers=get_headers(encoded)) -# assert response.status_code == 403 -# assert response.json()["detail"] == "Unable to verify token!" - - -# @pytest.mark.django_db -# def test_certificate_expired_1_min_forbidden(live_server): -# current_datetime = datetime.now() -# token_payload = get_token_payload(exp=(current_datetime - one_min).timestamp()) -# encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") -# client = RequestsClient() -# response = client.get(f"{live_server.url}", headers=get_headers(encoded)) -# assert response.status_code == 403 -# assert response.json()["detail"] == "Token is expired!" - - -# @pytest.mark.django_db -# def test_certificate_expires_in_1_min_accepted(live_server): -# current_datetime = datetime.now() -# token_payload = get_token_payload(exp=(current_datetime + one_min).timestamp()) -# encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") -# client = RequestsClient() -# response = client.get(f"{live_server.url}", headers=get_headers(encoded)) -# assert response.status_code == 200 - - -# @pytest.mark.django_db -# def test_urls_unauthorized(live_server): -# token_payload = get_token_payload(authorities=["ROLE_USER"]) -# encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") -# client = RequestsClient() -# headers = get_headers(encoded) - -# capabilities = reverse("capabilities", kwargs=V1) -# response = client.get(f"{live_server.url}{capabilities}", headers=headers) -# assert response.status_code == 403 - -# schedule_list = reverse("schedule-list", kwargs=V1) -# response = client.get(f"{live_server.url}{schedule_list}", headers=headers) -# assert response.status_code == 403 - -# status = reverse("status", kwargs=V1) -# response = client.get(f"{live_server.url}{status}", headers=headers) -# assert response.status_code == 403 - -# task_root = reverse("task-root", kwargs=V1) -# response = client.get(f"{live_server.url}{task_root}", headers=headers) -# assert response.status_code == 403 - -# task_results_overview = reverse("task-results-overview", kwargs=V1) -# response = client.get(f"{live_server.url}{task_results_overview}", headers=headers) -# assert response.status_code == 403 - -# upcoming_tasks = reverse("upcoming-tasks", kwargs=V1) -# response = client.get(f"{live_server.url}{upcoming_tasks}", headers=headers) -# assert response.status_code == 403 - -# user_list = reverse("user-list", kwargs=V1) -# response = client.get(f"{live_server.url}{user_list}", headers=headers) -# assert response.status_code == 403 - - -# @pytest.mark.django_db -# def test_urls_authorized(live_server): -# token_payload = get_token_payload(authorities=["ROLE_MANAGER"]) -# encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") -# client = RequestsClient() -# headers = get_headers(encoded) - -# capabilities = reverse("capabilities", kwargs=V1) -# response = client.get(f"{live_server.url}{capabilities}", headers=headers) -# assert response.status_code == 200 - -# schedule_list = reverse("schedule-list", kwargs=V1) -# response = client.get(f"{live_server.url}{schedule_list}", headers=headers) -# assert response.status_code == 200 - -# status = reverse("status", kwargs=V1) -# response = client.get(f"{live_server.url}{status}", headers=headers) -# assert response.status_code == 200 - -# task_root = reverse("task-root", kwargs=V1) -# response = client.get(f"{live_server.url}{task_root}", headers=headers) -# assert response.status_code == 200 - -# task_results_overview = reverse("task-results-overview", kwargs=V1) -# response = client.get(f"{live_server.url}{task_results_overview}", headers=headers) -# assert response.status_code == 200 - -# upcoming_tasks = reverse("upcoming-tasks", kwargs=V1) -# response = client.get(f"{live_server.url}{upcoming_tasks}", headers=headers) -# assert response.status_code == 200 - -# user_list = reverse("user-list", kwargs=V1) -# response = client.get(f"{live_server.url}{user_list}", headers=headers) -# assert response.status_code == 200 - - -# @pytest.mark.django_db -# def test_user_cannot_view_user_detail(live_server): -# sensor01_token_payload = get_token_payload(authorities=["ROLE_MANAGER"]) -# encoded = jwt.encode( -# sensor01_token_payload, str(jwt_keys.private_key), algorithm="RS256" -# ) -# client = RequestsClient() -# response = client.get(f"{live_server.url}", headers=get_headers(encoded)) -# assert response.status_code == 200 - -# sensor02_token_payload = get_token_payload(authorities=["ROLE_USER"]) -# sensor02_token_payload["user_name"] = "sensor02" -# encoded = jwt.encode( -# sensor02_token_payload, str(jwt_keys.private_key), algorithm="RS256" -# ) -# client = RequestsClient() - -# sensor01_user = User.objects.get(username=sensor01_token_payload["user_name"]) -# kws = {"pk": sensor01_user.pk} -# kws.update(V1) -# user_detail = reverse("user-detail", kwargs=kws) -# response = client.get( -# f"{live_server.url}{user_detail}", headers=get_headers(encoded) -# ) -# assert response.status_code == 403 - - -# @pytest.mark.django_db -# def test_user_cannot_view_user_detail_role_change(live_server): -# sensor01_token_payload = get_token_payload(authorities=["ROLE_MANAGER"]) -# encoded = jwt.encode( -# sensor01_token_payload, str(jwt_keys.private_key), algorithm="RS256" -# ) -# client = RequestsClient() -# response = client.get(f"{live_server.url}", headers=get_headers(encoded)) -# assert response.status_code == 200 - -# token_payload = get_token_payload(authorities=["ROLE_USER"]) -# encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") -# client = RequestsClient() - -# sensor01_user = User.objects.get(username=sensor01_token_payload["user_name"]) -# kws = {"pk": sensor01_user.pk} -# kws.update(V1) -# user_detail = reverse("user-detail", kwargs=kws) -# response = client.get( -# f"{live_server.url}{user_detail}", headers=get_headers(encoded) -# ) -# assert response.status_code == 403 - - -# @pytest.mark.django_db -# def test_admin_can_view_user_detail(live_server): -# token_payload = get_token_payload(authorities=["ROLE_MANAGER"]) -# encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") -# client = RequestsClient() -# headers = get_headers(encoded) -# response = client.get(f"{live_server.url}", headers=headers) -# assert response.status_code == 200 - -# sensor01_user = User.objects.get(username=token_payload["user_name"]) -# kws = {"pk": sensor01_user.pk} -# kws.update(V1) -# user_detail = reverse("user-detail", kwargs=kws) -# response = client.get(f"{live_server.url}{user_detail}", headers=headers) -# assert response.status_code == 200 - - -# @pytest.mark.django_db -# def test_admin_can_view_other_user_detail(live_server): -# sensor01_token_payload = get_token_payload(authorities=["ROLE_MANAGER"]) -# encoded = jwt.encode( -# sensor01_token_payload, str(jwt_keys.private_key), algorithm="RS256" -# ) -# client = RequestsClient() -# response = client.get(f"{live_server.url}", headers=get_headers(encoded)) -# assert response.status_code == 200 - -# sensor02_token_payload = get_token_payload(authorities=["ROLE_MANAGER"]) -# sensor02_token_payload["user_name"] = "sensor02" -# encoded = jwt.encode( -# sensor02_token_payload, str(jwt_keys.private_key), algorithm="RS256" -# ) -# client = RequestsClient() - -# sensor01_user = User.objects.get(username=sensor01_token_payload["user_name"]) -# kws = {"pk": sensor01_user.pk} -# kws.update(V1) -# user_detail = reverse("user-detail", kwargs=kws) -# response = client.get( -# f"{live_server.url}{user_detail}", headers=get_headers(encoded) -# ) -# assert response.status_code == 200 - - -# @pytest.mark.django_db -# def test_token_hidden(live_server): -# token_payload = get_token_payload(authorities=["ROLE_MANAGER"]) -# encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") -# client = RequestsClient() -# headers = get_headers(encoded) -# response = client.get(f"{live_server.url}", headers=headers) -# assert response.status_code == 200 - -# sensor01_user = User.objects.get(username=token_payload["user_name"]) -# kws = {"pk": sensor01_user.pk} -# kws.update(V1) -# user_detail = reverse("user-detail", kwargs=kws) -# client = RequestsClient() -# response = client.get(f"{live_server.url}{user_detail}", headers=headers) -# assert response.status_code == 200 -# assert ( -# response.json()["auth_token"] -# == "rest_framework.authentication.TokenAuthentication is not enabled" -# ) - - -# @pytest.mark.django_db -# def test_change_token_role_bad_signature(live_server): -# """Make sure token modified after it was signed is rejected""" -# token_payload = get_token_payload(authorities=["ROLE_USER"]) -# encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256") -# first_period = encoded.find(".") -# second_period = encoded.find(".", first_period + 1) -# payload = encoded[first_period + 1 : second_period] -# payload_bytes = payload.encode("utf-8") -# # must be multiple of 4 for b64decode -# for i in range(len(payload_bytes) % 4): -# payload_bytes = payload_bytes + b"=" -# decoded = base64.b64decode(payload_bytes) -# payload_str = decoded.decode("utf-8") -# payload_data = json.loads(payload_str) -# payload_data["user_name"] = "sensor013" -# payload_data["authorities"] = ["ROLE_MANAGER"] -# payload_data["userDetails"]["authorities"] = [{"authority": "ROLE_MANAGER"}] -# payload_str = json.dumps(payload_data) -# modified_payload = base64.b64encode(payload_str.encode("utf-8")) -# modified_payload = modified_payload.decode("utf-8") -# # remove padding -# if modified_payload.endswith("="): -# last_padded_index = len(modified_payload) - 1 -# for i in range(len(modified_payload) - 1, -1, -1): -# if modified_payload[i] != "=": -# last_padded_index = i -# break -# modified_payload = modified_payload[: last_padded_index + 1] -# modified_token = ( -# encoded[:first_period] -# + "." -# + modified_payload -# + "." -# + encoded[second_period + 1 :] -# ) -# client = RequestsClient() -# response = client.get(f"{live_server.url}", headers=get_headers(modified_token)) -# assert response.status_code == 403 -# assert response.json()["detail"] == "Unable to verify token!" +@pytest.mark.django_db +def test_urls_unauthorized_not_superuser(live_server, user): + client = RequestsClient() + headers = get_requests_ssl_dn_header("user") + + capabilities = reverse("capabilities", kwargs=V1) + response = client.get(f"{live_server.url}{capabilities}", headers=headers) + assert response.status_code == 403 + + schedule_list = reverse("schedule-list", kwargs=V1) + response = client.get(f"{live_server.url}{schedule_list}", headers=headers) + assert response.status_code == 403 + + status = reverse("status", kwargs=V1) + response = client.get(f"{live_server.url}{status}", headers=headers) + assert response.status_code == 403 + + task_root = reverse("task-root", kwargs=V1) + response = client.get(f"{live_server.url}{task_root}", headers=headers) + assert response.status_code == 403 + + task_results_overview = reverse("task-results-overview", kwargs=V1) + response = client.get(f"{live_server.url}{task_results_overview}", headers=headers) + assert response.status_code == 403 + + upcoming_tasks = reverse("upcoming-tasks", kwargs=V1) + response = client.get(f"{live_server.url}{upcoming_tasks}", headers=headers) + assert response.status_code == 403 + + user_list = reverse("user-list", kwargs=V1) + response = client.get(f"{live_server.url}{user_list}", headers=headers) + assert response.status_code == 403 + + +@pytest.mark.django_db +def test_urls_authorized(live_server, admin_user): + client = RequestsClient() + headers = get_requests_ssl_dn_header("admin") + + capabilities = reverse("capabilities", kwargs=V1) + response = client.get(f"{live_server.url}{capabilities}", headers=headers) + assert response.status_code == 200 + + schedule_list = reverse("schedule-list", kwargs=V1) + response = client.get(f"{live_server.url}{schedule_list}", headers=headers) + assert response.status_code == 200 + + status = reverse("status", kwargs=V1) + response = client.get(f"{live_server.url}{status}", headers=headers) + assert response.status_code == 200 + + task_root = reverse("task-root", kwargs=V1) + response = client.get(f"{live_server.url}{task_root}", headers=headers) + assert response.status_code == 200 + + task_results_overview = reverse("task-results-overview", kwargs=V1) + response = client.get(f"{live_server.url}{task_results_overview}", headers=headers) + assert response.status_code == 200 + + upcoming_tasks = reverse("upcoming-tasks", kwargs=V1) + response = client.get(f"{live_server.url}{upcoming_tasks}", headers=headers) + assert response.status_code == 200 + + user_list = reverse("user-list", kwargs=V1) + response = client.get(f"{live_server.url}{user_list}", headers=headers) + assert response.status_code == 200 + + +@pytest.mark.django_db +def test_token_hidden(live_server, admin_user): + client = RequestsClient() + headers = get_requests_ssl_dn_header("admin") + response = client.get(f"{live_server.url}", headers=headers) + assert response.status_code == 200 + + sensor01_user = User.objects.get(username=admin_user.username) + kws = {"pk": sensor01_user.pk} + kws.update(V1) + user_detail = reverse("user-detail", kwargs=kws) + client = RequestsClient() + response = client.get(f"{live_server.url}{user_detail}", headers=headers) + assert response.status_code == 200 + assert ( + response.json()["auth_token"] + == "rest_framework.authentication.TokenAuthentication is not enabled" + ) + + +@pytest.mark.django_db +def test_empty_common_name_unauthorized(live_server, admin_user): + client = RequestsClient() + response = client.get(f"{live_server.url}", headers=get_requests_ssl_dn_header("")) + assert response.status_code == 403 + + +@pytest.mark.django_db +def test_invalid_dn_unauthorized(live_server, admin_user): + client = RequestsClient() + headers = { + "X-Ssl-Client-Dn": f"C=TC,ST=test_state,L=test_locality,O=test_org,OU=test_ou", + } + response = client.get(f"{live_server.url}", headers=headers) + assert response.status_code == 403 + + +@pytest.mark.django_db +def test_empty_dn_unauthorized(live_server, admin_user): + client = RequestsClient() + headers = { + "X-Ssl-Client-Dn": "", + } + response = client.get(f"{live_server.url}", headers=headers) + assert response.status_code == 403 \ No newline at end of file diff --git a/src/authentication/tests/test_token_auth.py b/src/authentication/tests/test_token_auth.py index c0e7aae6..f8f1d48b 100644 --- a/src/authentication/tests/test_token_auth.py +++ b/src/authentication/tests/test_token_auth.py @@ -6,6 +6,7 @@ from authentication.auth import token_auth_enabled from sensor import V1 +from sensor.tests.utils import HTTPS_KWARG pytestmark = pytest.mark.skipif( not token_auth_enabled, reason="Token authentication is not enabled!" @@ -160,6 +161,7 @@ def test_user_cannot_view_user_detail(settings, live_server, user_client, user): response = user_client.get( f"{live_server.url}{user_detail}", headers={"Authorization": f"Token {user.auth_token.key}"}, + **HTTPS_KWARG ) assert response.status_code == 403 diff --git a/src/conftest.py b/src/conftest.py index a615cba8..acdc0b75 100644 --- a/src/conftest.py +++ b/src/conftest.py @@ -63,7 +63,7 @@ def user(db): def user_client(db, user): """A Django test client logged in as a normal user""" client = CertificateAuthClient() - client.login(username=user.username, password=user.password) + assert client.login(username=user.username, password=user.password) return client @@ -71,7 +71,7 @@ def user_client(db, user): def admin_client(db, admin_user): """A Django test client logged in as an admin user""" client = CertificateAuthClient() - client.login(username=admin_user.username, password=admin_user.password) + assert client.login(username=admin_user.username, password="password") return client @@ -96,7 +96,7 @@ def alt_user(db): def alt_user_client(db, alt_user): """A Django test client logged in as a normal user""" client = CertificateAuthClient() - client.login(username=alt_user.username, password=alt_user.password) + assert client.login(username=alt_user.username, password=alt_user.password) return client @@ -133,6 +133,6 @@ def alt_admin_client(db, alt_admin_user): from django.test.client import Client client = CertificateAuthClient() - client.login(username=alt_admin_user.username, password="password") + assert client.login(username=alt_admin_user.username, password="password") return client diff --git a/src/schedule/tests/test_user_views.py b/src/schedule/tests/test_user_views.py index 1d9b3a65..10b8c511 100644 --- a/src/schedule/tests/test_user_views.py +++ b/src/schedule/tests/test_user_views.py @@ -32,6 +32,6 @@ def test_user_cannot_view_schedule_entry_detail(user_client, admin_client): kws = {"pk": admin_entry_name} kws.update(V1) admin_url = reverse("schedule-detail", kwargs=kws) - response = user_client.get(admin_url) + response = user_client.get(admin_url, **HTTPS_KWARG) rjson = validate_response(response, status.HTTP_403_FORBIDDEN) assert rjson != EMPTY_SCHEDULE_RESPONSE diff --git a/src/scheduler/tests/test_scheduler.py b/src/scheduler/tests/test_scheduler.py index 10350ad4..8d73d586 100644 --- a/src/scheduler/tests/test_scheduler.py +++ b/src/scheduler/tests/test_scheduler.py @@ -415,15 +415,20 @@ def cb_request_handler(sess, resp): @pytest.mark.django_db def test_notification_failed_status_unknown_host(test_scheduler): - entry = create_entry("t", 1, 1, 100, 5, "logger", "https://badmgr.its.bldrdoc.gov") - entry.save() - entry.refresh_from_db() - print("entry = " + entry.name) - s = test_scheduler - advance_testclock(s.timefn, 1) - s.run(blocking=False) # queue first 10 tasks - result = TaskResult.objects.first() - assert result.status == "notification_failed" + with requests_mock.Mocker() as m: + callback_url = "https://results" + entry = create_entry("t", 1, 1, 100, 5, "logger", callback_url) + entry.save() + token = entry.owner.auth_token + m.post(callback_url, request_headers={"Authorization": "Token " + str(token)}, text='Not Found', status_code=404) + + entry.refresh_from_db() + print("entry = " + entry.name) + s = test_scheduler + advance_testclock(s.timefn, 1) + s.run(blocking=False) # queue first 10 tasks + result = TaskResult.objects.first() + assert result.status == "notification_failed" @pytest.mark.django_db diff --git a/src/sensor/migration_settings.py b/src/sensor/migration_settings.py index 05dabed1..b63e0a95 100644 --- a/src/sensor/migration_settings.py +++ b/src/sensor/migration_settings.py @@ -372,7 +372,6 @@ CALLBACK_SSL_VERIFICATION = env.bool("CALLBACK_SSL_VERIFICATION", default=True) -# OAuth Password Flow Authentication CALLBACK_AUTHENTICATION = env("CALLBACK_AUTHENTICATION", default="") CERTS_DIR = path.join(CONFIG_DIR, "certs") diff --git a/src/sensor/runtime_settings.py b/src/sensor/runtime_settings.py index 4cf5854e..22dcaf75 100644 --- a/src/sensor/runtime_settings.py +++ b/src/sensor/runtime_settings.py @@ -372,7 +372,6 @@ CALLBACK_SSL_VERIFICATION = env.bool("CALLBACK_SSL_VERIFICATION", default=True) -# OAuth Password Flow Authentication CALLBACK_AUTHENTICATION = env("CALLBACK_AUTHENTICATION", default="") CERTS_DIR = path.join(CONFIG_DIR, "certs") diff --git a/src/sensor/settings.py b/src/sensor/settings.py index e799bf65..c6fbc477 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -373,7 +373,6 @@ CALLBACK_SSL_VERIFICATION = env.bool("CALLBACK_SSL_VERIFICATION", default=True) -# OAuth Password Flow Authentication CALLBACK_AUTHENTICATION = env("CALLBACK_AUTHENTICATION", default="") CERTS_DIR = path.join(CONFIG_DIR, "certs") diff --git a/src/sensor/tests/certificate_auth_client.py b/src/sensor/tests/certificate_auth_client.py index e8fca0a5..098555b4 100644 --- a/src/sensor/tests/certificate_auth_client.py +++ b/src/sensor/tests/certificate_auth_client.py @@ -21,32 +21,41 @@ def get_kwargs(self, extra): return kwargs def get(self, path, data=None, follow=False, secure=False, **extra): + assert secure return super().get(path, data, follow, secure, **self.get_kwargs(extra)) def post(self, path, data=None, content_type=MULTIPART_CONTENT, follow=False, secure=False, **extra): + assert secure return super().post(path, data, content_type, follow, secure, **self.get_kwargs(extra)) def head(self, path, data=None, follow=False, secure=False, **extra): - pass + assert secure + return super().head(path, data, follow, secure, **self.get_kwargs(extra)) def options(self, path, data='', content_type='application/octet-stream', follow=False, secure=False, **extra): - pass + assert secure + return super().options(self, path, data, content_type, follow, secure, **self.get_kwargs(extra)) def put(self, path, data='', content_type='application/octet-stream', follow=False, secure=False, **extra): + assert secure return super().put(path, data, content_type, follow, secure, **self.get_kwargs(extra)) def patch(self, path, data='', content_type='application/octet-stream', follow=False, secure=False, **extra): - pass + assert secure + return super().patch(path, data, content_type, follow, secure, **self.get_kwargs(extra)) def delete(self, path, data='', content_type='application/octet-stream', follow=False, secure=False, **extra): + assert secure return super().delete(path, data, content_type, follow, secure, **self.get_kwargs(extra)) def trace(self, path, follow=False, secure=False, **extra): - pass + assert secure + return super().trace(path, follow, secure, **self.get_kwargs(extra)) def login(self, **credentials): if cert_auth_enabled: assert "username" in credentials self.username = credentials["username"] + return True else: - super().login(**credentials) + return super().login(**credentials) diff --git a/src/sensor/tests/test_api_docs.py b/src/sensor/tests/test_api_docs.py index 5f73b507..8594bdce 100644 --- a/src/sensor/tests/test_api_docs.py +++ b/src/sensor/tests/test_api_docs.py @@ -4,6 +4,7 @@ from rest_framework.reverse import reverse from sensor import V1, settings +from sensor.tests.utils import HTTPS_KWARG def test_api_docs_up_to_date(admin_client): @@ -16,7 +17,7 @@ def test_api_docs_up_to_date(admin_client): return True schema_url = reverse("api_schema", kwargs=V1) + "?format=openapi" - response = admin_client.get(schema_url) + response = admin_client.get(schema_url, **HTTPS_KWARG) with open(settings.OPENAPI_FILE, "w+") as openapi_file: openapi_json = json.loads(response.content) diff --git a/src/sensor/tests/utils.py b/src/sensor/tests/utils.py index b2de0c51..39f84203 100644 --- a/src/sensor/tests/utils.py +++ b/src/sensor/tests/utils.py @@ -1,6 +1,6 @@ from rest_framework import status -HTTPS_KWARG = {"wsgi.url_scheme": "https"} +HTTPS_KWARG = {"wsgi.url_scheme": "https", "secure": True} def validate_response(response, expected_code=None): diff --git a/src/status/fixtures/greyhound.json b/src/status/fixtures/greyhound.json deleted file mode 100644 index 0ba0d957..00000000 --- a/src/status/fixtures/greyhound.json +++ /dev/null @@ -1,12 +0,0 @@ -[ -{ - "model": "status.location", - "pk": 1, - "fields": { - "active": true, - "description": "DOC Boulder Labs Bldg 1 3420", - "latitude": "39.994793", - "longitude": "-105.262078" - } -} -] diff --git a/src/tasks/tests/test_overview_view.py b/src/tasks/tests/test_overview_view.py index bf15b80e..0bdc99b8 100644 --- a/src/tasks/tests/test_overview_view.py +++ b/src/tasks/tests/test_overview_view.py @@ -31,10 +31,10 @@ def test_admin_get_overview(admin_client): assert overview["schedule_entry"] # is non-empty string -def test_user_delete_overview_not_allowed(admin_client): +def test_user_delete_overview_not_allowed(user_client): url = reverse_results_overview() - response = admin_client.delete(url, **HTTPS_KWARG) - assert validate_response(response, status.HTTP_405_METHOD_NOT_ALLOWED) + response = user_client.delete(url, **HTTPS_KWARG) + assert validate_response(response, status.HTTP_403_FORBIDDEN) def test_admin_delete_overview_not_allowed(admin_client): diff --git a/src/test_utils/task_test_utils.py b/src/test_utils/task_test_utils.py index 2f8534ce..78ce02c2 100644 --- a/src/test_utils/task_test_utils.py +++ b/src/test_utils/task_test_utils.py @@ -187,6 +187,7 @@ def update_result_detail(client, schedule_entry_name, task_id, new_acquisition): "data": json.dumps(new_acquisition), "content_type": "application/json", "wsgi.url_scheme": "https", + "secure": True } return client.put(url, **kwargs) diff --git a/src/tox.ini b/src/tox.ini index 3ee4d73b..8dcf7ff1 100644 --- a/src/tox.ini +++ b/src/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py38,py39,py310 +envlist = py38,py39,py310,cert skip_missing_interpreters = True skipsdist = True From 1846c2ca5889e3d779eaf083ab2f028b990565d3 Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Fri, 27 Jan 2023 14:56:49 -0700 Subject: [PATCH 012/255] update openapi security definitions, autoformatting, settings comment --- docs/openapi.json | 13 +++- src/authentication/auth.py | 15 ++-- src/authentication/tests/test_cert_auth.py | 10 ++- src/authentication/tests/test_token_auth.py | 2 +- src/conftest.py | 2 + src/scheduler/scheduler.py | 2 +- src/scheduler/tests/test_scheduler.py | 9 ++- src/sensor/migration_settings.py | 69 +++++++++--------- src/sensor/runtime_settings.py | 69 +++++++++--------- src/sensor/settings.py | 69 +++++++++--------- src/sensor/tests/certificate_auth_client.py | 77 +++++++++++++++++---- src/sensor/tests/utils.py | 2 + src/test_utils/task_test_utils.py | 2 +- 13 files changed, 207 insertions(+), 134 deletions(-) diff --git a/docs/openapi.json b/docs/openapi.json index 270b63cc..602ce87d 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -14,7 +14,7 @@ }, "host": "testserver", "schemes": [ - "http" + "https" ], "basePath": "/api", "consumes": [ @@ -24,14 +24,21 @@ "application/json" ], "securityDefinitions": { + "cert": { + "type": "cert", + "description": "Certificate based mutual TLS authentication. AUTHENTICATION must be set to 'CERT'. This is done by the client verifying the server certificate and the server verifying the client certificate. The client certificate Common Name (CN) should contain the username of a user that exists in the database. Client certificate verification is handled by NGINX. For more information, see https://www.rfc-editor.org/rfc/rfc5246." + }, "token": { "type": "apiKey", - "description": "Tokens are automatically generated for all users. You can view yours by going to your User Details view in the browsable API at `/api/v1/users/me` and looking for the `auth_token` key. New user accounts do not initially have a password and so can not log in to the browsable API. To set a password for a user (for testing purposes), an admin can do that in the Sensor Configuration Portal, but only the account's token should be stored and used for general purpose API access. Example cURL call: `curl -kLsS -H \"Authorization: Token 529c30e6e04b3b546f2e073e879b75fdfa147c15\" https://localhost/api/v1`", + "description": "Tokens are automatically generated for all users. You can view yours by going to your User Details view in the browsable API at `/api/v1/users/me` and looking for the `auth_token` key. New user accounts do not initially have a password and so can not log in to the browsable API. To set a password for a user (for testing purposes), an admin can do that in the Sensor Configuration Portal, but only the account's token should be stored and used for general purpose API access. Example cURL call: `curl -kLsS -H \"Authorization: Token 529c30e6e04b3b546f2e073e879b75fdfa147c15\" https://localhost/api/v1`. AUTHENTICATION should be set to 'TOKEN'", "name": "Token", "in": "header" } }, "security": [ + { + "cert": [] + }, { "token": [] } @@ -1941,4 +1948,4 @@ } } } -} +} \ No newline at end of file diff --git a/src/authentication/auth.py b/src/authentication/auth.py index 9d13e833..448a9d30 100644 --- a/src/authentication/auth.py +++ b/src/authentication/auth.py @@ -1,5 +1,6 @@ import logging import re + from django.conf import settings from django.contrib.auth import get_user_model from rest_framework import authentication, exceptions @@ -7,12 +8,12 @@ logger = logging.getLogger(__name__) token_auth_enabled = ( - "rest_framework.authentication.TokenAuthentication" - in settings.REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"] + "rest_framework.authentication.TokenAuthentication" + in settings.REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"] ) certificate_authentication_enabled = ( - "authentication.auth.CertificateAuthentication" - in settings.REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"] + "authentication.auth.CertificateAuthentication" + in settings.REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"] ) @@ -28,13 +29,15 @@ def authenticate(self, request): except user_model.DoesNotExist: raise exceptions.AuthenticationFailed("No matching username found!") except Exception: - raise exceptions.AuthenticationFailed("Error occurred during certificate authentication!") + raise exceptions.AuthenticationFailed( + "Error occurred during certificate authentication!" + ) return user, None return None, None def get_cn_from_dn(cert_dn): - p = re.compile("CN=(.*?)(?:,|\+|$)") + p = re.compile(r"CN=(.*?)(?:,|\+|$)") match = p.search(cert_dn) if not match: raise Exception("No CN found in certificate!") diff --git a/src/authentication/tests/test_cert_auth.py b/src/authentication/tests/test_cert_auth.py index 36312fa2..a041cccc 100644 --- a/src/authentication/tests/test_cert_auth.py +++ b/src/authentication/tests/test_cert_auth.py @@ -51,13 +51,17 @@ def test_no_client_cert_unauthorized_no_dn(live_server): @pytest.mark.django_db def test_client_cert_accepted(live_server, admin_user): client = RequestsClient() - response = client.get(f"{live_server.url}", headers=get_requests_ssl_dn_header("admin")) + response = client.get( + f"{live_server.url}", headers=get_requests_ssl_dn_header("admin") + ) assert response.status_code == 200 def test_mismatching_user_forbidden(live_server, admin_user): client = RequestsClient() - response = client.get(f"{live_server.url}", headers=get_requests_ssl_dn_header("user")) + response = client.get( + f"{live_server.url}", headers=get_requests_ssl_dn_header("user") + ) assert response.status_code == 403 assert "No matching username found!" in response.json()["detail"] @@ -174,4 +178,4 @@ def test_empty_dn_unauthorized(live_server, admin_user): "X-Ssl-Client-Dn": "", } response = client.get(f"{live_server.url}", headers=headers) - assert response.status_code == 403 \ No newline at end of file + assert response.status_code == 403 diff --git a/src/authentication/tests/test_token_auth.py b/src/authentication/tests/test_token_auth.py index f8f1d48b..e0948f74 100644 --- a/src/authentication/tests/test_token_auth.py +++ b/src/authentication/tests/test_token_auth.py @@ -161,7 +161,7 @@ def test_user_cannot_view_user_detail(settings, live_server, user_client, user): response = user_client.get( f"{live_server.url}{user_detail}", headers={"Authorization": f"Token {user.auth_token.key}"}, - **HTTPS_KWARG + **HTTPS_KWARG, ) assert response.status_code == 403 diff --git a/src/conftest.py b/src/conftest.py index acdc0b75..9897bcdc 100644 --- a/src/conftest.py +++ b/src/conftest.py @@ -67,6 +67,7 @@ def user_client(db, user): return client + @pytest.fixture def admin_client(db, admin_user): """A Django test client logged in as an admin user""" @@ -75,6 +76,7 @@ def admin_client(db, admin_user): return client + @pytest.fixture def alt_user(db): """A normal user.""" diff --git a/src/scheduler/scheduler.py b/src/scheduler/scheduler.py index 2b012397..332761f8 100644 --- a/src/scheduler/scheduler.py +++ b/src/scheduler/scheduler.py @@ -188,7 +188,7 @@ def _finalize_task_result(self, started, finished, status, detail): data=json.dumps(result_json), headers=headers, verify=verify_ssl, - cert=settings.PATH_TO_CLIENT_CERT + cert=settings.PATH_TO_CLIENT_CERT, ) self._callback_response_handler(response, tr) else: diff --git a/src/scheduler/tests/test_scheduler.py b/src/scheduler/tests/test_scheduler.py index 8d73d586..195708d1 100644 --- a/src/scheduler/tests/test_scheduler.py +++ b/src/scheduler/tests/test_scheduler.py @@ -420,8 +420,13 @@ def test_notification_failed_status_unknown_host(test_scheduler): entry = create_entry("t", 1, 1, 100, 5, "logger", callback_url) entry.save() token = entry.owner.auth_token - m.post(callback_url, request_headers={"Authorization": "Token " + str(token)}, text='Not Found', status_code=404) - + m.post( + callback_url, + request_headers={"Authorization": "Token " + str(token)}, + text="Not Found", + status_code=404, + ) + entry.refresh_from_db() print("entry = " + entry.name) s = test_scheduler diff --git a/src/sensor/migration_settings.py b/src/sensor/migration_settings.py index b63e0a95..7b658e05 100644 --- a/src/sensor/migration_settings.py +++ b/src/sensor/migration_settings.py @@ -252,45 +252,44 @@ # https://drf-yasg.readthedocs.io/en/stable/settings.html SWAGGER_SETTINGS = { - "SECURITY_DEFINITIONS": {}, + "SECURITY_DEFINITIONS": { + "cert": { + "type": "cert", + "description": ( + "Certificate based mutual TLS authentication. " + "AUTHENTICATION must be set to 'CERT'. " + "This is done by the client verifying the server certificate and the server verifying the client certificate. " + "The client certificate Common Name (CN) should contain the username of a user that exists in the database. " + "Client certificate verification is handled by NGINX. " + "For more information, see https://www.rfc-editor.org/rfc/rfc5246." + ), + }, + "token": { + "type": "apiKey", + "description": ( + "Tokens are automatically generated for all users. You can " + "view yours by going to your User Details view in the " + "browsable API at `/api/v1/users/me` and looking for the " + "`auth_token` key. New user accounts do not initially " + "have a password and so can not log in to the browsable API. " + "To set a password for a user (for testing purposes), an " + "admin can do that in the Sensor Configuration Portal, but " + "only the account's token should be stored and used for " + "general purpose API access. " + 'Example cURL call: `curl -kLsS -H "Authorization: Token' + ' 529c30e6e04b3b546f2e073e879b75fdfa147c15" ' + "https://localhost/api/v1`. " + "AUTHENTICATION should be set to 'TOKEN'" + ), + "name": "Token", + "in": "header", + } + }, "APIS_SORTER": "alpha", "OPERATIONS_SORTER": "method", "VALIDATOR_URL": None, } -if AUTHENTICATION == "JWT": - SWAGGER_SETTINGS["SECURITY_DEFINITIONS"]["oAuth2JWT"] = { - "type": "oauth2", - "description": ( - "OAuth2 authentication using resource owner password flow." - "This is done by verifing JWT bearer tokens signed with RS256 algorithm." - "The JWT_PUBLIC_KEY_FILE setting controls the public key used for signature verification." - "Only authorizes users who have an authority matching the REQUIRED_ROLE setting." - "For more information, see https://tools.ietf.org/html/rfc6749#section-4.3." - ), - "flows": {"password": {"scopes": {}}}, # scopes are not used - } -else: - SWAGGER_SETTINGS["SECURITY_DEFINITIONS"]["token"] = { - "type": "apiKey", - "description": ( - "Tokens are automatically generated for all users. You can " - "view yours by going to your User Details view in the " - "browsable API at `/api/v1/users/me` and looking for the " - "`auth_token` key. New user accounts do not initially " - "have a password and so can not log in to the browsable API. " - "To set a password for a user (for testing purposes), an " - "admin can do that in the Sensor Configuration Portal, but " - "only the account's token should be stored and used for " - "general purpose API access. " - 'Example cURL call: `curl -kLsS -H "Authorization: Token' - ' 529c30e6e04b3b546f2e073e879b75fdfa147c15" ' - "https://localhost/api/v1`" - ), - "name": "Token", - "in": "header", - } - # Database # https://docs.djangoproject.com/en/1.11/ref/settings/#databases @@ -379,7 +378,7 @@ PATH_TO_CLIENT_CERT = env("PATH_TO_CLIENT_CERT", default="") if PATH_TO_CLIENT_CERT != "": PATH_TO_CLIENT_CERT = path.join(CERTS_DIR, PATH_TO_CLIENT_CERT) -# Trusted Certificate Authority certificate to verify authserver and callback URL server certificate +# Trusted Certificate Authority certificate to verify callback URL server certificate PATH_TO_VERIFY_CERT = env("PATH_TO_VERIFY_CERT", default="") if PATH_TO_VERIFY_CERT != "": PATH_TO_VERIFY_CERT = path.join(CERTS_DIR, PATH_TO_VERIFY_CERT) diff --git a/src/sensor/runtime_settings.py b/src/sensor/runtime_settings.py index 22dcaf75..9f9dc89d 100644 --- a/src/sensor/runtime_settings.py +++ b/src/sensor/runtime_settings.py @@ -252,45 +252,44 @@ # https://drf-yasg.readthedocs.io/en/stable/settings.html SWAGGER_SETTINGS = { - "SECURITY_DEFINITIONS": {}, + "SECURITY_DEFINITIONS": { + "cert": { + "type": "cert", + "description": ( + "Certificate based mutual TLS authentication. " + "AUTHENTICATION must be set to 'CERT'. " + "This is done by the client verifying the server certificate and the server verifying the client certificate. " + "The client certificate Common Name (CN) should contain the username of a user that exists in the database. " + "Client certificate verification is handled by NGINX. " + "For more information, see https://www.rfc-editor.org/rfc/rfc5246." + ), + }, + "token": { + "type": "apiKey", + "description": ( + "Tokens are automatically generated for all users. You can " + "view yours by going to your User Details view in the " + "browsable API at `/api/v1/users/me` and looking for the " + "`auth_token` key. New user accounts do not initially " + "have a password and so can not log in to the browsable API. " + "To set a password for a user (for testing purposes), an " + "admin can do that in the Sensor Configuration Portal, but " + "only the account's token should be stored and used for " + "general purpose API access. " + 'Example cURL call: `curl -kLsS -H "Authorization: Token' + ' 529c30e6e04b3b546f2e073e879b75fdfa147c15" ' + "https://localhost/api/v1`. " + "AUTHENTICATION should be set to 'TOKEN'" + ), + "name": "Token", + "in": "header", + } + }, "APIS_SORTER": "alpha", "OPERATIONS_SORTER": "method", "VALIDATOR_URL": None, } -if AUTHENTICATION == "JWT": - SWAGGER_SETTINGS["SECURITY_DEFINITIONS"]["oAuth2JWT"] = { - "type": "oauth2", - "description": ( - "OAuth2 authentication using resource owner password flow." - "This is done by verifing JWT bearer tokens signed with RS256 algorithm." - "The JWT_PUBLIC_KEY_FILE setting controls the public key used for signature verification." - "Only authorizes users who have an authority matching the REQUIRED_ROLE setting." - "For more information, see https://tools.ietf.org/html/rfc6749#section-4.3." - ), - "flows": {"password": {"scopes": {}}}, # scopes are not used - } -else: - SWAGGER_SETTINGS["SECURITY_DEFINITIONS"]["token"] = { - "type": "apiKey", - "description": ( - "Tokens are automatically generated for all users. You can " - "view yours by going to your User Details view in the " - "browsable API at `/api/v1/users/me` and looking for the " - "`auth_token` key. New user accounts do not initially " - "have a password and so can not log in to the browsable API. " - "To set a password for a user (for testing purposes), an " - "admin can do that in the Sensor Configuration Portal, but " - "only the account's token should be stored and used for " - "general purpose API access. " - 'Example cURL call: `curl -kLsS -H "Authorization: Token' - ' 529c30e6e04b3b546f2e073e879b75fdfa147c15" ' - "https://localhost/api/v1`" - ), - "name": "Token", - "in": "header", - } - # Database # https://docs.djangoproject.com/en/1.11/ref/settings/#databases @@ -379,7 +378,7 @@ PATH_TO_CLIENT_CERT = env("PATH_TO_CLIENT_CERT", default="") if PATH_TO_CLIENT_CERT != "": PATH_TO_CLIENT_CERT = path.join(CERTS_DIR, PATH_TO_CLIENT_CERT) -# Trusted Certificate Authority certificate to verify authserver and callback URL server certificate +# Trusted Certificate Authority certificate to verify callback URL server certificate PATH_TO_VERIFY_CERT = env("PATH_TO_VERIFY_CERT", default="") if PATH_TO_VERIFY_CERT != "": PATH_TO_VERIFY_CERT = path.join(CERTS_DIR, PATH_TO_VERIFY_CERT) diff --git a/src/sensor/settings.py b/src/sensor/settings.py index c6fbc477..25c3549b 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -253,45 +253,44 @@ # https://drf-yasg.readthedocs.io/en/stable/settings.html SWAGGER_SETTINGS = { - "SECURITY_DEFINITIONS": {}, + "SECURITY_DEFINITIONS": { + "cert": { + "type": "cert", + "description": ( + "Certificate based mutual TLS authentication. " + "AUTHENTICATION must be set to 'CERT'. " + "This is done by the client verifying the server certificate and the server verifying the client certificate. " + "The client certificate Common Name (CN) should contain the username of a user that exists in the database. " + "Client certificate verification is handled by NGINX. " + "For more information, see https://www.rfc-editor.org/rfc/rfc5246." + ), + }, + "token": { + "type": "apiKey", + "description": ( + "Tokens are automatically generated for all users. You can " + "view yours by going to your User Details view in the " + "browsable API at `/api/v1/users/me` and looking for the " + "`auth_token` key. New user accounts do not initially " + "have a password and so can not log in to the browsable API. " + "To set a password for a user (for testing purposes), an " + "admin can do that in the Sensor Configuration Portal, but " + "only the account's token should be stored and used for " + "general purpose API access. " + 'Example cURL call: `curl -kLsS -H "Authorization: Token' + ' 529c30e6e04b3b546f2e073e879b75fdfa147c15" ' + "https://localhost/api/v1`. " + "AUTHENTICATION should be set to 'TOKEN'" + ), + "name": "Token", + "in": "header", + } + }, "APIS_SORTER": "alpha", "OPERATIONS_SORTER": "method", "VALIDATOR_URL": None, } -if AUTHENTICATION == "JWT": - SWAGGER_SETTINGS["SECURITY_DEFINITIONS"]["oAuth2JWT"] = { - "type": "oauth2", - "description": ( - "OAuth2 authentication using resource owner password flow." - "This is done by verifing JWT bearer tokens signed with RS256 algorithm." - "The JWT_PUBLIC_KEY_FILE setting controls the public key used for signature verification." - "Only authorizes users who have an authority matching the REQUIRED_ROLE setting." - "For more information, see https://tools.ietf.org/html/rfc6749#section-4.3." - ), - "flows": {"password": {"scopes": {}}}, # scopes are not used - } -else: - SWAGGER_SETTINGS["SECURITY_DEFINITIONS"]["token"] = { - "type": "apiKey", - "description": ( - "Tokens are automatically generated for all users. You can " - "view yours by going to your User Details view in the " - "browsable API at `/api/v1/users/me` and looking for the " - "`auth_token` key. New user accounts do not initially " - "have a password and so can not log in to the browsable API. " - "To set a password for a user (for testing purposes), an " - "admin can do that in the Sensor Configuration Portal, but " - "only the account's token should be stored and used for " - "general purpose API access. " - 'Example cURL call: `curl -kLsS -H "Authorization: Token' - ' 529c30e6e04b3b546f2e073e879b75fdfa147c15" ' - "https://localhost/api/v1`" - ), - "name": "Token", - "in": "header", - } - # Database # https://docs.djangoproject.com/en/1.11/ref/settings/#databases @@ -380,7 +379,7 @@ PATH_TO_CLIENT_CERT = env("PATH_TO_CLIENT_CERT", default="") if PATH_TO_CLIENT_CERT != "": PATH_TO_CLIENT_CERT = path.join(CERTS_DIR, PATH_TO_CLIENT_CERT) -# Trusted Certificate Authority certificate to verify authserver and callback URL server certificate +# Trusted Certificate Authority certificate to verify callback URL server certificate PATH_TO_VERIFY_CERT = env("PATH_TO_VERIFY_CERT", default="") if PATH_TO_VERIFY_CERT != "": PATH_TO_VERIFY_CERT = path.join(CERTS_DIR, PATH_TO_VERIFY_CERT) diff --git a/src/sensor/tests/certificate_auth_client.py b/src/sensor/tests/certificate_auth_client.py index 098555b4..fabc0592 100644 --- a/src/sensor/tests/certificate_auth_client.py +++ b/src/sensor/tests/certificate_auth_client.py @@ -1,15 +1,18 @@ -from django.test.client import Client, MULTIPART_CONTENT from django.conf import settings from django.core.serializers.json import DjangoJSONEncoder +from django.test.client import MULTIPART_CONTENT, Client from sensor.tests.utils import get_http_request_ssl_dn_header cert_auth_enabled = settings.AUTHENTICATION == "CERT" + class CertificateAuthClient(Client): """Adds SSL DN header if certificate authentication is being used""" - def __init__(self, enforce_csrf_checks=False, json_encoder=DjangoJSONEncoder, **defaults) -> None: + def __init__( + self, enforce_csrf_checks=False, json_encoder=DjangoJSONEncoder, **defaults + ) -> None: super().__init__(enforce_csrf_checks, json_encoder=json_encoder, **defaults) self.username = "" @@ -24,29 +27,79 @@ def get(self, path, data=None, follow=False, secure=False, **extra): assert secure return super().get(path, data, follow, secure, **self.get_kwargs(extra)) - def post(self, path, data=None, content_type=MULTIPART_CONTENT, follow=False, secure=False, **extra): + def post( + self, + path, + data=None, + content_type=MULTIPART_CONTENT, + follow=False, + secure=False, + **extra + ): assert secure - return super().post(path, data, content_type, follow, secure, **self.get_kwargs(extra)) + return super().post( + path, data, content_type, follow, secure, **self.get_kwargs(extra) + ) def head(self, path, data=None, follow=False, secure=False, **extra): assert secure return super().head(path, data, follow, secure, **self.get_kwargs(extra)) - def options(self, path, data='', content_type='application/octet-stream', follow=False, secure=False, **extra): + def options( + self, + path, + data="", + content_type="application/octet-stream", + follow=False, + secure=False, + **extra + ): assert secure - return super().options(self, path, data, content_type, follow, secure, **self.get_kwargs(extra)) + return super().options( + self, path, data, content_type, follow, secure, **self.get_kwargs(extra) + ) - def put(self, path, data='', content_type='application/octet-stream', follow=False, secure=False, **extra): + def put( + self, + path, + data="", + content_type="application/octet-stream", + follow=False, + secure=False, + **extra + ): assert secure - return super().put(path, data, content_type, follow, secure, **self.get_kwargs(extra)) + return super().put( + path, data, content_type, follow, secure, **self.get_kwargs(extra) + ) - def patch(self, path, data='', content_type='application/octet-stream', follow=False, secure=False, **extra): + def patch( + self, + path, + data="", + content_type="application/octet-stream", + follow=False, + secure=False, + **extra + ): assert secure - return super().patch(path, data, content_type, follow, secure, **self.get_kwargs(extra)) + return super().patch( + path, data, content_type, follow, secure, **self.get_kwargs(extra) + ) - def delete(self, path, data='', content_type='application/octet-stream', follow=False, secure=False, **extra): + def delete( + self, + path, + data="", + content_type="application/octet-stream", + follow=False, + secure=False, + **extra + ): assert secure - return super().delete(path, data, content_type, follow, secure, **self.get_kwargs(extra)) + return super().delete( + path, data, content_type, follow, secure, **self.get_kwargs(extra) + ) def trace(self, path, follow=False, secure=False, **extra): assert secure diff --git a/src/sensor/tests/utils.py b/src/sensor/tests/utils.py index 39f84203..aee48d73 100644 --- a/src/sensor/tests/utils.py +++ b/src/sensor/tests/utils.py @@ -15,11 +15,13 @@ def validate_response(response, expected_code=None): rjson = response.json() return rjson + def get_requests_ssl_dn_header(common_name): return { "X-Ssl-Client-Dn": f"C=TC,ST=test_state,L=test_locality,O=test_org,OU=test_ou,CN={common_name}", } + def get_http_request_ssl_dn_header(common_name): return { "HTTP_X-SSL-CLIENT-DN": f"C=TC,ST=test_state,L=test_locality,O=test_org,OU=test_ou,CN={common_name}", diff --git a/src/test_utils/task_test_utils.py b/src/test_utils/task_test_utils.py index 78ce02c2..d5f73077 100644 --- a/src/test_utils/task_test_utils.py +++ b/src/test_utils/task_test_utils.py @@ -187,7 +187,7 @@ def update_result_detail(client, schedule_entry_name, task_id, new_acquisition): "data": json.dumps(new_acquisition), "content_type": "application/json", "wsgi.url_scheme": "https", - "secure": True + "secure": True, } return client.put(url, **kwargs) From 6b21fc623a31d371dbda5731cf4f10588a4ca800 Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Fri, 27 Jan 2023 15:02:50 -0700 Subject: [PATCH 013/255] remove oauth/jwt from isort; autoformat settings, openapi.json --- docs/openapi.json | 2 +- src/.isort.cfg | 2 +- src/sensor/migration_settings.py | 2 +- src/sensor/runtime_settings.py | 2 +- src/sensor/settings.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/openapi.json b/docs/openapi.json index 602ce87d..fe9fd1ce 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -1948,4 +1948,4 @@ } } } -} \ No newline at end of file +} diff --git a/src/.isort.cfg b/src/.isort.cfg index d4c96609..29e00ce5 100644 --- a/src/.isort.cfg +++ b/src/.isort.cfg @@ -4,4 +4,4 @@ include_trailing_comma=True force_grid_wrap=0 use_parentheses=True line_length=88 -known_third_party=cryptography,django,drf_yasg,environs,its_preselector,jsonfield,jwt,numpy,oauthlib,pytest,requests,requests_mock,requests_oauthlib,rest_framework,scos_actions,sigmf +known_third_party=cryptography,django,drf_yasg,environs,its_preselector,jsonfield,numpy,pytest,requests,requests_mock,rest_framework,scos_actions,sigmf diff --git a/src/sensor/migration_settings.py b/src/sensor/migration_settings.py index 7b658e05..5b39d8a3 100644 --- a/src/sensor/migration_settings.py +++ b/src/sensor/migration_settings.py @@ -283,7 +283,7 @@ ), "name": "Token", "in": "header", - } + }, }, "APIS_SORTER": "alpha", "OPERATIONS_SORTER": "method", diff --git a/src/sensor/runtime_settings.py b/src/sensor/runtime_settings.py index 9f9dc89d..6159c571 100644 --- a/src/sensor/runtime_settings.py +++ b/src/sensor/runtime_settings.py @@ -283,7 +283,7 @@ ), "name": "Token", "in": "header", - } + }, }, "APIS_SORTER": "alpha", "OPERATIONS_SORTER": "method", diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 25c3549b..a328c24a 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -284,7 +284,7 @@ ), "name": "Token", "in": "header", - } + }, }, "APIS_SORTER": "alpha", "OPERATIONS_SORTER": "method", From 89a3b4884fe3d6c86b542591a65230ceda2b09f0 Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Mon, 30 Jan 2023 13:17:40 -0700 Subject: [PATCH 014/255] remove old oauth params --- docker-compose.yml | 2 -- env.template | 3 --- 2 files changed, 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7b061cc3..6d36e758 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,8 +38,6 @@ services: - AUTHENTICATION - CALLBACK_AUTHENTICATION - CALLBACK_SSL_VERIFICATION - - CLIENT_ID - - CLIENT_SECRET - DEBUG - DOCKER_TAG - DOMAINS diff --git a/env.template b/env.template index 7550cc82..d7420f00 100644 --- a/env.template +++ b/env.template @@ -61,9 +61,6 @@ BASE_IMAGE=ghcr.io/ntia/scos-tekrsa/tekrsa_usb:0.2.1 # Set to CERT for certificate authentication CALLBACK_AUTHENTICATION=TOKEN -CLIENT_ID=sensor01.sms.internal -CLIENT_SECRET=sensor-secret - # Sensor certificate with private key used as client cert PATH_TO_CLIENT_CERT=sensor01.pem # Trusted Certificate Authority certificate to verify authserver and callback URL server certificate From 6d1a25696db9d3debb147eecb8c1b419cb1f5617 Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Tue, 31 Jan 2023 08:13:32 -0700 Subject: [PATCH 015/255] remove login url when using cert auth --- src/sensor/urls.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/sensor/urls.py b/src/sensor/urls.py index b04a8d00..88b7cb7c 100644 --- a/src/sensor/urls.py +++ b/src/sensor/urls.py @@ -23,12 +23,13 @@ from django.views.generic import RedirectView from rest_framework.urlpatterns import format_suffix_patterns -from . import settings +from django.conf import settings from .views import api_v1_root, schema_view # Matches api/v1, api/v2, etc... API_PREFIX = r"^api/(?Pv[0-9]+)/" DEFAULT_API_VERSION = settings.REST_FRAMEWORK["DEFAULT_VERSION"] +AUTHENTICATION = settings.AUTHENTICATION api_urlpatterns = format_suffix_patterns( ( @@ -57,5 +58,7 @@ path("admin/", admin.site.urls), path("api/", RedirectView.as_view(url="/api/{}/".format(DEFAULT_API_VERSION))), re_path(API_PREFIX, include(api_urlpatterns)), - path("api/auth/", include("rest_framework.urls")), ] + +if AUTHENTICATION != "CERT": + urlpatterns.append(path("api/auth/", include("rest_framework.urls"))) From a5fd9de275fe79c08fb9e51bb7d10a43509ea33c Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Tue, 31 Jan 2023 12:57:20 -0700 Subject: [PATCH 016/255] update last_login, nginx check ssl_client_verify --- nginx/conf.template | 4 ++++ src/authentication/auth.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/nginx/conf.template b/nginx/conf.template index f39db8c7..e561e3b7 100644 --- a/nginx/conf.template +++ b/nginx/conf.template @@ -47,6 +47,10 @@ server { # Pass off requests to Gunicorn location @proxy_to_wsgi_server { + if ($ssl_client_verify != SUCCESS) { + return 403; + } + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto https; proxy_set_header Host $http_host; diff --git a/src/authentication/auth.py b/src/authentication/auth.py index 448a9d30..09507302 100644 --- a/src/authentication/auth.py +++ b/src/authentication/auth.py @@ -1,3 +1,4 @@ +import datetime import logging import re @@ -26,6 +27,8 @@ def authenticate(self, request): try: cn = get_cn_from_dn(cert_dn) user = user_model.objects.get(username=cn) + user.last_login = datetime.datetime.now() + user.save() except user_model.DoesNotExist: raise exceptions.AuthenticationFailed("No matching username found!") except Exception: From fcbc88ef9dc7d888bd8b04a97780e4e2abb129b5 Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Wed, 29 Mar 2023 10:08:02 -0600 Subject: [PATCH 017/255] fix merge issue for requirements-dev.in --- src/requirements-dev.in | 1 - 1 file changed, 1 deletion(-) diff --git a/src/requirements-dev.in b/src/requirements-dev.in index e8dba275..7f6d00c8 100644 --- a/src/requirements-dev.in +++ b/src/requirements-dev.in @@ -1,6 +1,5 @@ -rrequirements.txt -cryptography>=36.0, <39.0 pre-commit>=2.0, <3.0 pytest-cov>=3.0, <4.0 pytest-django>=4.0, <5.0 From bfcc86c7c2365fd3f75086a22d64c8838de10405 Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Wed, 5 Apr 2023 09:05:16 -0600 Subject: [PATCH 018/255] fix comment --- src/tasks/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tasks/views.py b/src/tasks/views.py index 330bba8f..56d63048 100644 --- a/src/tasks/views.py +++ b/src/tasks/views.py @@ -217,7 +217,7 @@ def build_sigmf_archive(fileobj, schedule_entry_name, acquisitions): raw_data = acq.data.read() data = fernet.decrypt(raw_data) del raw_data - tmpdata.write(data) # decrypted data will be stored on disk in tmp file + tmpdata.write(data) # decrypted data stored in file del data else: tmpdata.write(acq.data.read()) From 7409e937e4c07ccd4ee0158975f496d9c839e15f Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Mon, 10 Apr 2023 14:59:54 -0600 Subject: [PATCH 019/255] update authentication information in readme --- README.md | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index a59cf7c7..cc2aa3d7 100644 --- a/README.md +++ b/README.md @@ -384,10 +384,9 @@ or using Django Rest Framework Token Authentication. #### Django Rest Framework Token Authentication -This is the default authentication method. To enable Django Rest Framework -Authentication, make sure `AUTHENTICATION` is set to `TOKEN` in the environment file -(this will be enabled if `AUTHENTICATION` set to anything other -than `CERT`). +To enable Django Rest Framework token and session authentication, make sure +`AUTHENTICATION` is set to `TOKEN` in the environment file (this will be enabled if +`AUTHENTICATION` set to anything other than `CERT`). A token is automatically created for each user. Django Rest Framework Token Authentication will check that the token in the Authorization header ("Token " + @@ -395,7 +394,8 @@ token) matches a user's token. #### Certificate Authentication -To enable Certificate Authentication, set `AUTHENTICATION` to `CERT` in the environment +This is the default authentication method. To enable Certificate Authentication, make +sure `AUTHENTICATION` is set to `CERT` in the environment file. To authenticate, the client will need to send a trusted client certificate. The Common Name must match the username of a user in the database. @@ -504,11 +504,31 @@ or client.pfx when communicating with the API programmatically. ###### Configure scos-sensor -The Nginx web server can be set to require client certificates (mutual TLS). This can -optionally be enabled. To require client certificates, make sure `ssl_verify_client` is -set to `on` in the [Nginx configuration file](nginx/conf.template). If you -use OCSP, also uncomment `ssl_ocsp on;`. Additional configuration may be needed for -Nginx to check certificate revocation lists (CRL). +The Nginx web server is configured by default to require client certificates (mutual +TLS). To require client certificates, make sure `ssl_verify_client` is set to `on` in +the [Nginx configuration file](nginx/conf.template). Comment out this line or set to +`off` to disable client certificates. This can also be set to `optional` or +`optional_no_ca`, but if a client certificate is not provided, scos-sensor +`AUTHENTICATION` setting must be set to `TOKEN` which requires a token for the API or a +username and password for the browsable API. If you use OCSP, also uncomment +`ssl_ocsp on;`. Additional configuration may be needed for Nginx to check certificate +revocation lists (CRL). Adjust the other Nginx parameters, such as `ssl_verify_depth`, +as desired. See the +[Nginx documentation](https://nginx.org/en/docs/http/ngx_http_ssl_module.html) for +more information about configuring Nginx. + +To disable client certificate authentication, comment out the +following in [nginx/conf.template](nginx/conf.template): + +``` +ssl_client_certificate /etc/ssl/certs/ca.crt; +ssl_verify_client on; +ssl_ocsp on; +... + if ($ssl_client_verify != SUCCESS) { # under location @proxy_to_wsgi_server { + return 403; + } +``` Copy the server certificate and server private key (sensor01_combined.pem) to `scos-sensor/configs/certs`. Then set `SSL_CERT_PATH` and `SSL_KEY_PATH` (in the From 761bcdc2fa447cd87e99a4c5c98e8d45b160ab77 Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Mon, 17 Apr 2023 09:30:22 -0600 Subject: [PATCH 020/255] keep logout url for page timeout --- src/sensor/urls.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sensor/urls.py b/src/sensor/urls.py index 88b7cb7c..d44326cb 100644 --- a/src/sensor/urls.py +++ b/src/sensor/urls.py @@ -17,13 +17,13 @@ """ +from django.conf import settings from django.conf.urls.static import static from django.contrib import admin from django.urls import include, path, re_path from django.views.generic import RedirectView from rest_framework.urlpatterns import format_suffix_patterns -from django.conf import settings from .views import api_v1_root, schema_view # Matches api/v1, api/v2, etc... @@ -60,5 +60,5 @@ re_path(API_PREFIX, include(api_urlpatterns)), ] -if AUTHENTICATION != "CERT": - urlpatterns.append(path("api/auth/", include("rest_framework.urls"))) +# logout/login does not do anything if AUTHENTICATION is set to "CERT" +urlpatterns.append(path("api/auth/", include("rest_framework.urls"))) From 38d369f37f965aeb0ada99305719c64c7530adcd Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Wed, 27 Sep 2023 09:27:20 -0600 Subject: [PATCH 021/255] update dependencies, fix dependabot alert --- .pre-commit-config.yaml | 6 +- src/requirements-dev.txt | 160 ++++++++++++++++++++------------------- src/requirements.in | 1 + src/requirements.txt | 97 ++++++++++++------------ 4 files changed, 135 insertions(+), 129 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b33e7e19..02a8e5ce 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/asottile/pyupgrade - rev: v3.4.0 + rev: v3.13.0 hooks: - id: pyupgrade args: ["--py38-plus"] @@ -31,12 +31,12 @@ repos: types: [file, python] args: ["--profile", "black", "--filter-files", "--gitignore"] - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 23.9.1 hooks: - id: black types: [file, python] - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.35.0 + rev: v0.37.0 hooks: - id: markdownlint types: [file, markdown] diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index 4da1a068..110be9e4 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -16,21 +16,21 @@ aiosignal==1.3.1 # -r requirements.txt # aiohttp # ray -asgiref==3.6.0 +asgiref==3.7.2 # via # -r requirements.txt # django -async-timeout==4.0.2 +async-timeout==4.0.3 # via aiohttp -attrs==22.2.0 +attrs==23.1.0 # via # -r requirements.txt # aiohttp # jsonschema - # pytest + # referencing blessed==1.20.0 # via gpustat -cachetools==5.3.0 +cachetools==5.3.1 # via # google-auth # tox @@ -42,16 +42,16 @@ cffi==1.15.1 # via # -r requirements.txt # cryptography -cfgv==3.3.1 +cfgv==3.4.0 # via pre-commit -chardet==5.1.0 +chardet==5.2.0 # via tox -charset-normalizer==3.0.1 +charset-normalizer==3.2.0 # via # -r requirements.txt # aiohttp # requests -click==8.1.3 +click==8.1.7 # via # -r requirements.txt # ray @@ -59,16 +59,7 @@ colorama==0.4.6 # via tox colorful==0.5.5 # via ray -coreapi==2.3.3 - # via - # -r requirements.txt - # drf-yasg -coreschema==0.0.4 - # via - # -r requirements.txt - # coreapi - # drf-yasg -coverage[toml]==7.2.1 +coverage[toml]==7.3.1 # via pytest-cov cryptography==41.0.4 # via -r requirements.txt @@ -76,9 +67,9 @@ defusedxml==0.7.1 # via # -r requirements.txt # its-preselector -distlib==0.3.6 +distlib==0.3.7 # via virtualenv -django==3.2.20 +django==3.2.21 # via # -r requirements.txt # django-session-timeout @@ -92,7 +83,7 @@ djangorestframework==3.14.0 # via # -r requirements.txt # drf-yasg -drf-yasg==1.21.5 +drf-yasg==1.21.7 # via -r requirements.txt environs==9.5.0 # via @@ -101,39 +92,44 @@ environs==9.5.0 # scos-tekrsa exceptiongroup==1.1.3 # via pytest -filelock==3.9.0 +filelock==3.12.4 # via # -r requirements.txt # ray # tox # virtualenv -frozenlist==1.3.3 +frozenlist==1.4.0 # via # -r requirements.txt # aiohttp # aiosignal # ray -google-api-core==2.11.0 +google-api-core==2.12.0 # via opencensus -google-auth==2.17.3 +google-auth==2.23.1 # via google-api-core -googleapis-common-protos==1.59.0 +googleapis-common-protos==1.60.0 # via google-api-core -gpustat==1.1 +gpustat==1.1.1 # via ray -grpcio==1.51.3 +grpcio==1.58.0 # via # -r requirements.txt # ray gunicorn==20.1.0 # via -r requirements.txt -identify==2.5.18 +identify==2.5.29 # via pre-commit idna==3.4 # via # -r requirements.txt # requests # yarl +importlib-resources==6.1.0 + # via + # -r requirements.txt + # jsonschema + # jsonschema-specifications inflection==0.5.1 # via # -r requirements.txt @@ -144,33 +140,25 @@ its-preselector @ git+https://github.com/NTIA/Preselector@3.0.2 # via # -r requirements.txt # scos-actions -itypes==1.2.0 - # via - # -r requirements.txt - # coreapi -jinja2==3.1.2 - # via - # -r requirements.txt - # coreschema jsonfield==3.1.0 # via -r requirements.txt -jsonschema==3.2.0 +jsonschema==4.19.1 # via # -r requirements.txt # ray -markupsafe==2.1.2 +jsonschema-specifications==2023.7.1 # via # -r requirements.txt - # jinja2 -marshmallow==3.19.0 + # jsonschema +marshmallow==3.20.1 # via # -r requirements.txt # environs -msgpack==1.0.5 +msgpack==1.0.6 # via # -r requirements.txt # ray -msgspec==0.16.0 +msgspec==0.18.2 # via # -r requirements.txt # scos-actions @@ -178,13 +166,13 @@ multidict==6.0.4 # via # aiohttp # yarl -nodeenv==1.7.0 +nodeenv==1.8.0 # via pre-commit -numexpr==2.8.4 +numexpr==2.8.6 # via # -r requirements.txt # scos-actions -numpy==1.24.2 +numpy==1.24.4 # via # -r requirements.txt # numexpr @@ -193,13 +181,13 @@ numpy==1.24.2 # scos-actions # sigmf # tekrsa-api-wrap -nvidia-ml-py==11.525.112 +nvidia-ml-py==12.535.108 # via gpustat -opencensus==0.11.2 +opencensus==0.11.3 # via ray opencensus-context==0.1.3 # via opencensus -packaging==23.0 +packaging==23.1 # via # -r requirements.txt # drf-yasg @@ -208,19 +196,23 @@ packaging==23.0 # pytest # ray # tox -platformdirs==3.0.0 +pkgutil-resolve-name==1.3.10 + # via + # -r requirements.txt + # jsonschema +platformdirs==3.10.0 # via # tox # virtualenv -pluggy==1.0.0 +pluggy==1.3.0 # via # pytest # tox -pre-commit==3.3.3 +pre-commit==3.4.0 # via -r requirements-dev.in -prometheus-client==0.13.1 +prometheus-client==0.17.1 # via ray -protobuf==4.23.3 +protobuf==4.24.3 # via # -r requirements.txt # google-api-core @@ -231,7 +223,7 @@ psutil==5.9.5 # -r requirements.txt # gpustat # scos-actions -psycopg2-binary==2.9.5 +psycopg2-binary==2.9.7 # via -r requirements.txt py-spy==0.3.14 # via ray @@ -245,15 +237,11 @@ pycparser==2.21 # via # -r requirements.txt # cffi -pydantic==1.10.7 +pydantic==1.10.12 # via ray -pyproject-api==1.5.0 +pyproject-api==1.5.4 # via tox -pyrsistent==0.19.3 - # via - # -r requirements.txt - # jsonschema -pytest==7.2.1 +pytest==7.4.2 # via # pytest-cov # pytest-django @@ -265,11 +253,11 @@ python-dateutil==2.8.2 # via # -r requirements.txt # scos-actions -python-dotenv==0.21.1 +python-dotenv==1.0.0 # via # -r requirements.txt # environs -pytz==2022.7.1 +pytz==2023.3.post1 # via # -r requirements.txt # django @@ -278,29 +266,38 @@ pytz==2022.7.1 pyyaml==6.0.1 # via # -r requirements.txt + # drf-yasg # pre-commit # ray -ray[default]==2.6.3 +ray[default]==2.7.0 # via # -r requirements-dev.in # -r requirements.txt # scos-actions +referencing==0.30.2 + # via + # -r requirements.txt + # jsonschema + # jsonschema-specifications requests==2.31.0 # via # -r requirements.txt - # coreapi # google-api-core # its-preselector # ray # requests-mock -requests-mock==1.10.0 +requests-mock==1.11.0 # via -r requirements.txt +rpds-py==0.10.3 + # via + # -r requirements.txt + # jsonschema + # referencing rsa==4.9 # via google-auth -ruamel-yaml==0.17.21 +ruamel-yaml==0.17.32 # via # -r requirements.txt - # drf-yasg # scos-actions ruamel-yaml-clib==0.2.7 # via @@ -325,12 +322,10 @@ six==1.16.0 # -r requirements.txt # blessed # django-session-timeout - # google-auth - # jsonschema # python-dateutil # requests-mock # sigmf -smart-open==6.3.0 +smart-open==6.4.0 # via ray sqlparse==0.4.4 # via @@ -346,20 +341,23 @@ tomli==2.0.1 # pyproject-api # pytest # tox -tox==4.4.6 +tox==4.5.1.1 # via -r requirements-dev.in -typing-extensions==4.5.0 - # via pydantic +typing-extensions==4.8.0 + # via + # -r requirements.txt + # asgiref + # pydantic uritemplate==4.1.1 # via # -r requirements.txt - # coreapi # drf-yasg -urllib3==1.26.14 +urllib3==2.0.5 # via # -r requirements.txt + # google-auth # requests -virtualenv==20.20.0 +virtualenv==20.21.0 # via # pre-commit # ray @@ -368,6 +366,10 @@ wcwidth==0.2.6 # via blessed yarl==1.9.2 # via aiohttp +zipp==3.17.0 + # via + # -r requirements.txt + # importlib-resources # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/src/requirements.in b/src/requirements.in index 16e860ab..65197d54 100644 --- a/src/requirements.in +++ b/src/requirements.in @@ -17,3 +17,4 @@ scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@3.1.3 # higher minimum patch version than the dependencies which require them. # This is done to ensure the inclusion of specific security patches. pyyaml>=5.4.0 # CVE-2020-14343 +grpcio>=1.53.0 # CVE-2023-32732, CVE-2023-32731, CVE-2023-1428 \ No newline at end of file diff --git a/src/requirements.txt b/src/requirements.txt index 0a30f05f..ddf2a625 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -6,29 +6,25 @@ # aiosignal==1.3.1 # via ray -asgiref==3.6.0 +asgiref==3.7.2 # via django -attrs==22.2.0 - # via jsonschema +attrs==23.1.0 + # via + # jsonschema + # referencing certifi==2023.7.22 # via requests cffi==1.15.1 # via cryptography -charset-normalizer==3.0.1 +charset-normalizer==3.2.0 # via requests -click==8.1.3 +click==8.1.7 # via ray -coreapi==2.3.3 - # via drf-yasg -coreschema==0.0.4 - # via - # coreapi - # drf-yasg cryptography==41.0.4 # via -r requirements.in defusedxml==0.7.1 # via its-preselector -django==3.2.20 +django==3.2.21 # via # -r requirements.in # django-session-timeout @@ -42,50 +38,50 @@ djangorestframework==3.14.0 # via # -r requirements.in # drf-yasg -drf-yasg==1.21.5 +drf-yasg==1.21.7 # via -r requirements.in environs==9.5.0 # via # -r requirements.in # scos-actions # scos-tekrsa -filelock==3.9.0 +filelock==3.12.4 # via # -r requirements.in # ray -frozenlist==1.3.3 +frozenlist==1.4.0 # via # aiosignal # ray -grpcio==1.51.3 - # via ray +grpcio==1.58.0 + # via -r requirements.in gunicorn==20.1.0 # via -r requirements.in idna==3.4 # via requests +importlib-resources==6.1.0 + # via + # jsonschema + # jsonschema-specifications inflection==0.5.1 # via drf-yasg its-preselector @ git+https://github.com/NTIA/Preselector@3.0.2 # via scos-actions -itypes==1.2.0 - # via coreapi -jinja2==3.1.2 - # via coreschema jsonfield==3.1.0 # via -r requirements.in -jsonschema==3.2.0 +jsonschema==4.19.1 # via ray -markupsafe==2.1.2 - # via jinja2 -marshmallow==3.19.0 +jsonschema-specifications==2023.7.1 + # via jsonschema +marshmallow==3.20.1 # via environs -msgpack==1.0.5 +msgpack==1.0.6 # via ray -msgspec==0.16.0 +msgspec==0.18.2 # via scos-actions -numexpr==2.8.4 +numexpr==2.8.6 # via scos-actions -numpy==1.24.2 +numpy==1.24.4 # via # numexpr # ray @@ -93,27 +89,27 @@ numpy==1.24.2 # scos-actions # sigmf # tekrsa-api-wrap -packaging==23.0 +packaging==23.1 # via # -r requirements.in # drf-yasg # marshmallow # ray -protobuf==4.23.3 +pkgutil-resolve-name==1.3.10 + # via jsonschema +protobuf==4.24.3 # via ray psutil==5.9.5 # via scos-actions -psycopg2-binary==2.9.5 +psycopg2-binary==2.9.7 # via -r requirements.in pycparser==2.21 # via cffi -pyrsistent==0.19.3 - # via jsonschema python-dateutil==2.8.2 # via scos-actions -python-dotenv==0.21.1 +python-dotenv==1.0.0 # via environs -pytz==2022.7.1 +pytz==2023.3.post1 # via # django # djangorestframework @@ -121,21 +117,27 @@ pytz==2022.7.1 pyyaml==6.0.1 # via # -r requirements.in + # drf-yasg # ray -ray==2.6.3 +ray==2.7.0 # via scos-actions +referencing==0.30.2 + # via + # jsonschema + # jsonschema-specifications requests==2.31.0 # via - # coreapi # its-preselector # ray # requests-mock -requests-mock==1.10.0 +requests-mock==1.11.0 # via -r requirements.in -ruamel-yaml==0.17.21 +rpds-py==0.10.3 # via - # drf-yasg - # scos-actions + # jsonschema + # referencing +ruamel-yaml==0.17.32 + # via scos-actions ruamel-yaml-clib==0.2.7 # via ruamel-yaml scipy==1.10.1 @@ -151,7 +153,6 @@ sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive six==1.16.0 # via # django-session-timeout - # jsonschema # python-dateutil # requests-mock # sigmf @@ -159,12 +160,14 @@ sqlparse==0.4.4 # via django tekrsa-api-wrap==1.3.2 # via scos-tekrsa +typing-extensions==4.8.0 + # via asgiref uritemplate==4.1.1 - # via - # coreapi - # drf-yasg -urllib3==1.26.14 + # via drf-yasg +urllib3==2.0.5 # via requests +zipp==3.17.0 + # via importlib-resources # The following packages are considered to be unsafe in a requirements file: # setuptools From a04ea354be6dfa055a9925ca049f6f304d4bf5ad Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Wed, 27 Sep 2023 09:30:52 -0600 Subject: [PATCH 022/255] formatting updates --- README.md | 2 +- docs/openapi.json | 42 +++++++++++++++++----------------- src/requirements-dev.in | 2 +- src/requirements.in | 2 +- src/sensor/runtime_settings.py | 1 - 5 files changed, 24 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 4de7027c..51cffb9e 100644 --- a/README.md +++ b/README.md @@ -518,7 +518,7 @@ more information about configuring Nginx. To disable client certificate authentication, comment out the following in [nginx/conf.template](nginx/conf.template): -``` +```text ssl_client_certificate /etc/ssl/certs/ca.crt; ssl_verify_client on; ssl_ocsp on; diff --git a/docs/openapi.json b/docs/openapi.json index fe9fd1ce..feee0bb5 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -649,13 +649,13 @@ }, "parameters": [ { - "name": "format", + "name": "schedule_entry_name", "in": "path", "required": true, "type": "string" }, { - "name": "schedule_entry_name", + "name": "format", "in": "path", "required": true, "type": "string" @@ -759,12 +759,6 @@ ] }, "parameters": [ - { - "name": "format", - "in": "path", - "required": true, - "type": "string" - }, { "name": "schedule_entry_name", "in": "path", @@ -777,6 +771,12 @@ "description": "The id of the task relative to the result", "required": true, "type": "integer" + }, + { + "name": "format", + "in": "path", + "required": true, + "type": "string" } ] }, @@ -811,12 +811,6 @@ ] }, "parameters": [ - { - "name": "format", - "in": "path", - "required": true, - "type": "string" - }, { "name": "schedule_entry_name", "in": "path", @@ -829,6 +823,12 @@ "description": "The id of the task relative to the result", "required": true, "type": "integer" + }, + { + "name": "format", + "in": "path", + "required": true, + "type": "string" } ] }, @@ -918,13 +918,13 @@ }, "parameters": [ { - "name": "format", + "name": "schedule_entry_name", "in": "path", "required": true, "type": "string" }, { - "name": "schedule_entry_name", + "name": "format", "in": "path", "required": true, "type": "string" @@ -1577,17 +1577,17 @@ }, "parameters": [ { - "name": "format", + "name": "id", "in": "path", + "description": "A unique integer value identifying this user.", "required": true, - "type": "string" + "type": "integer" }, { - "name": "id", + "name": "format", "in": "path", - "description": "A unique integer value identifying this user.", "required": true, - "type": "integer" + "type": "string" } ] }, diff --git a/src/requirements-dev.in b/src/requirements-dev.in index cf2964ad..7a80f7f1 100644 --- a/src/requirements-dev.in +++ b/src/requirements-dev.in @@ -9,4 +9,4 @@ tox>=4.0,<5.0 # The following are sub-dependencies for which SCOS Sensor enforces a # higher minimum patch version than the dependencies which require them. # This is done to ensure the inclusion of specific security patches. -aiohttp>=3.8.5 # CVE-2023-37276 \ No newline at end of file +aiohttp>=3.8.5 # CVE-2023-37276 diff --git a/src/requirements.in b/src/requirements.in index 65197d54..7d9924e9 100644 --- a/src/requirements.in +++ b/src/requirements.in @@ -17,4 +17,4 @@ scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@3.1.3 # higher minimum patch version than the dependencies which require them. # This is done to ensure the inclusion of specific security patches. pyyaml>=5.4.0 # CVE-2020-14343 -grpcio>=1.53.0 # CVE-2023-32732, CVE-2023-32731, CVE-2023-1428 \ No newline at end of file +grpcio>=1.53.0 # CVE-2023-32732, CVE-2023-32731, CVE-2023-1428 diff --git a/src/sensor/runtime_settings.py b/src/sensor/runtime_settings.py index a427970c..d7c3f47b 100644 --- a/src/sensor/runtime_settings.py +++ b/src/sensor/runtime_settings.py @@ -39,7 +39,6 @@ SCOS_SENSOR_GIT_TAG = env("SCOS_SENSOR_GIT_TAG", default="Unknown") if not DOCKER_TAG or DOCKER_TAG == "latest": - VERSION_STRING = GIT_BRANCH else: VERSION_STRING = DOCKER_TAG From 457e38cad1a1a5e58fa0e82d7de0943eff9a2333 Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Tue, 24 Oct 2023 10:49:29 -0600 Subject: [PATCH 023/255] Update env.template remove SENTRY_DSN which is no longer used --- env.template | 3 --- 1 file changed, 3 deletions(-) diff --git a/env.template b/env.template index c6bc3abc..62b5275b 100644 --- a/env.template +++ b/env.template @@ -45,9 +45,6 @@ ADMIN_PASSWORD=password # `openssl rand -base64 12` POSTGRES_PASSWORD="$(python3 -c 'import secrets; import base64; print(base64.b64encode(secrets.token_bytes(32)).decode("utf-8"))')" -# Set to enable monitoring sensors with your sentry.io account -SENTRY_DSN= - if $DEBUG; then GUNICORN_LOG_LEVEL=debug RAY_record_ref_creation_sites=1 From 8850365f1a717cf3d9cb92886d16ae310a2c4bbe Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Wed, 1 Nov 2023 10:00:08 -0600 Subject: [PATCH 024/255] dependency updates --- src/requirements-dev.txt | 61 ++++++++++++++++++++-------------------- src/requirements.txt | 42 ++++++++++++++------------- 2 files changed, 53 insertions(+), 50 deletions(-) diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index 6baaba0a..b611a76b 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -4,7 +4,7 @@ # # pip-compile requirements-dev.in # -aiohttp==3.8.5 +aiohttp==3.8.6 # via # -r requirements-dev.in # aiohttp-cors @@ -30,7 +30,7 @@ attrs==23.1.0 # referencing blessed==1.20.0 # via gpustat -cachetools==5.3.1 +cachetools==5.3.2 # via # google-auth # tox @@ -38,7 +38,7 @@ certifi==2023.7.22 # via # -r requirements.txt # requests -cffi==1.15.1 +cffi==1.16.0 # via # -r requirements.txt # cryptography @@ -46,7 +46,7 @@ cfgv==3.4.0 # via pre-commit chardet==5.2.0 # via tox -charset-normalizer==3.2.0 +charset-normalizer==3.3.2 # via # -r requirements.txt # aiohttp @@ -59,9 +59,9 @@ colorama==0.4.6 # via tox colorful==0.5.5 # via ray -coverage[toml]==7.3.1 +coverage[toml]==7.3.2 # via pytest-cov -cryptography==41.0.4 +cryptography==41.0.5 # via -r requirements.txt defusedxml==0.7.1 # via @@ -69,7 +69,7 @@ defusedxml==0.7.1 # its-preselector distlib==0.3.7 # via virtualenv -django==3.2.21 +django==3.2.23 # via # -r requirements.txt # django-session-timeout @@ -92,7 +92,7 @@ environs==9.5.0 # scos-tekrsa exceptiongroup==1.1.3 # via pytest -filelock==3.12.4 +filelock==3.13.1 # via # -r requirements.txt # ray @@ -106,19 +106,19 @@ frozenlist==1.4.0 # ray google-api-core==2.12.0 # via opencensus -google-auth==2.23.1 +google-auth==2.23.4 # via google-api-core -googleapis-common-protos==1.60.0 +googleapis-common-protos==1.61.0 # via google-api-core gpustat==1.1.1 # via ray -grpcio==1.58.0 +grpcio==1.59.2 # via # -r requirements.txt # ray gunicorn==20.1.0 # via -r requirements.txt -identify==2.5.29 +identify==2.5.31 # via pre-commit idna==3.4 # via @@ -142,7 +142,7 @@ its-preselector @ git+https://github.com/NTIA/Preselector@3.0.2 # scos-actions jsonfield==3.1.0 # via -r requirements.txt -jsonschema==4.19.1 +jsonschema==4.19.2 # via # -r requirements.txt # ray @@ -154,11 +154,11 @@ marshmallow==3.20.1 # via # -r requirements.txt # environs -msgpack==1.0.6 +msgpack==1.0.7 # via # -r requirements.txt # ray -msgspec==0.18.2 +msgspec==0.18.4 # via # -r requirements.txt # scos-actions @@ -187,7 +187,7 @@ opencensus==0.11.3 # via ray opencensus-context==0.1.3 # via opencensus -packaging==23.1 +packaging==23.2 # via # -r requirements.txt # drf-yasg @@ -200,7 +200,7 @@ pkgutil-resolve-name==1.3.10 # via # -r requirements.txt # jsonschema -platformdirs==3.10.0 +platformdirs==3.11.0 # via # tox # virtualenv @@ -208,22 +208,22 @@ pluggy==1.3.0 # via # pytest # tox -pre-commit==3.4.0 +pre-commit==3.5.0 # via -r requirements-dev.in -prometheus-client==0.17.1 +prometheus-client==0.18.0 # via ray -protobuf==4.24.3 +protobuf==4.24.4 # via # -r requirements.txt # google-api-core # googleapis-common-protos # ray -psutil==5.9.5 +psutil==5.9.6 # via # -r requirements.txt # gpustat # scos-actions -psycopg2-binary==2.9.7 +psycopg2-binary==2.9.9 # via -r requirements.txt py-spy==0.3.14 # via ray @@ -237,17 +237,17 @@ pycparser==2.21 # via # -r requirements.txt # cffi -pydantic==1.10.12 +pydantic==1.10.13 # via ray pyproject-api==1.5.4 # via tox -pytest==7.4.2 +pytest==7.4.3 # via # pytest-cov # pytest-django pytest-cov==3.0.0 # via -r requirements-dev.in -pytest-django==4.5.2 +pytest-django==4.6.0 # via -r requirements-dev.in python-dateutil==2.8.2 # via @@ -269,7 +269,7 @@ pyyaml==6.0.1 # drf-yasg # pre-commit # ray -ray[default]==2.7.0 +ray[default]==2.7.1 # via # -r requirements-dev.in # -r requirements.txt @@ -288,14 +288,14 @@ requests==2.31.0 # requests-mock requests-mock==1.11.0 # via -r requirements.txt -rpds-py==0.10.3 +rpds-py==0.10.6 # via # -r requirements.txt # jsonschema # referencing rsa==4.9 # via google-auth -ruamel-yaml==0.17.32 +ruamel-yaml==0.18.3 # via # -r requirements.txt # scos-actions @@ -352,17 +352,16 @@ uritemplate==4.1.1 # via # -r requirements.txt # drf-yasg -urllib3==2.0.5 +urllib3==2.0.7 # via # -r requirements.txt - # google-auth # requests virtualenv==20.21.0 # via # pre-commit # ray # tox -wcwidth==0.2.6 +wcwidth==0.2.9 # via blessed yarl==1.9.2 # via aiohttp diff --git a/src/requirements.txt b/src/requirements.txt index 56c714e7..58b80ea9 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -14,17 +14,17 @@ attrs==23.1.0 # referencing certifi==2023.7.22 # via requests -cffi==1.15.1 +cffi==1.16.0 # via cryptography -charset-normalizer==3.2.0 +charset-normalizer==3.3.2 # via requests click==8.1.7 # via ray -cryptography==41.0.4 +cryptography==41.0.5 # via -r requirements.in defusedxml==0.7.1 # via its-preselector -django==3.2.21 +django==3.2.23 # via # -r requirements.in # django-session-timeout @@ -45,7 +45,7 @@ environs==9.5.0 # -r requirements.in # scos-actions # scos-tekrsa -filelock==3.12.4 +filelock==3.13.1 # via # -r requirements.in # ray @@ -53,7 +53,7 @@ frozenlist==1.4.0 # via # aiosignal # ray -grpcio==1.58.0 +grpcio==1.59.2 # via -r requirements.in gunicorn==20.1.0 # via -r requirements.in @@ -69,15 +69,15 @@ its-preselector @ git+https://github.com/NTIA/Preselector@3.0.2 # via scos-actions jsonfield==3.1.0 # via -r requirements.in -jsonschema==4.19.1 +jsonschema==4.19.2 # via ray jsonschema-specifications==2023.7.1 # via jsonschema marshmallow==3.20.1 # via environs -msgpack==1.0.6 +msgpack==1.0.7 # via ray -msgspec==0.18.2 +msgspec==0.18.4 # via scos-actions numexpr==2.8.6 # via scos-actions @@ -89,7 +89,7 @@ numpy==1.24.4 # scos-actions # sigmf # tekrsa-api-wrap -packaging==23.1 +packaging==23.2 # via # -r requirements.in # drf-yasg @@ -97,11 +97,11 @@ packaging==23.1 # ray pkgutil-resolve-name==1.3.10 # via jsonschema -protobuf==4.24.3 +protobuf==4.24.4 # via ray -psutil==5.9.5 +psutil==5.9.6 # via scos-actions -psycopg2-binary==2.9.7 +psycopg2-binary==2.9.9 # via -r requirements.in pycparser==2.21 # via cffi @@ -119,7 +119,7 @@ pyyaml==6.0.1 # -r requirements.in # drf-yasg # ray -ray==2.7.0 +ray==2.7.1 # via scos-actions referencing==0.30.2 # via @@ -132,10 +132,12 @@ requests==2.31.0 # requests-mock requests-mock==1.11.0 # via -r requirements.in -rpds-py==0.10.3 +rpds-py==0.10.6 # via - # drf-yasg - # scos-actions + # jsonschema + # referencing +ruamel-yaml==0.18.3 + # via scos-actions ruamel-yaml-clib==0.2.8 # via ruamel-yaml scipy==1.10.1 @@ -160,8 +162,10 @@ typing-extensions==4.8.0 # via asgiref uritemplate==4.1.1 # via drf-yasg -urllib3==2.0.5 - # via requests +urllib3==2.0.7 + # via + # -r requirements.in + # requests zipp==3.17.0 # via importlib-resources From 740c89a0a417f596313a89a0fc9a4a97a9e727c8 Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Mon, 20 Nov 2023 12:59:29 -0700 Subject: [PATCH 025/255] support multiple users in create_superuser, fix get username from cert, additional tests, update readme --- README.md | 93 +++++++++++++++------- docker-compose.yml | 2 + env.template | 13 +-- scripts/create_superuser.py | 54 ++++++++++--- src/authentication/auth.py | 8 +- src/authentication/tests/test_cert_auth.py | 20 +++++ src/conftest.py | 6 +- 7 files changed, 148 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 51cffb9e..e4e59022 100644 --- a/README.md +++ b/README.md @@ -197,10 +197,13 @@ actions. ## Overview of scos-sensor Repo Structure - configs: This folder is used to store the sensor_definition.json file. + - certs: CA, server, and client certificates. - docker: Contains the docker files used by scos-sensor. - docs: Documentation including the [documentation hosted on GitHub pages]( ) generated from the OpenAPI specification. +- drivers: Driver files for signal anaylzers. - entrypoints: Docker entrypoint scripts which are executed when starting a container. +- files: Folder where task results are stored. - gunicorn: Gunicorn configuration file. - nginx: Nginx configuration template and SSL certificates. - scripts: Various utility scripts. @@ -208,6 +211,7 @@ actions. - actions: Code to discover actions in plugins and to perform a simple logger action. - authentication: Code related to user authentication. - capabilities: Code used to generate capabilities endpoint. + - constants: Constants shared by the other source code folders. - handlers: Code to handle signals received from actions. - schedule: Schedule API endpoint for scheduling actions. - scheduler: Scheduler responsible for executing actions. @@ -217,9 +221,13 @@ actions. - status: Status endpoint. - tasks: Tasks endpoint used to display upcoming and completed tasks. - templates: HTML templates used by the browsable API. + - test_utils: Utility code used in tests. + - utils: Utility code shared by the other source code folders. - conftest.py: Used to configure pytest fixtures. - manage.py: Django’s command line tool for administrative tasks. - - requirements.txt and requirements-dev.txt: Python dependencies. + - requirements.in and requirements-dev.in: Direct Python dependencies. + - requirements.txt and requirements-dev.txt: Python dependencies including transitive + dependencies. - tox.ini: Used to configure tox. - docker-compose.yml: Used by Docker Compose to create services from containers. This is needed to run scos-sensor. @@ -299,11 +307,19 @@ environment (env) file is created from the env.template file. These settings can be set in the environment file or set directly in docker-compose.yml. Here are the settings in the environment file: +- ADDITIONAL_USER_NAMES: Comma separated list of additional admin usernames. +- ADDITIONAL_USER_PASSWORD: Password for additional admin users. +- ADMIN_NAME: Username for the admin user. - ADMIN_EMAIL: Email used to generate admin user. Change in production. - ADMIN_PASSWORD: Password used to generate admin user. Change in production. +- AUTHENTICATION: Authentication method used for scos-sensor. Supports `TOKEN` or + `CERT`. - BASE_IMAGE: Base docker image used to build the API container. +- CALLBACK_AUTHENTICATION: Sets how to authenticate to the callback URL. Supports + `TOKEN` or `CERT`. - CALLBACK_SSL_VERIFICATION: Set to “true” in production environment. If false, the SSL certificate validation will be ignored when posting results to the callback URL. +- CALLBACK_TIMEOUT: The timeout for the requests sent to the callback URL. - DEBUG: Django debug mode. Set to False in production. - DOCKER_TAG: Always set to “latest” to install newest version of docker containers. - DOMAINS: A space separated list of domain names. Used to generate [ALLOWED_HOSTS]( @@ -321,9 +337,14 @@ settings in the environment file: results. Defaults to 85%. This disk usage detected by scos-sensor (using the Python `shutil.disk_usage` function) may not match the usage reported by the Linux `df` command. +- PATH_TO_CLIENT_CERT: Path to file containing certificate and private key used as + client certificate when CALLBACK_AUTHENTICATION is `CERT`. +- PATH_TO_VERIFY_CERT: Trusted CA certificate to verify callback URL server + certificate. - POSTGRES_PASSWORD: Sets password for the Postgres database for the “postgres” user. Change in production. The env.template file sets to a randomly generated value. - REPO_ROOT: Root folder of the repository. Should be correctly set by default. +- SCOS_SENSOR_GIT_TAG: The scos-sensor branch name. - SECRET_KEY: Used by Django to provide cryptographic signing. Change to a unique, unpredictable value. See . The env.template @@ -332,6 +353,8 @@ settings in the environment file: scos-sensor repository with a valid certificate in production. - SSL_KEY_PATH: Path to server SSL private key. Use the private key for your valid certificate in production. +- SSL_CA_PATH: Path to a CA certificate used to verify scos-sensor client + certificate(s) when authentication is set to CERT. ### Sensor Definition File @@ -388,7 +411,8 @@ To enable Django Rest Framework token and session authentication, make sure A token is automatically created for each user. Django Rest Framework Token Authentication will check that the token in the Authorization header ("Token " + -token) matches a user's token. +token) matches a user's token. Login session authentication with username and password +is used for the browsable API. #### Certificate Authentication @@ -413,16 +437,16 @@ Below instructions adapted from This is the SSL certificate used for the scos-sensor web server and is always required. -To be able to sign server-side and client-side certificates, we need to create our own -self-signed root CA certificate first. The command will prompt you to enter a -password and the values for the CA subject. +To be able to sign server-side and client-side certificates in this example, we need to +create our own self-signed root CA certificate first. The command will prompt you to +enter a password and the values for the CA subject. ```bash openssl req -x509 -sha512 -days 365 -newkey rsa:4096 -keyout scostestca.key -out scostestca.pem ``` -Generate a host certificate signing request. Replace the values in square brackets in the -subject for the server certificate. +Generate a host certificate signing request. Replace the values in square brackets in +the subject for the server certificate. ```bash openssl req -new -newkey rsa:4096 -keyout sensor01.key -out sensor01.csr -subj "/C=[2 letter country code]/ST=[state or province]/L=[locality]/O=[organization]/OU=[organizational unit]/CN=[common name]" @@ -430,20 +454,20 @@ openssl req -new -newkey rsa:4096 -keyout sensor01.key -out sensor01.csr -subj " Before we proceed with openssl, we need to create a configuration file -- sensor01.ext. It'll store some additional parameters needed when signing the certificate. Adjust the -settings, especially DNS names and IP addresses, in the below example for your sensor: +settings, especially DNS names, in the below example for your sensor. For more +information and to customize your certificate, see the X.509 standard +[here](https://www.rfc-editor.org/rfc/rfc5280). ```text -authorityKeyIdentifier=keyid,issuer:always +authorityKeyIdentifier=keyid basicConstraints=CA:FALSE subjectAltName = @alt_names subjectKeyIdentifier = hash keyUsage = critical, digitalSignature, keyEncipherment -extendedKeyUsage = serverAuth, clientAuth +extendedKeyUsage = serverAuth, # add , clientAuth to use as client SSL cert (2-way SSL) [alt_names] -DNS.1 = sensor01.domain -DNS.2 = localhost -IP.1 = xxx.xxx.xxx.xxx -IP.2 = 127.0.0.1 +DNS.1 = localhost +# Add additional DNS names as needed, e.g. DNS.2, DNS.3, etc ``` Sign the host certificate. @@ -467,7 +491,8 @@ cat sensor01_decrypted.key sensor01.pem > sensor01_combined.pem ##### Client Certificate This certificate is required for using the sensor with mutual TLS certificate -authentication. +authentication (2 way SSL, AUTHENTICATION=CERT). This example uses the same self-signed +CA used for creating the example scos-sensor server certificate. Replace the brackets with the information specific to your user and organization. @@ -480,8 +505,8 @@ Create client.ext with the following: ```text basicConstraints = CA:FALSE subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer -keyUsage = digitalSignature +authorityKeyIdentifier = keyid +keyUsage = critical, digitalSignature extendedKeyUsage = clientAuth ``` @@ -539,21 +564,22 @@ mutual TLS, also copy the CA certificate to the same directory. Then, set If you are using client certificates, use client.pfx to connect to the browsable API by importing this certificate into your browser. -For callback functionality with a server that uses certificate authentication, set -`PATH_TO_CLIENT_CERT` and `PATH_TO_VERIFY_CERT`, both relative to configs/certs. -Depending on the configuration of the callback URL server and the authorization server, -the sensor server certificate could be used as a client certificate by setting -`PATH_TO_CLIENT_CERT` to the path of sensor01_combined.pem relative to configs/certs. -Also the CA used to verify the client certificate could potentially be used to verify -the callback URL server certificate by setting `PATH_TO_VERIFY_CERT` to the same file -as used for `SSL_CA_PATH` (scostestca.pem). - #### Permissions and Users The API requires the user to be a superuser. New users created using the API initially do not have superuser access. However, an admin can mark a user as a superuser in the Sensor Configuration Portal. +When scos-sensor starts, an admin user is created using the ADMIN_NAME, ADMIN_EMAIL and +ADMIN_PASSWORD environment variables. The ADMIN_NAME is the username for the admin +user. Additional admin users can be created using the ADDITIONAL_USER_NAMES and +ADDITIONAL_USER_PASSWORD environment variables. ADDITIONAL_USER_NAMES is a comma +separated list. ADDITIONAL_USER_PASSWORD is a single password used for each additional +admin user. If ADDITIONAL_USER_PASSWORD is not specified, the additional users will +be created with an unusable password, which is sufficient if only using certificates +or tokens to authenticate. However, a password is required to access the Sensor +Configuration Portal. + ### Callback URL Authentication Certificate and token authentication are supported for authenticating against the @@ -577,7 +603,7 @@ used. #### Certificate -Certificate authetnication (mutual TLS) is supported for callback URL authentication. +Certificate authentication (mutual TLS) is supported for callback URL authentication. The following settings in the environment file are used to configure certificate authentication for the callback URL. @@ -590,6 +616,19 @@ authentication for the callback URL. https://requests.readthedocs.io/en/master/user/advanced/#ca-certificates) will be used. +Set `PATH_TO_CLIENT_CERT` and `PATH_TO_VERIFY_CERT` relative to configs/certs. +Depending on the configuration of the callback URL server, the scos-sensor server +certificate could be used as a client certificate (if created with clientAuth extended +key usage) by setting `PATH_TO_CLIENT_CERT` to the same value as `SSL_CERT_PATH` +if the private key is bundled with the certificate. Also +the CA used to verify the scos-sensor client certificate(s) could potentially be used +to verify the callback URL server certificate by setting `PATH_TO_VERIFY_CERT` to the +same file as used for `SSL_CA_PATH`. This would require the callback URL server +certificate to be issued by the same CA as the scos-sensor client certficate(s) or have +the callback URL server's CA cert bundled with the scos-sensor client CA cert. Make +sure to consider the security implications of these configurations and settings, +especially using the same files for multiple settings. + ### Data File Encryption The data files are encrypted on disk by default using Cryptography Fernet module. The diff --git a/docker-compose.yml b/docker-compose.yml index 1963ec82..4abfd274 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,6 +35,8 @@ services: - ADMIN_NAME - ADMIN_EMAIL - ADMIN_PASSWORD + - ADDITIONAL_USER_NAMES + - ADDITIONAL_USER_PASSWORD - AUTHENTICATION - CALLBACK_AUTHENTICATION - CALLBACK_SSL_VERIFICATION diff --git a/env.template b/env.template index 52919680..23fb714a 100644 --- a/env.template +++ b/env.template @@ -39,6 +39,9 @@ GIT_BRANCH="git:$(git rev-parse --abbrev-ref HEAD)@$(git rev-parse --short HEAD) # If admin user email and password set, admin user will be generated. ADMIN_EMAIL="admin@example.com" ADMIN_PASSWORD=password +ADMIN_NAME=Admin +ADDITIONAL_USER_NAMES="" # comma separated +ADDITIONAL_USER_PASSWORD="" # Session password for Postgres. Username is "postgres". # SECURITY WARNING: generate unique key with something like @@ -65,11 +68,11 @@ BASE_IMAGE=ghcr.io/ntia/scos-tekrsa/tekrsa_usb:0.2.3 CALLBACK_AUTHENTICATION=TOKEN CALLBACK_TIMEOUT=2 -# Sensor certificate with private key used as client cert +# Sensor certificate with private key used as client cert for callback URL +# Paths relative to configs/certs PATH_TO_CLIENT_CERT=sensor01.pem -# Trusted Certificate Authority certificate to verify authserver and callback URL server certificate +# Trusted Certificate Authority certificate to verify callback URL server certificate PATH_TO_VERIFY_CERT=scos_test_ca.crt -# Path relative to configs/certs -# set to CERT to enable certificate authentication + +# set to CERT to enable scos-sensor certificate authentication AUTHENTICATION=CERT -ADMIN_NAME=Admin diff --git a/scripts/create_superuser.py b/scripts/create_superuser.py index dea05b30..89529023 100755 --- a/scripts/create_superuser.py +++ b/scripts/create_superuser.py @@ -13,24 +13,60 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "sensor.settings") django.setup() +UserModel = get_user_model() + + +def add_user(username, password, email=None): + try: + admin_user = UserModel._default_manager.get(username=username) + if email: + admin_user.email = email + admin_user.set_password(password) + print("Reset admin account password and email from environment") + except UserModel.DoesNotExist: + UserModel._default_manager.create_superuser(username, email, password) + print("Created admin account with password and email from environment") + try: password = os.environ["ADMIN_PASSWORD"] print("Retreived admin password from environment variable ADMIN_PASSWORD") email = os.environ["ADMIN_EMAIL"] print("Retreived admin email from environment variable ADMIN_EMAIL") + username = os.environ["ADMIN_NAME"] + print("Retreived admin name from environment variable ADMIN_NAME") + add_user(username, password, email) except KeyError: print("Not on a managed sensor, so not auto-generating admin account.") print("You can add an admin later with `./manage.py createsuperuser`") sys.exit(0) -UserModel = get_user_model() -username = os.environ["ADMIN_NAME"] +additional_user_names = "" +additional_user_password = "" try: - admin_user = UserModel._default_manager.get(username=username) - admin_user.email = email - admin_user.set_password(password) - print("Reset admin account password and email from environment") -except UserModel.DoesNotExist: - UserModel._default_manager.create_superuser(username, email, password) - print("Created admin account with password and email from environment") + additional_user_names = os.environ["ADDITIONAL_USER_NAMES"] + print( + "Retreived additional user names from environment variable ADDITIONAL_USER_NAMES" + ) + if ( + "ADDITIONAL_USER_PASSWORD" in os.environ + and os.environ["ADDITIONAL_USER_PASSWORD"] + ): + additional_user_password = os.environ["ADDITIONAL_USER_PASSWORD"] + else: + # user will have unusable password + # https://docs.djangoproject.com/en/3.2/ref/contrib/auth/#django.contrib.auth.models.UserManager.create_user + additional_user_password = None + print( + "Retreived additional user password from environment variable ADDITIONAL_USER_PASSWORD" + ) +except KeyError: + print("Not creating any additonal users.") + + +if additional_user_names != "" and additional_user_password != "": + if "," in additional_user_names: + for additional_user_name in additional_user_names.split(","): + add_user(additional_user_name, additional_user_password) + else: + add_user(additional_user_names, additional_user_password) diff --git a/src/authentication/auth.py b/src/authentication/auth.py index 09507302..3af870e9 100644 --- a/src/authentication/auth.py +++ b/src/authentication/auth.py @@ -40,10 +40,10 @@ def authenticate(self, request): def get_cn_from_dn(cert_dn): - p = re.compile(r"CN=(.*?)(?:,|\+|$)") + p = re.compile(r"CN=[a-zA-Z0-9\s.]*") match = p.search(cert_dn) if not match: raise Exception("No CN found in certificate!") - uid_raw = match.group() - uid = uid_raw.split("=")[1].rstrip(",") - return uid + cn = match.group() + cn = cn[3:] + return cn diff --git a/src/authentication/tests/test_cert_auth.py b/src/authentication/tests/test_cert_auth.py index a041cccc..9d98cb07 100644 --- a/src/authentication/tests/test_cert_auth.py +++ b/src/authentication/tests/test_cert_auth.py @@ -179,3 +179,23 @@ def test_empty_dn_unauthorized(live_server, admin_user): } response = client.get(f"{live_server.url}", headers=headers) assert response.status_code == 403 + + +@pytest.mark.django_db +def test_dn_with_uid(live_server, admin_user): + client = RequestsClient() + headers = { + "X-Ssl-Client-Dn": f"C=TC,ST=test_state,L=test_locality,O=test_org,OU=test_ou,CN={admin_user.username}+UID=11111", + } + response = client.get(f"{live_server.url}", headers=headers) + assert response.status_code == 200 + + +@pytest.mark.django_db +def test_dn_cn_space_reverse_order(live_server, alt_admin_user): + client = RequestsClient() + headers = { + "X-Ssl-Client-Dn": f"CN={alt_admin_user.username}+UID=111111,OU=test_ou,O=test_org,L=test_locality,ST=test_state,C=TC", + } + response = client.get(f"{live_server.url}", headers=headers) + assert response.status_code == 200 diff --git a/src/conftest.py b/src/conftest.py index 9897bcdc..58b70608 100644 --- a/src/conftest.py +++ b/src/conftest.py @@ -115,15 +115,15 @@ def alt_admin_user(db, django_user_model, django_username_field): username_field = django_username_field try: - user = UserModel._default_manager.get(**{username_field: "alt_admin"}) + user = UserModel._default_manager.get(**{username_field: "ALT ADMIN"}) except UserModel.DoesNotExist: extra_fields = {} if username_field != "username": - extra_fields[username_field] = "alt_admin" + extra_fields[username_field] = "ALT ADMIN" user = UserModel._default_manager.create_superuser( - "alt_admin", "alt_admin@example.com", "password", **extra_fields + "ALT ADMIN", "alt_admin@example.com", "password", **extra_fields ) return user From 0f9e3b3a9593f03ac803d411bc42f445361acd34 Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Mon, 20 Nov 2023 14:37:18 -0700 Subject: [PATCH 026/255] update min versions for dependabot alerts --- src/requirements-dev.in | 2 +- src/requirements.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/requirements-dev.in b/src/requirements-dev.in index 7a80f7f1..bbcf7391 100644 --- a/src/requirements-dev.in +++ b/src/requirements-dev.in @@ -9,4 +9,4 @@ tox>=4.0,<5.0 # The following are sub-dependencies for which SCOS Sensor enforces a # higher minimum patch version than the dependencies which require them. # This is done to ensure the inclusion of specific security patches. -aiohttp>=3.8.5 # CVE-2023-37276 +aiohttp>=3.8.6 # CVE-2023-37276, CVE-2023-47627 diff --git a/src/requirements.in b/src/requirements.in index f0448d63..e49746e2 100644 --- a/src/requirements.in +++ b/src/requirements.in @@ -1,5 +1,5 @@ cryptography>=41.0.4 -django>=3.2.20, <4.0 +django>=3.2.23, <4.0 djangorestframework>=3.0, <4.0 django-session-timeout>=0.1, <1.0 drf-yasg>=1.0, <2.0 From 4de3426965e0a7ea3f6a4baba9ae82d89bb75bd5 Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Wed, 29 Nov 2023 15:35:55 -0700 Subject: [PATCH 027/255] set defaults for token auth --- env.template | 2 +- nginx/conf.template | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/env.template b/env.template index 23fb714a..2f6429c4 100644 --- a/env.template +++ b/env.template @@ -75,4 +75,4 @@ PATH_TO_CLIENT_CERT=sensor01.pem PATH_TO_VERIFY_CERT=scos_test_ca.crt # set to CERT to enable scos-sensor certificate authentication -AUTHENTICATION=CERT +AUTHENTICATION=TOKEN diff --git a/nginx/conf.template b/nginx/conf.template index 8ca7aadb..085f8843 100644 --- a/nginx/conf.template +++ b/nginx/conf.template @@ -33,8 +33,8 @@ server { ssl_certificate /etc/ssl/certs/ssl-cert.pem; ssl_certificate_key /etc/ssl/private/ssl-cert.key; ssl_protocols TLSv1.2 TLSv1.3; - ssl_client_certificate /etc/ssl/certs/ca.crt; - ssl_verify_client on; + # ssl_client_certificate /etc/ssl/certs/ca.crt; + # ssl_verify_client on; # ssl_ocsp on; # Enable OCSP validation ssl_verify_depth 4; # path for static files @@ -49,9 +49,9 @@ server { # Pass off requests to Gunicorn location @proxy_to_wsgi_server { - if ($ssl_client_verify != SUCCESS) { - return 403; - } + # if ($ssl_client_verify != SUCCESS) { + # return 403; + # } proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto https; From 2c11834d44eba51f2d0d46e125c9153c3fe463d5 Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Fri, 1 Dec 2023 08:44:34 -0700 Subject: [PATCH 028/255] update requirements --- src/requirements-dev.txt | 61 ++++++++++++++++------------------------ src/requirements.txt | 32 +++++++++------------ 2 files changed, 38 insertions(+), 55 deletions(-) diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index 05b3a01e..6d983ae4 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -4,7 +4,7 @@ # # pip-compile requirements-dev.in # -aiohttp==3.9.0 +aiohttp==3.9.1 # via # -r requirements-dev.in # aiohttp-cors @@ -34,7 +34,7 @@ cachetools==5.3.2 # via # google-auth # tox -certifi==2023.7.22 +certifi==2023.11.17 # via # -r requirements.txt # requests @@ -58,20 +58,9 @@ colorama==0.4.6 # via tox colorful==0.5.5 # via ray -coreapi==2.3.3 - # via - # -r requirements.txt - # drf-yasg -coreschema==0.0.4 - # via - # -r requirements.txt - # coreapi - # drf-yasg coverage[toml]==7.3.2 - # via - # coverage - # pytest-cov -cryptography==41.0.6 + # via pytest-cov +cryptography==41.0.7 # via -r requirements.txt defusedxml==0.7.1 # via @@ -100,7 +89,7 @@ environs==9.5.0 # -r requirements.txt # scos-actions # scos-tekrsa -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 # via pytest filelock==3.13.1 # via @@ -114,28 +103,28 @@ frozenlist==1.4.0 # aiohttp # aiosignal # ray -google-api-core==2.12.0 +google-api-core==2.14.0 # via opencensus -google-auth==2.23.4 +google-auth==2.24.0 # via google-api-core googleapis-common-protos==1.61.0 # via google-api-core gpustat==1.1.1 # via ray -grpcio==1.59.2 +grpcio==1.59.3 # via # -r requirements.txt # ray gunicorn==20.1.0 # via -r requirements.txt -identify==2.5.31 +identify==2.5.32 # via pre-commit -idna==3.4 +idna==3.6 # via # -r requirements.txt # requests # yarl -importlib-resources==6.1.0 +importlib-resources==6.1.1 # via # -r requirements.txt # jsonschema @@ -152,11 +141,11 @@ its-preselector @ git+https://github.com/NTIA/Preselector@3.0.2 # scos-actions jsonfield==3.1.0 # via -r requirements.txt -jsonschema==4.19.2 +jsonschema==4.20.0 # via # -r requirements.txt # ray -jsonschema-specifications==2023.7.1 +jsonschema-specifications==2023.11.2 # via # -r requirements.txt # jsonschema @@ -191,7 +180,7 @@ numpy==1.24.4 # scos-actions # sigmf # tekrsa-api-wrap -nvidia-ml-py==12.535.108 +nvidia-ml-py==12.535.133 # via gpustat opencensus==0.11.3 # via ray @@ -220,9 +209,9 @@ pluggy==1.3.0 # tox pre-commit==3.5.0 # via -r requirements-dev.in -prometheus-client==0.18.0 +prometheus-client==0.19.0 # via ray -protobuf==4.24.4 +protobuf==4.25.1 # via # -r requirements.txt # google-api-core @@ -237,7 +226,7 @@ psycopg2-binary==2.9.9 # via -r requirements.txt py-spy==0.3.14 # via ray -pyasn1==0.5.0 +pyasn1==0.5.1 # via # pyasn1-modules # rsa @@ -257,7 +246,7 @@ pytest==7.4.3 # pytest-django pytest-cov==3.0.0 # via -r requirements-dev.in -pytest-django==4.6.0 +pytest-django==4.7.0 # via -r requirements-dev.in python-dateutil==2.8.2 # via @@ -279,12 +268,12 @@ pyyaml==6.0.1 # drf-yasg # pre-commit # ray -ray[default]==2.7.1 +ray[default]==2.8.1 # via # -r requirements-dev.in # -r requirements.txt # scos-actions -referencing==0.30.2 +referencing==0.31.1 # via # -r requirements.txt # jsonschema @@ -298,14 +287,14 @@ requests==2.31.0 # requests-mock requests-mock==1.11.0 # via -r requirements.txt -rpds-py==0.10.6 +rpds-py==0.13.2 # via # -r requirements.txt # jsonschema # referencing rsa==4.9 # via google-auth -ruamel-yaml==0.18.3 +ruamel-yaml==0.18.5 # via # -r requirements.txt # scos-actions @@ -362,7 +351,7 @@ uritemplate==4.1.1 # via # -r requirements.txt # drf-yasg -urllib3==2.0.7 +urllib3==2.1.0 # via # -r requirements.txt # requests @@ -371,9 +360,9 @@ virtualenv==20.21.0 # pre-commit # ray # tox -wcwidth==0.2.9 +wcwidth==0.2.12 # via blessed -yarl==1.9.2 +yarl==1.9.3 # via aiohttp zipp==3.17.0 # via diff --git a/src/requirements.txt b/src/requirements.txt index 7359a545..191be082 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -12,7 +12,7 @@ attrs==23.1.0 # via # jsonschema # referencing -certifi==2023.7.22 +certifi==2023.11.17 # via requests cffi==1.16.0 # via cryptography @@ -20,13 +20,7 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via ray -coreapi==2.3.3 - # via drf-yasg -coreschema==0.0.4 - # via - # coreapi - # drf-yasg -cryptography==41.0.6 +cryptography==41.0.7 # via -r requirements.in defusedxml==0.7.1 # via its-preselector @@ -59,13 +53,13 @@ frozenlist==1.4.0 # via # aiosignal # ray -grpcio==1.59.2 +grpcio==1.59.3 # via -r requirements.in gunicorn==20.1.0 # via -r requirements.in -idna==3.4 +idna==3.6 # via requests -importlib-resources==6.1.0 +importlib-resources==6.1.1 # via # jsonschema # jsonschema-specifications @@ -75,9 +69,9 @@ its-preselector @ git+https://github.com/NTIA/Preselector@3.0.2 # via scos-actions jsonfield==3.1.0 # via -r requirements.in -jsonschema==4.19.2 +jsonschema==4.20.0 # via ray -jsonschema-specifications==2023.7.1 +jsonschema-specifications==2023.11.2 # via jsonschema marshmallow==3.20.1 # via environs @@ -103,7 +97,7 @@ packaging==23.2 # ray pkgutil-resolve-name==1.3.10 # via jsonschema -protobuf==4.24.4 +protobuf==4.25.1 # via ray psutil==5.9.6 # via scos-actions @@ -125,9 +119,9 @@ pyyaml==6.0.1 # -r requirements.in # drf-yasg # ray -ray==2.7.1 +ray==2.8.1 # via scos-actions -referencing==0.30.2 +referencing==0.31.1 # via # jsonschema # jsonschema-specifications @@ -138,11 +132,11 @@ requests==2.31.0 # requests-mock requests-mock==1.11.0 # via -r requirements.in -rpds-py==0.10.6 +rpds-py==0.13.2 # via # jsonschema # referencing -ruamel-yaml==0.18.3 +ruamel-yaml==0.18.5 # via scos-actions ruamel-yaml-clib==0.2.8 # via ruamel-yaml @@ -168,7 +162,7 @@ typing-extensions==4.8.0 # via asgiref uritemplate==4.1.1 # via drf-yasg -urllib3==2.0.7 +urllib3==2.1.0 # via # -r requirements.in # requests From e33d47917d1be9590ecaf1daf92911ceddffbcc2 Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Fri, 1 Dec 2023 13:24:10 -0700 Subject: [PATCH 029/255] revert ray version --- src/requirements-dev.txt | 2 +- src/requirements.txt | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index 6d983ae4..42e98189 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -268,7 +268,7 @@ pyyaml==6.0.1 # drf-yasg # pre-commit # ray -ray[default]==2.8.1 +ray[default]==2.6.3 # via # -r requirements-dev.in # -r requirements.txt diff --git a/src/requirements.txt b/src/requirements.txt index 191be082..249111d3 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -54,7 +54,9 @@ frozenlist==1.4.0 # aiosignal # ray grpcio==1.59.3 - # via -r requirements.in + # via + # -r requirements.in + # ray gunicorn==20.1.0 # via -r requirements.in idna==3.6 @@ -119,7 +121,7 @@ pyyaml==6.0.1 # -r requirements.in # drf-yasg # ray -ray==2.8.1 +ray==2.6.3 # via scos-actions referencing==0.31.1 # via From f26bfbadd72d71bdf0e3f522b8c46f9323a61f15 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 5 Dec 2023 15:51:33 -0700 Subject: [PATCH 030/255] Enable mock sensor. Set BASE_IMAGE to USRP. Update requirements for USRP. --- docker-compose.yml | 4 ++-- env.template | 2 +- src/requirements-dev.txt | 24 +++++++++++++++--------- src/requirements.in | 2 +- src/requirements.txt | 13 +++++++------ 5 files changed, 26 insertions(+), 19 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 19792cd0..3afffe59 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -51,8 +51,8 @@ services: - IN_DOCKER=1 - IPS - MAX_DISK_USAGE - - MOCK_SIGAN - - MOCK_SIGAN_RANDOM + - MOCK_SIGAN=1 + - MOCK_SIGAN_RANDOM=1 - OAUTH_TOKEN_URL - PATH_TO_CLIENT_CERT - PATH_TO_JWT_PUBLIC_KEY diff --git a/env.template b/env.template index 6ba31d84..bd00a56f 100644 --- a/env.template +++ b/env.template @@ -62,7 +62,7 @@ CALLBACK_SSL_VERIFICATION=true MANAGER_FQDN="$(hostname -f)" MANAGER_IP="$(hostname -I | cut -d' ' -f1)" -BASE_IMAGE=ghcr.io/ntia/scos-tekrsa/tekrsa_usb:0.2.3 +BASE_IMAGE=ghcr.io/ntia/scos-usrp/scos_usrp_uhd:latest # Default callback api/results # Set to OAUTH if using OAuth Password Flow Authentication, callback url needs to be api/v2/results CALLBACK_AUTHENTICATION=TOKEN diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index b4a97319..5a6f58c9 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -16,6 +16,8 @@ aiosignal==1.3.1 # -r requirements.txt # aiohttp # ray +ansicon==1.89.0 + # via jinxed asgiref==3.6.0 # via # -r requirements.txt @@ -55,7 +57,12 @@ click==8.1.3 # -r requirements.txt # ray colorama==0.4.6 - # via tox + # via + # -r requirements.txt + # click + # colorful + # pytest + # tox colorful==0.5.5 # via ray coreapi==2.3.3 @@ -87,6 +94,7 @@ django==3.2.23 # drf-yasg # jsonfield # scos-actions + # scos-usrp django-session-timeout==0.1.0 # via -r requirements.txt djangorestframework==3.14.0 @@ -99,7 +107,7 @@ environs==9.5.0 # via # -r requirements.txt # scos-actions - # scos-tekrsa + # scos-usrp exceptiongroup==1.1.3 # via pytest filelock==3.9.0 @@ -153,6 +161,8 @@ jinja2==3.1.2 # via # -r requirements.txt # coreschema +jinxed==1.2.1 + # via blessed jsonfield==3.1.0 # via -r requirements.txt jsonschema==3.2.0 @@ -192,8 +202,8 @@ numpy==1.24.2 # ray # scipy # scos-actions + # scos-usrp # sigmf - # tekrsa-api-wrap nvidia-ml-py==11.525.112 # via gpustat oauthlib==3.2.2 @@ -323,8 +333,8 @@ scipy==1.10.1 scos-actions @ git+https://github.com/NTIA/scos-actions@7.0.0 # via # -r requirements.txt - # scos-tekrsa -scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@4.0.0 + # scos-usrp +scos-usrp @ git+https://github.com/NTIA/scos-usrp@scos-actions-7.0.0 # via -r requirements.txt sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive # via @@ -346,10 +356,6 @@ sqlparse==0.4.4 # via # -r requirements.txt # django -tekrsa-api-wrap==1.3.2 - # via - # -r requirements.txt - # scos-tekrsa tomli==2.0.1 # via # coverage diff --git a/src/requirements.in b/src/requirements.in index c34c131a..bc1c7b1c 100644 --- a/src/requirements.in +++ b/src/requirements.in @@ -13,7 +13,7 @@ psycopg2-binary>=2.0, <3.0 pyjwt>=2.4.0, <3.0 requests-mock>=1.0, <2.0 requests_oauthlib>=1.0, <2.0 -scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@4.0.0 +scos_usrp @ git+https://github.com/NTIA/scos-usrp@scos-actions-7.0.0 # The following are sub-dependencies for which SCOS Sensor enforces a # higher minimum patch version than the dependencies which require them. diff --git a/src/requirements.txt b/src/requirements.txt index c73aa7f2..ea792224 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -18,6 +18,8 @@ charset-normalizer==3.0.1 # via requests click==8.1.3 # via ray +colorama==0.4.6 + # via click coreapi==2.3.3 # via drf-yasg coreschema==0.0.4 @@ -36,6 +38,7 @@ django==3.2.23 # drf-yasg # jsonfield # scos-actions + # scos-usrp django-session-timeout==0.1.0 # via -r requirements.in djangorestframework==3.14.0 @@ -48,7 +51,7 @@ environs==9.5.0 # via # -r requirements.in # scos-actions - # scos-tekrsa + # scos-usrp filelock==3.9.0 # via # -r requirements.in @@ -91,8 +94,8 @@ numpy==1.24.2 # ray # scipy # scos-actions + # scos-usrp # sigmf - # tekrsa-api-wrap oauthlib==3.2.2 # via # -r requirements.in @@ -150,8 +153,8 @@ ruamel-yaml-clib==0.2.8 scipy==1.10.1 # via scos-actions scos-actions @ git+https://github.com/NTIA/scos-actions@7.0.0 - # via scos-tekrsa -scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@4.0.0 + # via scos-usrp +scos-usrp @ git+https://github.com/NTIA/scos-usrp@scos-actions-7.0.0 # via -r requirements.in sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive # via scos-actions @@ -164,8 +167,6 @@ six==1.16.0 # sigmf sqlparse==0.4.4 # via django -tekrsa-api-wrap==1.3.2 - # via scos-tekrsa uritemplate==4.1.1 # via # coreapi From 32d0d7924b79ceae8d09cd578e79b6e952125e9c Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Wed, 6 Dec 2023 15:42:05 -0700 Subject: [PATCH 031/255] debug scos_actions. --- src/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/requirements.txt b/src/requirements.txt index ea792224..910407bd 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -152,7 +152,7 @@ ruamel-yaml-clib==0.2.8 # via ruamel-yaml scipy==1.10.1 # via scos-actions -scos-actions @ git+https://github.com/NTIA/scos-actions@7.0.0 +scos-actions @ git+https://github.com/NTIA/scos-actions@usrp_debug # via scos-usrp scos-usrp @ git+https://github.com/NTIA/scos-usrp@scos-actions-7.0.0 # via -r requirements.in From 56d7434dd8c55f3f28a3b3758fa577a6a30fa14c Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 7 Dec 2023 07:43:44 -0700 Subject: [PATCH 032/255] debugging. --- src/requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index 5a6f58c9..f807ad4c 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -330,7 +330,7 @@ scipy==1.10.1 # via # -r requirements.txt # scos-actions -scos-actions @ git+https://github.com/NTIA/scos-actions@7.0.0 +scos-actions @ git+https://github.com/NTIA/scos-actions@usrp_debug # via # -r requirements.txt # scos-usrp From 478ff00ab6c2aa7a1f593250a2d61b33b63b4beb Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 7 Dec 2023 12:02:40 -0700 Subject: [PATCH 033/255] scos-actions 7.0.1 --- src/requirements-dev.txt | 2 +- src/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index f807ad4c..679ab720 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -330,7 +330,7 @@ scipy==1.10.1 # via # -r requirements.txt # scos-actions -scos-actions @ git+https://github.com/NTIA/scos-actions@usrp_debug +scos-actions @ git+https://github.com/NTIA/scos-actions@7.0.1 # via # -r requirements.txt # scos-usrp diff --git a/src/requirements.txt b/src/requirements.txt index 910407bd..da5ce363 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -152,7 +152,7 @@ ruamel-yaml-clib==0.2.8 # via ruamel-yaml scipy==1.10.1 # via scos-actions -scos-actions @ git+https://github.com/NTIA/scos-actions@usrp_debug +scos-actions @ git+https://github.com/NTIA/scos-actions@7.0.1 # via scos-usrp scos-usrp @ git+https://github.com/NTIA/scos-usrp@scos-actions-7.0.0 # via -r requirements.in From dc85300743d89f84007e35898e447badd67d9ede Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 7 Dec 2023 12:50:18 -0700 Subject: [PATCH 034/255] empty_calibration_params --- src/requirements-dev.txt | 4 ++-- src/requirements.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index 679ab720..a83d1a01 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -330,11 +330,11 @@ scipy==1.10.1 # via # -r requirements.txt # scos-actions -scos-actions @ git+https://github.com/NTIA/scos-actions@7.0.1 +scos-actions @ git+https://github.com/NTIA/scos-actions@empty_calibration_params # via # -r requirements.txt # scos-usrp -scos-usrp @ git+https://github.com/NTIA/scos-usrp@scos-actions-7.0.0 +scos-usrp @ git+https://github.com/NTIA/scos-usrp@empty_calibration_params # via -r requirements.txt sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive # via diff --git a/src/requirements.txt b/src/requirements.txt index da5ce363..242ed431 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -152,9 +152,9 @@ ruamel-yaml-clib==0.2.8 # via ruamel-yaml scipy==1.10.1 # via scos-actions -scos-actions @ git+https://github.com/NTIA/scos-actions@7.0.1 +scos-actions @ git+https://github.com/NTIA/scos-actions@empty_calibration_params # via scos-usrp -scos-usrp @ git+https://github.com/NTIA/scos-usrp@scos-actions-7.0.0 +scos-usrp @ git+https://github.com/NTIA/scos-usrp@empty_calibration_params # via -r requirements.in sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive # via scos-actions From 27f1babf52a54c5a2fb34046af3b7d6609ba03dc Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 7 Dec 2023 12:51:14 -0700 Subject: [PATCH 035/255] don't use mock_sigan --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3afffe59..19792cd0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -51,8 +51,8 @@ services: - IN_DOCKER=1 - IPS - MAX_DISK_USAGE - - MOCK_SIGAN=1 - - MOCK_SIGAN_RANDOM=1 + - MOCK_SIGAN + - MOCK_SIGAN_RANDOM - OAUTH_TOKEN_URL - PATH_TO_CLIENT_CERT - PATH_TO_JWT_PUBLIC_KEY From b30e6942369b4590121cee876491c6c3a07b4e21 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 11 Dec 2023 09:13:03 -0700 Subject: [PATCH 036/255] Update readme and add default sensor_calibration.json file. --- README.md | 167 ++++++++++++++++++++++++++++++++ configs/sensor_calibration.json | 16 +++ 2 files changed, 183 insertions(+) create mode 100644 configs/sensor_calibration.json diff --git a/README.md b/README.md index 6c84746b..a284dde1 100644 --- a/README.md +++ b/README.md @@ -368,6 +368,173 @@ specific to the sensor you are using. } ``` +### Sensor Calibration File +By default, scos-sensor will use `configs/sensor_calibration.json` as the sensor calibration file. +Sensor calibration files allow scos-sensor to apply a gain based on a laboratory calibration of +the signal analyzer and may also contain other useful metadata that characterizes the sensor performance. +The default calibration file is shown below: +```json +{ + "calibration_data":{ + "datetime": "1970-01-01T00:00:00.000000Z", + "gain": 0, + "noise_figure": null, + "1db_compression_point": null, + "enbw": null, + "temperature": 26.85 + }, + "last_calibration_datetime": "1970-01-01T00:00:00.000000Z", + "calibration_parameters": [], + "clock_rate_lookup_by_sample_rate": [ + ], + "sensor_uid": "DEFAULT CALIBRATION", + "calibration_reference": "noise source output" +} +``` +The `calibration_parameters` key lists the parameters that will be used to obtain +the calibration data. In the case of the default calibration, there are no +`calibration_parameters` so the calibration data is found directly within the +`calibration_data` element and by default scos-sensor will not apply any additional +gain. Typically, a sensor would be calibrated at particular +sensing parameters. For example, the calibration below provides an example of a +sensor calibrated at a sample rate of 14000000.0 at several frequencies with a +signal analyzer reference level setting of -25. +```json +{ + "calibration_parameters": [ + "sample_rate", + "frequency", + "reference_level" + ], + "calibration_datetime": "2020-11-18T23:13:09.156274Z", + "calibration_data": { + "14000000.0":{ + "3555000000":{ + "-25":{ + "noise_figure": 46.03993010994134, + "enbw": 15723428.858731967, + "gain": 0.40803345928877379 + } + }, + "3565000000":{ + "-25":{ + "noise_figure": 46.03993010994134, + "enbw": 15723428.858731967, + "gain": 0.40803345928877379 + } + }, + "3575000000":{ + + "-25":{ + "noise_figure": 46.03993010994134, + "enbw": 15723428.858731967, + "gain": 0.40803345928877379 + } + }, + "3585000000":{ + "-25":{ + "noise_figure": 46.03993010994134, + "enbw": 15723428.858731967, + "gain": 0.40803345928877379 + } + }, + "3595000000":{ + "-25":{ + "noise_figure": 46.03993010994134, + "enbw": 15723428.858731967, + "gain": 0.40803345928877379 + } + }, + "3605000000":{ + "-25":{ + "noise_figure": 46.03993010994134, + "enbw": 15723428.858731967, + "gain": 0.40803345928877379 + } + }, + "3615000000":{ + "-25":{ + "noise_figure": 46.03993010994134, + "enbw": 15723428.858731967, + "gain": 0.40803345928877379 + } + }, + "3625000000":{ + "-25":{ + "noise_figure": 46.03993010994134, + "enbw": 15723428.858731967, + "gain": 0.40803345928877379 + } + }, + "3635000000":{ + "-25":{ + "noise_figure": 46.03993010994134, + "enbw": 15723428.858731967, + "gain": 0.40803345928877379 + } + }, + "3645000000":{ + "-25":{ + "noise_figure": 46.03993010994134, + "enbw": 15723428.858731967, + "gain": 0.40803345928877379 + } + }, + "3655000000":{ + "-25":{ + "noise_figure": 46.03993010994134, + "enbw": 15723428.858731967, + "gain": 0.40803345928877379 + } + }, + "3665000000":{ + "-25":{ + "noise_figure": 46.03993010994134, + "enbw": 15723428.858731967, + "gain": 0.40803345928877379 + } + }, + "3675000000":{ + "-25":{ + "noise_figure": 46.03993010994134, + "enbw": 15723428.858731967, + "gain": 0.40803345928877379 + } + }, + "3685000000":{ + "-25":{ + "noise_figure": 46.03993010994134, + "enbw": 15723428.858731967, + "gain": 0.40803345928877379 + } + }, + "3695000000":{ + "-25":{ + "noise_figure": 46.03993010994134, + "enbw": 15723428.858731967, + "gain": 0.40803345928877379 + } + } + } + }, + "clock_rate_lookup_by_sample_rate": [ + { + "sample_rate": 3500000.0, + "clock_frequency": 3500000.0 + }, + { + "sample_rate": 28000000.0, + "clock_frequency": 28000000.0 + } + ], + "sensor_uid": "US55120115" +} + +``` +When an action is run with the above calibration, scos will expect the action to have +a sample_rate, frequency, and reference_level specified in the action config. The values +specified for these parameters will then be used to retrieve the calibration data. + ## Security This section covers authentication, permissions, and certificates used to access the diff --git a/configs/sensor_calibration.json b/configs/sensor_calibration.json new file mode 100644 index 00000000..0b1ca100 --- /dev/null +++ b/configs/sensor_calibration.json @@ -0,0 +1,16 @@ +{ + "calibration_data":{ + "datetime": "1970-01-01T00:00:00.000000Z", + "gain": 0, + "noise_figure": null, + "1db_compression_point": null, + "enbw": null, + "temperature": 26.85 + }, + "last_calibration_datetime": "1970-01-01T00:00:00.000000Z", + "calibration_parameters": [], + "clock_rate_lookup_by_sample_rate": [ + ], + "sensor_uid": "DEFAULT CALIBRATION", + "calibration_reference": "noise source output" +} From 22f57340b74aef1916a53d3f765dfeff6bd30598 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 11 Dec 2023 10:26:21 -0700 Subject: [PATCH 037/255] Update BASE_IMAGE and add link to readme. --- README.md | 19 ++++++++++--------- env.template | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index a284dde1..7d82a0d2 100644 --- a/README.md +++ b/README.md @@ -371,8 +371,9 @@ specific to the sensor you are using. ### Sensor Calibration File By default, scos-sensor will use `configs/sensor_calibration.json` as the sensor calibration file. Sensor calibration files allow scos-sensor to apply a gain based on a laboratory calibration of -the signal analyzer and may also contain other useful metadata that characterizes the sensor performance. -The default calibration file is shown below: +the sensor and may also contain other useful metadata that characterizes the sensor performance. For additional +information on the calibration data, see the [NTIA-Sensor SigMF Calibration Object](https://github.com/NTIA/sigmf-ns-ntia/blob/master/ntia-sensor.sigmf-ext.md#08-the-calibration-object). +The default calibration file is shown below: ```json { "calibration_data":{ @@ -393,12 +394,12 @@ The default calibration file is shown below: ``` The `calibration_parameters` key lists the parameters that will be used to obtain the calibration data. In the case of the default calibration, there are no -`calibration_parameters` so the calibration data is found directly within the +`calibration_parameters` so the calibration data is found directly within the `calibration_data` element and by default scos-sensor will not apply any additional gain. Typically, a sensor would be calibrated at particular -sensing parameters. For example, the calibration below provides an example of a -sensor calibrated at a sample rate of 14000000.0 at several frequencies with a -signal analyzer reference level setting of -25. +sensing parameters. For example, the calibration below provides an example of a +sensor calibrated at a sample rate of 14000000.0 at several frequencies with a +signal analyzer reference level setting of -25. ```json { "calibration_parameters": [ @@ -424,7 +425,7 @@ signal analyzer reference level setting of -25. } }, "3575000000":{ - + "-25":{ "noise_figure": 46.03993010994134, "enbw": 15723428.858731967, @@ -531,9 +532,9 @@ signal analyzer reference level setting of -25. } ``` -When an action is run with the above calibration, scos will expect the action to have +When an action is run with the above calibration, scos will expect the action to have a sample_rate, frequency, and reference_level specified in the action config. The values -specified for these parameters will then be used to retrieve the calibration data. +specified for these parameters will then be used to retrieve the calibration data. ## Security diff --git a/env.template b/env.template index bd00a56f..91814306 100644 --- a/env.template +++ b/env.template @@ -62,7 +62,7 @@ CALLBACK_SSL_VERIFICATION=true MANAGER_FQDN="$(hostname -f)" MANAGER_IP="$(hostname -I | cut -d' ' -f1)" -BASE_IMAGE=ghcr.io/ntia/scos-usrp/scos_usrp_uhd:latest +BASE_IMAGE=ghcr.io/ntia/scos-usrp/scos_usrp_uhd:0.0.2 # Default callback api/results # Set to OAUTH if using OAuth Password Flow Authentication, callback url needs to be api/v2/results CALLBACK_AUTHENTICATION=TOKEN From 5139b279abf224ccac1b42a71eb30224157f9428 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Wed, 3 Jan 2024 08:32:35 -0700 Subject: [PATCH 038/255] Rename default calibration file. Set DEFAULT_CALIBRATION_FILE variable. Use default calibration file if configs/sensor_calibration.json or configs/sigan_calibration.json don't exist. --- ...alibration.json => default_calibration.json} | 0 env.template | 2 +- src/requirements-dev.txt | 17 +++++++++-------- src/requirements.in | 2 +- src/requirements.txt | 11 ++++++----- src/sensor/migration_settings.py | 6 ++++++ src/sensor/runtime_settings.py | 6 ++++++ src/sensor/settings.py | 6 ++++++ 8 files changed, 35 insertions(+), 15 deletions(-) rename configs/{sensor_calibration.json => default_calibration.json} (100%) diff --git a/configs/sensor_calibration.json b/configs/default_calibration.json similarity index 100% rename from configs/sensor_calibration.json rename to configs/default_calibration.json diff --git a/env.template b/env.template index 91814306..6ba31d84 100644 --- a/env.template +++ b/env.template @@ -62,7 +62,7 @@ CALLBACK_SSL_VERIFICATION=true MANAGER_FQDN="$(hostname -f)" MANAGER_IP="$(hostname -I | cut -d' ' -f1)" -BASE_IMAGE=ghcr.io/ntia/scos-usrp/scos_usrp_uhd:0.0.2 +BASE_IMAGE=ghcr.io/ntia/scos-tekrsa/tekrsa_usb:0.2.3 # Default callback api/results # Set to OAUTH if using OAuth Password Flow Authentication, callback url needs to be api/v2/results CALLBACK_AUTHENTICATION=TOKEN diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index a83d1a01..6866099b 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -75,9 +75,7 @@ coreschema==0.0.4 # coreapi # drf-yasg coverage[toml]==7.2.1 - # via - # coverage - # pytest-cov + # via pytest-cov cryptography==41.0.6 # via -r requirements.txt defusedxml==0.7.1 @@ -94,7 +92,6 @@ django==3.2.23 # drf-yasg # jsonfield # scos-actions - # scos-usrp django-session-timeout==0.1.0 # via -r requirements.txt djangorestframework==3.14.0 @@ -107,7 +104,7 @@ environs==9.5.0 # via # -r requirements.txt # scos-actions - # scos-usrp + # scos-tekrsa exceptiongroup==1.1.3 # via pytest filelock==3.9.0 @@ -202,8 +199,8 @@ numpy==1.24.2 # ray # scipy # scos-actions - # scos-usrp # sigmf + # tekrsa-api-wrap nvidia-ml-py==11.525.112 # via gpustat oauthlib==3.2.2 @@ -333,8 +330,8 @@ scipy==1.10.1 scos-actions @ git+https://github.com/NTIA/scos-actions@empty_calibration_params # via # -r requirements.txt - # scos-usrp -scos-usrp @ git+https://github.com/NTIA/scos-usrp@empty_calibration_params + # scos-tekrsa +scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@empty_calibration_params # via -r requirements.txt sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive # via @@ -356,6 +353,10 @@ sqlparse==0.4.4 # via # -r requirements.txt # django +tekrsa-api-wrap==1.3.2 + # via + # -r requirements.txt + # scos-tekrsa tomli==2.0.1 # via # coverage diff --git a/src/requirements.in b/src/requirements.in index bc1c7b1c..de1f7981 100644 --- a/src/requirements.in +++ b/src/requirements.in @@ -13,7 +13,7 @@ psycopg2-binary>=2.0, <3.0 pyjwt>=2.4.0, <3.0 requests-mock>=1.0, <2.0 requests_oauthlib>=1.0, <2.0 -scos_usrp @ git+https://github.com/NTIA/scos-usrp@scos-actions-7.0.0 +scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@empty_calibration_params # The following are sub-dependencies for which SCOS Sensor enforces a # higher minimum patch version than the dependencies which require them. diff --git a/src/requirements.txt b/src/requirements.txt index 242ed431..fd58f601 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -38,7 +38,6 @@ django==3.2.23 # drf-yasg # jsonfield # scos-actions - # scos-usrp django-session-timeout==0.1.0 # via -r requirements.in djangorestframework==3.14.0 @@ -51,7 +50,7 @@ environs==9.5.0 # via # -r requirements.in # scos-actions - # scos-usrp + # scos-tekrsa filelock==3.9.0 # via # -r requirements.in @@ -94,8 +93,8 @@ numpy==1.24.2 # ray # scipy # scos-actions - # scos-usrp # sigmf + # tekrsa-api-wrap oauthlib==3.2.2 # via # -r requirements.in @@ -153,8 +152,8 @@ ruamel-yaml-clib==0.2.8 scipy==1.10.1 # via scos-actions scos-actions @ git+https://github.com/NTIA/scos-actions@empty_calibration_params - # via scos-usrp -scos-usrp @ git+https://github.com/NTIA/scos-usrp@empty_calibration_params + # via scos-tekrsa +scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@empty_calibration_params # via -r requirements.in sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive # via scos-actions @@ -167,6 +166,8 @@ six==1.16.0 # sigmf sqlparse==0.4.4 # via django +tekrsa-api-wrap==1.3.2 + # via scos-tekrsa uritemplate==4.1.1 # via # coreapi diff --git a/src/sensor/migration_settings.py b/src/sensor/migration_settings.py index 0cf779a6..29ca0a3e 100644 --- a/src/sensor/migration_settings.py +++ b/src/sensor/migration_settings.py @@ -68,11 +68,17 @@ CONFIG_DIR = path.join(REPO_ROOT, "configs") DRIVERS_DIR = path.join(REPO_ROOT, "drivers") +DEFAULT_CALIBRATION_FILE= path.join(CONFIG_DIR, "default_calibration.json") # JSON configs if path.exists(path.join(CONFIG_DIR, "sensor_calibration.json")): SENSOR_CALIBRATION_FILE = path.join(CONFIG_DIR, "sensor_calibration.json") +else: + SENSOR_CALIBRATION_FILE = DEFAULT_CALIBRATION_FILE if path.exists(path.join(CONFIG_DIR, "sigan_calibration.json")): SIGAN_CALIBRATION_FILE = path.join(CONFIG_DIR, "sigan_calibration.json") +else: + SIGAN_CALIBRATION_FILE = DEFAULT_CALIBRATION_FILE + if path.exists(path.join(CONFIG_DIR, "sensor_definition.json")): SENSOR_DEFINITION_FILE = path.join(CONFIG_DIR, "sensor_definition.json") MEDIA_ROOT = path.join(REPO_ROOT, "files") diff --git a/src/sensor/runtime_settings.py b/src/sensor/runtime_settings.py index 069a5b4a..44de5442 100644 --- a/src/sensor/runtime_settings.py +++ b/src/sensor/runtime_settings.py @@ -67,11 +67,17 @@ CONFIG_DIR = path.join(REPO_ROOT, "configs") DRIVERS_DIR = path.join(REPO_ROOT, "drivers") +DEFAULT_CALIBRATION_FILE= path.join(CONFIG_DIR, "default_calibration.json") # JSON configs if path.exists(path.join(CONFIG_DIR, "sensor_calibration.json")): SENSOR_CALIBRATION_FILE = path.join(CONFIG_DIR, "sensor_calibration.json") +else: + SENSOR_CALIBRATION_FILE = DEFAULT_CALIBRATION_FILE if path.exists(path.join(CONFIG_DIR, "sigan_calibration.json")): SIGAN_CALIBRATION_FILE = path.join(CONFIG_DIR, "sigan_calibration.json") +else: + SIGAN_CALIBRATION_FILE = DEFAULT_CALIBRATION_FILE + if path.exists(path.join(CONFIG_DIR, "sensor_definition.json")): SENSOR_DEFINITION_FILE = path.join(CONFIG_DIR, "sensor_definition.json") MEDIA_ROOT = path.join(REPO_ROOT, "files") diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 8c512804..ef41f4fc 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -68,11 +68,17 @@ CONFIG_DIR = path.join(REPO_ROOT, "configs") DRIVERS_DIR = path.join(REPO_ROOT, "drivers") +DEFAULT_CALIBRATION_FILE= path.join(CONFIG_DIR, "default_calibration.json") # JSON configs if path.exists(path.join(CONFIG_DIR, "sensor_calibration.json")): SENSOR_CALIBRATION_FILE = path.join(CONFIG_DIR, "sensor_calibration.json") +else: + SENSOR_CALIBRATION_FILE = DEFAULT_CALIBRATION_FILE if path.exists(path.join(CONFIG_DIR, "sigan_calibration.json")): SIGAN_CALIBRATION_FILE = path.join(CONFIG_DIR, "sigan_calibration.json") +else: + SIGAN_CALIBRATION_FILE = DEFAULT_CALIBRATION_FILE + if path.exists(path.join(CONFIG_DIR, "sensor_definition.json")): SENSOR_DEFINITION_FILE = path.join(CONFIG_DIR, "sensor_definition.json") MEDIA_ROOT = path.join(REPO_ROOT, "files") From 3590675efc0316848a34425f40a6d337454a3bd3 Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Mon, 8 Jan 2024 09:19:06 -0700 Subject: [PATCH 039/255] address feedback, remove unneeded env variables from tox.ini --- README.md | 15 +++++++-------- scripts/create_superuser.py | 2 +- src/authentication/auth.py | 5 ++++- src/tox.ini | 2 -- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index e4e59022..54c5fc8b 100644 --- a/README.md +++ b/README.md @@ -405,9 +405,9 @@ or using Django Rest Framework Token Authentication. #### Django Rest Framework Token Authentication -To enable Django Rest Framework token and session authentication, make sure -`AUTHENTICATION` is set to `TOKEN` in the environment file (this will be enabled if -`AUTHENTICATION` set to anything other than `CERT`). +This is the default authentication method. To enable Django Rest Framework token and +session authentication, make sure `AUTHENTICATION` is set to `TOKEN` in the environment +file (this will be enabled if `AUTHENTICATION` set to anything other than `CERT`). A token is automatically created for each user. Django Rest Framework Token Authentication will check that the token in the Authorization header ("Token " + @@ -416,10 +416,9 @@ is used for the browsable API. #### Certificate Authentication -This is the default authentication method. To enable Certificate Authentication, make -sure `AUTHENTICATION` is set to `CERT` in the environment -file. To authenticate, the client will need to send a trusted client certificate. The -Common Name must match the username of a user in the database. +To enable Certificate Authentication, make sure `AUTHENTICATION` is set to `CERT` in +the environment file. To authenticate, the client will need to send a trusted client +certificate. The Common Name must match the username of a user in the database. #### Certificates @@ -527,7 +526,7 @@ or client.pfx when communicating with the API programmatically. ###### Configure scos-sensor -The Nginx web server is configured by default to require client certificates (mutual +The Nginx web server is not configured by default to require client certificates (mutual TLS). To require client certificates, make sure `ssl_verify_client` is set to `on` in the [Nginx configuration file](nginx/conf.template). Comment out this line or set to `off` to disable client certificates. This can also be set to `optional` or diff --git a/scripts/create_superuser.py b/scripts/create_superuser.py index 89529023..97cd9b4c 100755 --- a/scripts/create_superuser.py +++ b/scripts/create_superuser.py @@ -67,6 +67,6 @@ def add_user(username, password, email=None): if additional_user_names != "" and additional_user_password != "": if "," in additional_user_names: for additional_user_name in additional_user_names.split(","): - add_user(additional_user_name, additional_user_password) + add_user(additional_user_name.strip(), additional_user_password) else: add_user(additional_user_names, additional_user_password) diff --git a/src/authentication/auth.py b/src/authentication/auth.py index 3af870e9..920a5db0 100644 --- a/src/authentication/auth.py +++ b/src/authentication/auth.py @@ -30,8 +30,11 @@ def authenticate(self, request): user.last_login = datetime.datetime.now() user.save() except user_model.DoesNotExist: + logger.error("No matching username found!") raise exceptions.AuthenticationFailed("No matching username found!") - except Exception: + except Exception as ex: + logger.error("Error occurred during certificate authentication!") + logger.error(ex) raise exceptions.AuthenticationFailed( "Error occurred during certificate authentication!" ) diff --git a/src/tox.ini b/src/tox.ini index 5f677be7..d44c4172 100644 --- a/src/tox.ini +++ b/src/tox.ini @@ -19,8 +19,6 @@ envlist = py38,py39,py310 setenv = AUTHENTICATION=CERT CALLBACK_AUTHENTICATION=CERT - PATH_TO_CLIENT_CERT=test/sensor01.pem - PATH_TO_VERIFY_CERT=test/scos_test_ca.crt SWITCH_CONFIGS_DIR=../configs/switches [testenv:coverage] From 834e9dd177922f170dfaa36106fee7d2053a6475 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 8 Jan 2024 11:01:24 -0700 Subject: [PATCH 040/255] default cal file and update readme. --- README.md | 13 ++++++++----- configs/default_calibration.json | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 7d82a0d2..c4e71dd2 100644 --- a/README.md +++ b/README.md @@ -369,8 +369,9 @@ specific to the sensor you are using. ``` ### Sensor Calibration File -By default, scos-sensor will use `configs/sensor_calibration.json` as the sensor calibration file. -Sensor calibration files allow scos-sensor to apply a gain based on a laboratory calibration of +By default, scos-sensor will use `configs/default_calibration.json` as the sensor calibration file. However, if +`configs/sensor_calibration.json` or `configs/sigan_calibration.json` exist they will be used instead of the default +calibration file. Sensor calibration files allow scos-sensor to apply a gain based on a laboratory calibration of the sensor and may also contain other useful metadata that characterizes the sensor performance. For additional information on the calibration data, see the [NTIA-Sensor SigMF Calibration Object](https://github.com/NTIA/sigmf-ns-ntia/blob/master/ntia-sensor.sigmf-ext.md#08-the-calibration-object). The default calibration file is shown below: @@ -397,9 +398,11 @@ the calibration data. In the case of the default calibration, there are no `calibration_parameters` so the calibration data is found directly within the `calibration_data` element and by default scos-sensor will not apply any additional gain. Typically, a sensor would be calibrated at particular -sensing parameters. For example, the calibration below provides an example of a -sensor calibrated at a sample rate of 14000000.0 at several frequencies with a -signal analyzer reference level setting of -25. +sensing parameters. The calibration data for specific parameters should be listed +within the calibration_data object and accessed by the values of the settings listed in the +calibration_parameters element. For example, the calibration below provides an example of a +sensor calibrated at a sample rate of 14000000.0 samples per second at several +frequencies with a signal analyzer reference level setting of -25. ```json { "calibration_parameters": [ diff --git a/configs/default_calibration.json b/configs/default_calibration.json index 0b1ca100..5b951871 100644 --- a/configs/default_calibration.json +++ b/configs/default_calibration.json @@ -2,7 +2,7 @@ "calibration_data":{ "datetime": "1970-01-01T00:00:00.000000Z", "gain": 0, - "noise_figure": null, + "noise_figure": 0.0, "1db_compression_point": null, "enbw": null, "temperature": 26.85 From e9b2f30b58a65d3e9bdf860d35c6909dc1198aea Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Mon, 8 Jan 2024 12:48:25 -0700 Subject: [PATCH 041/255] address more feedback --- README.md | 40 ++++++++++++++++++++-------------------- nginx/conf.template | 3 --- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 54c5fc8b..bb593504 100644 --- a/README.md +++ b/README.md @@ -526,30 +526,30 @@ or client.pfx when communicating with the API programmatically. ###### Configure scos-sensor -The Nginx web server is not configured by default to require client certificates (mutual -TLS). To require client certificates, make sure `ssl_verify_client` is set to `on` in -the [Nginx configuration file](nginx/conf.template). Comment out this line or set to -`off` to disable client certificates. This can also be set to `optional` or -`optional_no_ca`, but if a client certificate is not provided, scos-sensor -`AUTHENTICATION` setting must be set to `TOKEN` which requires a token for the API or a -username and password for the browsable API. If you use OCSP, also uncomment -`ssl_ocsp on;`. Additional configuration may be needed for Nginx to check certificate -revocation lists (CRL). Adjust the other Nginx parameters, such as `ssl_verify_depth`, -as desired. See the -[Nginx documentation](https://nginx.org/en/docs/http/ngx_http_ssl_module.html) for -more information about configuring Nginx. - -To disable client certificate authentication, comment out the -following in [nginx/conf.template](nginx/conf.template): +The Nginx web server is not configured by default to require client certificates +(mutual TLS). To require client certificates, uncomment out the following in +[nginx/conf.template](nginx/conf.template): ```text ssl_client_certificate /etc/ssl/certs/ca.crt; ssl_verify_client on; -ssl_ocsp on; -... - if ($ssl_client_verify != SUCCESS) { # under location @proxy_to_wsgi_server { - return 403; - } +``` + +Note that additional configuration may be needed for Nginx to +use OCSP validation and/or check certificate revocation lists (CRL). Adjust the other +Nginx parameters, such as `ssl_verify_depth`, as desired. See the +[Nginx documentation](https://nginx.org/en/docs/http/ngx_http_ssl_module.html) for more +information about configuring Nginx SSL settings. The `ssl_verify_client` setting can +also be set to `optional` or `optional_no_ca`, but if a client certificate is not +provided, scos-sensor `AUTHENTICATION` setting must be set to `TOKEN` which requires a +token for the API or a username and password for the browsable API. + +To disable client certificate authentication, comment out the following in +[nginx/conf.template](nginx/conf.template): + +```text +# ssl_client_certificate /etc/ssl/certs/ca.crt; +# ssl_verify_client on; ``` Copy the server certificate and server private key (sensor01_combined.pem) to diff --git a/nginx/conf.template b/nginx/conf.template index 085f8843..e5e0d46b 100644 --- a/nginx/conf.template +++ b/nginx/conf.template @@ -49,9 +49,6 @@ server { # Pass off requests to Gunicorn location @proxy_to_wsgi_server { - # if ($ssl_client_verify != SUCCESS) { - # return 403; - # } proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto https; From 7c63bf2200c2d59fc1ac26a6a0e1663d0027bc0d Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 8 Jan 2024 13:41:15 -0700 Subject: [PATCH 042/255] sigan monitor --- src/actions/__init__.py | 6 ++++++ src/requirements-dev.txt | 4 ++-- src/requirements.in | 2 +- src/requirements.txt | 4 ++-- src/sensor/settings.py | 1 + 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/actions/__init__.py b/src/actions/__init__.py index dc621227..caeca853 100644 --- a/src/actions/__init__.py +++ b/src/actions/__init__.py @@ -2,8 +2,11 @@ import logging import pkgutil +from scos_actions.hardware import signa_analyzer_monitor from scos_actions.discover import test_actions from scos_actions.signals import register_action +from scos_actions.discover import init + from sensor import settings from sensor.utils import copy_driver_files @@ -31,4 +34,7 @@ for name, action in discover.actions.items(): logger.debug("action: " + name + "=" + str(action)) register_action.send(sender=__name__, action=action) + +yaml_actions, yaml_test_actions = init(sigan=signa_analyzer_monitor.signal_analyzer, yaml_dir=settings.ACTIONS_DIR) logger.debug("Finished loading actions") + diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index 6866099b..4e958650 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -327,11 +327,11 @@ scipy==1.10.1 # via # -r requirements.txt # scos-actions -scos-actions @ git+https://github.com/NTIA/scos-actions@empty_calibration_params +scos-actions @ git+https://github.com/NTIA/scos-actions@SiganMonitor # via # -r requirements.txt # scos-tekrsa -scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@empty_calibration_params +scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@SiganMonitor # via -r requirements.txt sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive # via diff --git a/src/requirements.in b/src/requirements.in index de1f7981..66bb3016 100644 --- a/src/requirements.in +++ b/src/requirements.in @@ -13,7 +13,7 @@ psycopg2-binary>=2.0, <3.0 pyjwt>=2.4.0, <3.0 requests-mock>=1.0, <2.0 requests_oauthlib>=1.0, <2.0 -scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@empty_calibration_params +scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@SiganMonitor # The following are sub-dependencies for which SCOS Sensor enforces a # higher minimum patch version than the dependencies which require them. diff --git a/src/requirements.txt b/src/requirements.txt index fd58f601..eb359a09 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -151,9 +151,9 @@ ruamel-yaml-clib==0.2.8 # via ruamel-yaml scipy==1.10.1 # via scos-actions -scos-actions @ git+https://github.com/NTIA/scos-actions@empty_calibration_params +scos-actions @ git+https://github.com/NTIA/scos-actions@SiganMonitor # via scos-tekrsa -scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@empty_calibration_params +scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@SiganMonitor # via -r requirements.in sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive # via scos-actions diff --git a/src/sensor/settings.py b/src/sensor/settings.py index ef41f4fc..65a8f3e6 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -66,6 +66,7 @@ OPENAPI_FILE = path.join(REPO_ROOT, "docs", "openapi.json") CONFIG_DIR = path.join(REPO_ROOT, "configs") +ACTIONS_DIR = path.join(CONFIG_DIR, "actions") DRIVERS_DIR = path.join(REPO_ROOT, "drivers") DEFAULT_CALIBRATION_FILE= path.join(CONFIG_DIR, "default_calibration.json") From de81df741774954198e67170695e2184b40d5542 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 8 Jan 2024 13:55:08 -0700 Subject: [PATCH 043/255] move signal_analyzer_monitor to scos_actions.status --- src/actions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/actions/__init__.py b/src/actions/__init__.py index caeca853..5722c84e 100644 --- a/src/actions/__init__.py +++ b/src/actions/__init__.py @@ -2,7 +2,7 @@ import logging import pkgutil -from scos_actions.hardware import signa_analyzer_monitor +from scos_actions.status import signal_analyzer_monitor from scos_actions.discover import test_actions from scos_actions.signals import register_action from scos_actions.discover import init From 95b97434fc625920e99c02109d0a8d5f97588894 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 8 Jan 2024 14:14:39 -0700 Subject: [PATCH 044/255] typo fix. --- src/actions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/actions/__init__.py b/src/actions/__init__.py index 5722c84e..bdf81098 100644 --- a/src/actions/__init__.py +++ b/src/actions/__init__.py @@ -35,6 +35,6 @@ logger.debug("action: " + name + "=" + str(action)) register_action.send(sender=__name__, action=action) -yaml_actions, yaml_test_actions = init(sigan=signa_analyzer_monitor.signal_analyzer, yaml_dir=settings.ACTIONS_DIR) +yaml_actions, yaml_test_actions = init(sigan=signal_analyzer_monitor.signal_analyzer, yaml_dir=settings.ACTIONS_DIR) logger.debug("Finished loading actions") From 598c098e2dbc12d58650cdf6ea06cf01879f510b Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 8 Jan 2024 14:25:32 -0700 Subject: [PATCH 045/255] print actions dir. --- src/actions/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/actions/__init__.py b/src/actions/__init__.py index bdf81098..1674e4cf 100644 --- a/src/actions/__init__.py +++ b/src/actions/__init__.py @@ -35,6 +35,7 @@ logger.debug("action: " + name + "=" + str(action)) register_action.send(sender=__name__, action=action) +logger.debug(f"Loading actions in {settings.ACTIONS_DIR}") yaml_actions, yaml_test_actions = init(sigan=signal_analyzer_monitor.signal_analyzer, yaml_dir=settings.ACTIONS_DIR) logger.debug("Finished loading actions") From 8a9a4790e2ee8ee2ea6d7361944620fdef717000 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 8 Jan 2024 15:38:23 -0700 Subject: [PATCH 046/255] Set ACTIONS_DIR in all settings. --- src/sensor/migration_settings.py | 1 + src/sensor/runtime_settings.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/sensor/migration_settings.py b/src/sensor/migration_settings.py index 29ca0a3e..3b9e2233 100644 --- a/src/sensor/migration_settings.py +++ b/src/sensor/migration_settings.py @@ -66,6 +66,7 @@ OPENAPI_FILE = path.join(REPO_ROOT, "docs", "openapi.json") CONFIG_DIR = path.join(REPO_ROOT, "configs") +ACTIONS_DIR = path.join(CONFIG_DIR, "actions") DRIVERS_DIR = path.join(REPO_ROOT, "drivers") DEFAULT_CALIBRATION_FILE= path.join(CONFIG_DIR, "default_calibration.json") diff --git a/src/sensor/runtime_settings.py b/src/sensor/runtime_settings.py index 44de5442..28491ca3 100644 --- a/src/sensor/runtime_settings.py +++ b/src/sensor/runtime_settings.py @@ -65,6 +65,7 @@ OPENAPI_FILE = path.join(REPO_ROOT, "docs", "openapi.json") CONFIG_DIR = path.join(REPO_ROOT, "configs") +ACTIONS_DIR = path.join(CONFIG_DIR, "actions") DRIVERS_DIR = path.join(REPO_ROOT, "drivers") DEFAULT_CALIBRATION_FILE= path.join(CONFIG_DIR, "default_calibration.json") From 114ea89c9293d8bacc310db821265a3a1760a51e Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 8 Jan 2024 16:00:23 -0700 Subject: [PATCH 047/255] Register actions in configs/actions. --- src/actions/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/actions/__init__.py b/src/actions/__init__.py index 1674e4cf..81358835 100644 --- a/src/actions/__init__.py +++ b/src/actions/__init__.py @@ -37,5 +37,8 @@ logger.debug(f"Loading actions in {settings.ACTIONS_DIR}") yaml_actions, yaml_test_actions = init(sigan=signal_analyzer_monitor.signal_analyzer, yaml_dir=settings.ACTIONS_DIR) +for name, action in yaml_actions.items(): + logger.debug("action: " + name + "=" + str(action)) + register_action.send(sender=__name__, action=action) logger.debug("Finished loading actions") From 9dd9201a7f2552ae9eb69a086c3fbcecc5ef99f1 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 8 Jan 2024 21:09:52 -0700 Subject: [PATCH 048/255] Switch actions and tekrsa branches. --- src/requirements-dev.txt | 4 ++-- src/requirements.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index 4e958650..829356b3 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -327,11 +327,11 @@ scipy==1.10.1 # via # -r requirements.txt # scos-actions -scos-actions @ git+https://github.com/NTIA/scos-actions@SiganMonitor +scos-actions @ git+https://github.com/NTIA/scos-actions@CoreSiganMonitor # via # -r requirements.txt # scos-tekrsa -scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@SiganMonitor +scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@CoreSiganMonitor # via -r requirements.txt sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive # via diff --git a/src/requirements.txt b/src/requirements.txt index eb359a09..bc6c42b2 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -151,9 +151,9 @@ ruamel-yaml-clib==0.2.8 # via ruamel-yaml scipy==1.10.1 # via scos-actions -scos-actions @ git+https://github.com/NTIA/scos-actions@SiganMonitor +scos-actions @ git+https://github.com/NTIA/scos-actions@CoreSiganMonitor # via scos-tekrsa -scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@SiganMonitor +scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@CoreSiganMonitor # via -r requirements.in sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive # via scos-actions From 51e8986bac51c02b22ad71e2619cc4b6e00e1ea0 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 8 Jan 2024 21:28:34 -0700 Subject: [PATCH 049/255] scos_actions.core refactor. --- src/actions/__init__.py | 2 +- src/status/views.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/actions/__init__.py b/src/actions/__init__.py index 81358835..32304afe 100644 --- a/src/actions/__init__.py +++ b/src/actions/__init__.py @@ -2,7 +2,7 @@ import logging import pkgutil -from scos_actions.status import signal_analyzer_monitor +from scos_actions.core import signal_analyzer_monitor from scos_actions.discover import test_actions from scos_actions.signals import register_action from scos_actions.discover import init diff --git a/src/status/views.py b/src/status/views.py index 64201795..70b26406 100644 --- a/src/status/views.py +++ b/src/status/views.py @@ -7,7 +7,7 @@ from rest_framework.decorators import api_view from rest_framework.response import Response from scos_actions.hardware.sigan_iface import SignalAnalyzerInterface -from scos_actions.status import status_registrar +from scos_actions.core import status_registrar from scos_actions.utils import ( convert_datetime_to_millisecond_iso_format, get_datetime_str_now, From 3d59c7727b61547d2ad8122b847f41d73e4fbfc1 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 11 Jan 2024 16:17:51 -0700 Subject: [PATCH 050/255] discover action types, gps and sigan and ensure one sigan and actions have sigan and gps. --- src/actions/__init__.py | 45 +++++++++++++++++++++++++++++++------- src/requirements-dev.txt | 4 ++-- src/requirements.txt | 4 ++-- src/scheduler/scheduler.py | 18 ++++++++++++++- src/status/views.py | 4 ++-- 5 files changed, 60 insertions(+), 15 deletions(-) diff --git a/src/actions/__init__.py b/src/actions/__init__.py index 32304afe..d1c41db3 100644 --- a/src/actions/__init__.py +++ b/src/actions/__init__.py @@ -2,7 +2,7 @@ import logging import pkgutil -from scos_actions.core import signal_analyzer_monitor +from scos_actions.actions import action_classes from scos_actions.discover import test_actions from scos_actions.signals import register_action from scos_actions.discover import init @@ -22,7 +22,11 @@ if name.startswith("scos_") and name != "scos_actions" } logger.debug(discovered_plugins) - +action_types = {} +action_types.update(action_classes) +signal_analyzer = None +gps = None +discovered_actions = {} if settings.MOCK_SIGAN or settings.RUNNING_TESTS: for name, action in test_actions.items(): logger.debug("test_action: " + name + "=" + str(action)) @@ -31,14 +35,39 @@ for name, module in discovered_plugins.items(): logger.debug("Looking for actions in " + name + ": " + str(module)) discover = importlib.import_module(name + ".discover") - for name, action in discover.actions.items(): - logger.debug("action: " + name + "=" + str(action)) - register_action.send(sender=__name__, action=action) + if hasattr(discover, "actions"): + for name, action in discover.actions.items(): + logger.debug("action: " + name + "=" + str(action)) + discovered_actions[name] = action + if hasattr(discover, "action_types") and discover.action_types is not None: + action_types.update(discover.action_types) + if hasattr(discover, "signal_analzyer") and discover.signal_analyzer is not None: + if signal_analyzer is not None: + raise Exception("Multiple signal analyzers discovered.") + signal_analyzer = discover.signal_analzyer + if hasattr(discover, "gps") and discover.gps is not None: + gps = discover.gps + +if sigan is None: + raise Exception("No signal analyzer found.") +#Ensure all actions have a sigan +logger.debug("Ensuring actions have signal analyzer.") +for name, action in discovered_actions.items(): + if action.signal_analzyer is None: + logger.debug(f"Setting signal analyzer for {name}") + action.set_signal_analyzer(sigan) + if gps is not None and action.gps is None: + logger.debug(f"Setting gps for {name}") + action.gps = gps + logger.debug(f"Loading actions in {settings.ACTIONS_DIR}") -yaml_actions, yaml_test_actions = init(sigan=signal_analyzer_monitor.signal_analyzer, yaml_dir=settings.ACTIONS_DIR) -for name, action in yaml_actions.items(): +yaml_actions, yaml_test_actions = init(sigan=sigan, action_classes = action_types, yaml_dir=settings.ACTIONS_DIR) +discovered_actions.update(yaml_actions) + +for name, action in discovered_actions.items(): logger.debug("action: " + name + "=" + str(action)) register_action.send(sender=__name__, action=action) -logger.debug("Finished loading actions") + +logger.debug("Finished loading and registering actions") diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index 829356b3..7c8c5ce7 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -327,11 +327,11 @@ scipy==1.10.1 # via # -r requirements.txt # scos-actions -scos-actions @ git+https://github.com/NTIA/scos-actions@CoreSiganMonitor +scos-actions @ git+https://github.com/NTIA/scos-actions@discover_action_types # via # -r requirements.txt # scos-tekrsa -scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@CoreSiganMonitor +scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@discover_action_types # via -r requirements.txt sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive # via diff --git a/src/requirements.txt b/src/requirements.txt index bc6c42b2..c8a0ab01 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -151,9 +151,9 @@ ruamel-yaml-clib==0.2.8 # via ruamel-yaml scipy==1.10.1 # via scos-actions -scos-actions @ git+https://github.com/NTIA/scos-actions@CoreSiganMonitor +scos-actions @ git+https://github.com/NTIA/scos-actions@discover_action_types # via scos-tekrsa -scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@CoreSiganMonitor +scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@discover_action_types # via -r requirements.in sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive # via scos-actions diff --git a/src/scheduler/scheduler.py b/src/scheduler/scheduler.py index b5849ccd..fcdc70b5 100644 --- a/src/scheduler/scheduler.py +++ b/src/scheduler/scheduler.py @@ -8,7 +8,11 @@ import requests from django.utils import timezone -from scos_actions.signals import trigger_api_restart +from scos_actions.hardware import SignalAnalyzerInterface +from scos_actions.signals import ( + trigger_api_restart, + register_signal_analyzer +) from authentication import oauth from schedule.models import ScheduleEntry @@ -18,6 +22,7 @@ from tasks.serializers import TaskResultSerializer from tasks.task_queue import TaskQueue + from . import utils logger = logging.getLogger(__name__) @@ -46,6 +51,11 @@ def __init__(self): self.task = None # Task object describing current task self.last_status = "" self.consecutive_failures = 0 + self._signal_analyzer = None + + @property + def signal_analyzer(self, sigan: SignalAnalzyerInterface): + self._signal_analyzer = sigan @property def schedule(self): @@ -343,3 +353,9 @@ def minimum_duration(blocking): # of running it in its own microservice is that we _must not_ run the # application server in multiple processes (multiple threads are fine). thread = Scheduler() + +#def register_sigan(sender, **kwargs): +# thread.signal_analzyzer = kwargs["signal_analyzer"]) + +#register_signal_analyzer.connect(register_sigan) +#logger.debug("Connected register_signal_analyzer signal") diff --git a/src/status/views.py b/src/status/views.py index 70b26406..7bb09f93 100644 --- a/src/status/views.py +++ b/src/status/views.py @@ -7,7 +7,7 @@ from rest_framework.decorators import api_view from rest_framework.response import Response from scos_actions.hardware.sigan_iface import SignalAnalyzerInterface -from scos_actions.core import status_registrar +from scos_actions.status import status_monitor from scos_actions.utils import ( convert_datetime_to_millisecond_iso_format, get_datetime_str_now, @@ -61,7 +61,7 @@ def status(request, version, format=None): "disk_usage": disk_usage(), "days_up": get_days_up(), } - for component in status_registrar.status_components: + for component in status_monitor.status_components: component_status = component.get_status() if isinstance(component, WebRelay): if "switches" in status_json: From 3b2d8921b93955245cb10e935946cbc548acf55d Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 11 Jan 2024 16:35:43 -0700 Subject: [PATCH 051/255] delete faulty import. --- src/scheduler/scheduler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/scheduler/scheduler.py b/src/scheduler/scheduler.py index fcdc70b5..ed8c3aab 100644 --- a/src/scheduler/scheduler.py +++ b/src/scheduler/scheduler.py @@ -8,7 +8,6 @@ import requests from django.utils import timezone -from scos_actions.hardware import SignalAnalyzerInterface from scos_actions.signals import ( trigger_api_restart, register_signal_analyzer From d2c55b68929a425faf3307d21af3afa3458db8d6 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 11 Jan 2024 16:41:20 -0700 Subject: [PATCH 052/255] remove erroneous import. --- src/scheduler/scheduler.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/scheduler/scheduler.py b/src/scheduler/scheduler.py index ed8c3aab..34bba329 100644 --- a/src/scheduler/scheduler.py +++ b/src/scheduler/scheduler.py @@ -8,11 +8,7 @@ import requests from django.utils import timezone -from scos_actions.signals import ( - trigger_api_restart, - register_signal_analyzer -) - +from scos_actions.signals import trigger_api_restart from authentication import oauth from schedule.models import ScheduleEntry from sensor import settings From 403899b150a97e864288d77bcdc25ec802412fda Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 11 Jan 2024 16:55:30 -0700 Subject: [PATCH 053/255] sigan->signal_analyzer. --- src/actions/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/actions/__init__.py b/src/actions/__init__.py index d1c41db3..4153bd6d 100644 --- a/src/actions/__init__.py +++ b/src/actions/__init__.py @@ -48,21 +48,21 @@ if hasattr(discover, "gps") and discover.gps is not None: gps = discover.gps -if sigan is None: +if signal_analyzer is None: raise Exception("No signal analyzer found.") #Ensure all actions have a sigan logger.debug("Ensuring actions have signal analyzer.") for name, action in discovered_actions.items(): if action.signal_analzyer is None: logger.debug(f"Setting signal analyzer for {name}") - action.set_signal_analyzer(sigan) + action.set_signal_analyzer(signal_analyzer) if gps is not None and action.gps is None: logger.debug(f"Setting gps for {name}") action.gps = gps logger.debug(f"Loading actions in {settings.ACTIONS_DIR}") -yaml_actions, yaml_test_actions = init(sigan=sigan, action_classes = action_types, yaml_dir=settings.ACTIONS_DIR) +yaml_actions, yaml_test_actions = init(sigan=signal_analyzer, action_classes = action_types, yaml_dir=settings.ACTIONS_DIR) discovered_actions.update(yaml_actions) for name, action in discovered_actions.items(): From 194a0a19f8b5eeee231dfdbb9716a4a6ad494645 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 11 Jan 2024 17:11:28 -0700 Subject: [PATCH 054/255] typo --- src/actions/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/actions/__init__.py b/src/actions/__init__.py index 4153bd6d..966a5589 100644 --- a/src/actions/__init__.py +++ b/src/actions/__init__.py @@ -41,10 +41,10 @@ discovered_actions[name] = action if hasattr(discover, "action_types") and discover.action_types is not None: action_types.update(discover.action_types) - if hasattr(discover, "signal_analzyer") and discover.signal_analyzer is not None: + if hasattr(discover, "signal_analyzer") and discover.signal_analyzer is not None: if signal_analyzer is not None: raise Exception("Multiple signal analyzers discovered.") - signal_analyzer = discover.signal_analzyer + signal_analyzer = discover.signal_analyzer if hasattr(discover, "gps") and discover.gps is not None: gps = discover.gps From 5d88874441a55895f914e9766ac31334c1136f73 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 11 Jan 2024 17:16:48 -0700 Subject: [PATCH 055/255] fix sigan detection. --- src/actions/__init__.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/actions/__init__.py b/src/actions/__init__.py index 966a5589..d46acf6c 100644 --- a/src/actions/__init__.py +++ b/src/actions/__init__.py @@ -41,15 +41,17 @@ discovered_actions[name] = action if hasattr(discover, "action_types") and discover.action_types is not None: action_types.update(discover.action_types) - if hasattr(discover, "signal_analyzer") and discover.signal_analyzer is not None: - if signal_analyzer is not None: - raise Exception("Multiple signal analyzers discovered.") - signal_analyzer = discover.signal_analyzer + logger.debug("Checking for signal analyzer.") + if hasattr(discover, "signal_analyzer") and discover.signal_analyzer is not None: + if signal_analyzer is not None: + raise Exception("Multiple signal analyzers discovered.") + logger.debug("signal analyzer found") + signal_analyzer = discover.signal_analyzer if hasattr(discover, "gps") and discover.gps is not None: gps = discover.gps if signal_analyzer is None: - raise Exception("No signal analyzer found.") + logger.error("No signal analyzer found.") #Ensure all actions have a sigan logger.debug("Ensuring actions have signal analyzer.") for name, action in discovered_actions.items(): From ae5a1ee9eb5c3d8cfddc1d760e7eb2f5301dc863 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 11 Jan 2024 18:31:35 -0700 Subject: [PATCH 056/255] fix setting sigan in actions. --- src/actions/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/actions/__init__.py b/src/actions/__init__.py index d46acf6c..e537550b 100644 --- a/src/actions/__init__.py +++ b/src/actions/__init__.py @@ -55,9 +55,9 @@ #Ensure all actions have a sigan logger.debug("Ensuring actions have signal analyzer.") for name, action in discovered_actions.items(): - if action.signal_analzyer is None: + if action.signal_analyzer is None: logger.debug(f"Setting signal analyzer for {name}") - action.set_signal_analyzer(signal_analyzer) + action.signal_analyzer = signal_analyzer if gps is not None and action.gps is None: logger.debug(f"Setting gps for {name}") action.gps = gps From 629b1995d60ac224b1228e6d4f8b2ab346d1ef65 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 11 Jan 2024 18:37:25 -0700 Subject: [PATCH 057/255] remove scheduler signal_analyzer. --- src/scheduler/scheduler.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/scheduler/scheduler.py b/src/scheduler/scheduler.py index 34bba329..27098a9b 100644 --- a/src/scheduler/scheduler.py +++ b/src/scheduler/scheduler.py @@ -48,10 +48,6 @@ def __init__(self): self.consecutive_failures = 0 self._signal_analyzer = None - @property - def signal_analyzer(self, sigan: SignalAnalzyerInterface): - self._signal_analyzer = sigan - @property def schedule(self): """An updated view of the current schedule""" From f74dc64e3a83d9d20849dce006f643c1d7a27fb2 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 11 Jan 2024 18:55:37 -0700 Subject: [PATCH 058/255] debugging --- src/utils/action_registrar.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/utils/action_registrar.py b/src/utils/action_registrar.py index ee6c79e6..feda0ce8 100644 --- a/src/utils/action_registrar.py +++ b/src/utils/action_registrar.py @@ -1,13 +1,18 @@ +import logging from collections import OrderedDict from scos_actions.signals import register_action +logger = logging.getLogger(__name__) + +logger.debug("Creating Actions dictionary") registered_actions = OrderedDict() def add_action_handler(sender, **kwargs): action = kwargs["action"] + logger.debug("adding action " + action) registered_actions[action.name] = action - +logger.debug("Connected register action handler") register_action.connect(add_action_handler) From fcc8910bc335e28c38f8fdd27fc55441694a3890 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 11 Jan 2024 19:03:31 -0700 Subject: [PATCH 059/255] fix log message. --- src/utils/action_registrar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/action_registrar.py b/src/utils/action_registrar.py index feda0ce8..5485e148 100644 --- a/src/utils/action_registrar.py +++ b/src/utils/action_registrar.py @@ -11,7 +11,7 @@ def add_action_handler(sender, **kwargs): action = kwargs["action"] - logger.debug("adding action " + action) + logger.debug(f"adding action {action}") registered_actions[action.name] = action logger.debug("Connected register action handler") From 315b46d735c1702d063a807b034f715cdf9ae287 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 12 Jan 2024 07:02:03 -0700 Subject: [PATCH 060/255] Add SIGAN_MODULE and SIGAN_CLASS settings. --- docker-compose.yml | 2 ++ src/scheduler/scheduler.py | 5 ----- src/sensor/settings.py | 3 +++ src/utils/action_registrar.py | 18 ------------------ 4 files changed, 5 insertions(+), 23 deletions(-) delete mode 100644 src/utils/action_registrar.py diff --git a/docker-compose.yml b/docker-compose.yml index 19792cd0..7d6cda35 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -60,6 +60,8 @@ services: - POSTGRES_PASSWORD - SCOS_SENSOR_GIT_TAG - SECRET_KEY + - SIGAN_MODULE + - SIGAN_CLASS - SIGAN_POWER_SWITCH - SIGAN_POWER_CYCLE_STATES expose: diff --git a/src/scheduler/scheduler.py b/src/scheduler/scheduler.py index 27098a9b..847fc1e4 100644 --- a/src/scheduler/scheduler.py +++ b/src/scheduler/scheduler.py @@ -345,8 +345,3 @@ def minimum_duration(blocking): # application server in multiple processes (multiple threads are fine). thread = Scheduler() -#def register_sigan(sender, **kwargs): -# thread.signal_analzyzer = kwargs["signal_analyzer"]) - -#register_signal_analyzer.connect(register_sigan) -#logger.debug("Connected register_signal_analyzer signal") diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 65a8f3e6..ad56f181 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -434,3 +434,6 @@ SIGAN_POWER_CYCLE_STATES = env("SIGAN_POWER_CYCLE_STATES", default=None) SIGAN_POWER_SWITCH = env("SIGAN_POWER_SWITCH", default=None) MAX_FAILURES = env("MAX_FAILURES", default=2) +DEVICE_MODEL = env("DEVICE_MODEL", default="RSA507A") +SIGAN_MODULE = env.str("SIGAN_MDOULE", default="scos_tekrsa.tekrsa_sigan") +SIGAN_CLASS = env.str("SIGAN_CLASS", default="TekRSASigan") \ No newline at end of file diff --git a/src/utils/action_registrar.py b/src/utils/action_registrar.py deleted file mode 100644 index 5485e148..00000000 --- a/src/utils/action_registrar.py +++ /dev/null @@ -1,18 +0,0 @@ -import logging -from collections import OrderedDict - -from scos_actions.signals import register_action - -logger = logging.getLogger(__name__) - -logger.debug("Creating Actions dictionary") -registered_actions = OrderedDict() - - -def add_action_handler(sender, **kwargs): - action = kwargs["action"] - logger.debug(f"adding action {action}") - registered_actions[action.name] = action - -logger.debug("Connected register action handler") -register_action.connect(add_action_handler) From c62c2826cdac7ad72f9d4181b8fba4eac338fb65 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 12 Jan 2024 07:26:04 -0700 Subject: [PATCH 061/255] Instantiate specified sigan and pass to scheduler. --- gunicorn/config.py | 14 +++++++++++--- src/scheduler/scheduler.py | 19 ++++++++++++++++++- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index 3a570541..c6b2cc49 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -1,14 +1,17 @@ +import importlib +import logging import os import sys from multiprocessing import cpu_count + bind = ":8000" workers = 1 worker_class = "gthread" threads = cpu_count() loglevel = os.environ.get("GUNICORN_LOG_LEVEL", "info") - +logger = logging.getLogger(__name__) def _modify_path(): """Ensure Django project is on sys.path.""" @@ -27,9 +30,14 @@ def post_worker_init(worker): import django django.setup() - + env = Env() from scheduler import scheduler - + sigan_module_setting = env("SIGAN_MODULE") + sigan_module = importlib.import_module(sigan_module_setting) + logger.info("Creating " + env("SIGAN_CLASS") + " from " + sigan_module) + sigan_constructor = getattr(sigan_module, env("SIGAN_CLASS")) + sigan = sigan_constructor() + scheduler.signal_analyzer = sigan scheduler.thread.start() diff --git a/src/scheduler/scheduler.py b/src/scheduler/scheduler.py index 847fc1e4..65651b14 100644 --- a/src/scheduler/scheduler.py +++ b/src/scheduler/scheduler.py @@ -47,6 +47,23 @@ def __init__(self): self.last_status = "" self.consecutive_failures = 0 self._signal_analyzer = None + self._gps = None + + @property + def signal_analyzer(self): + return self._signal_analyzer + + @signal_analyzer.setter + def signal_analzyer(self, sigan): + self._signal_analyzer = sigan + + @property + def gps(self): + return self._gps + + @gps.setter + def gps(self, gps): + self._gps = gps @property def schedule(self): @@ -164,7 +181,7 @@ def _call_task_action(self): try: logger.debug(f"running task {entry_name}/{task_id}") - detail = self.task.action_caller(schedule_entry_json, task_id) + detail = self.task.action_caller(self.signal_analzyer, self.gps, schedule_entry_json, task_id) self.delayfn(0) # let other threads run status = "success" if not isinstance(detail, str): From 4ba3080bc9c09166839cdeadb2b9638e156dc077 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 12 Jan 2024 08:17:05 -0700 Subject: [PATCH 062/255] remove action_registrar. --- scripts/print_action_docstring.py | 6 +++--- src/actions/__init__.py | 13 ++++--------- src/capabilities/__init__.py | 4 ++-- src/schedule/__init__.py | 2 +- src/scheduler/tests/utils.py | 4 ++-- src/tasks/__init__.py | 3 +-- src/tasks/models/task.py | 2 +- 7 files changed, 14 insertions(+), 20 deletions(-) diff --git a/scripts/print_action_docstring.py b/scripts/print_action_docstring.py index bfc03769..99023f32 100755 --- a/scripts/print_action_docstring.py +++ b/scripts/print_action_docstring.py @@ -6,7 +6,7 @@ import django -from utils.action_registrar import registered_actions +from actions import actions PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "src") @@ -16,7 +16,7 @@ django.setup() -action_names = sorted(registered_actions.keys()) +action_names = sorted(actions.keys()) if __name__ == "__main__": @@ -28,4 +28,4 @@ ), args = parser.parse_args() - print(registered_actions[args.action].description) + print(actions[args.action].description) diff --git a/src/actions/__init__.py b/src/actions/__init__.py index e537550b..8c72f51b 100644 --- a/src/actions/__init__.py +++ b/src/actions/__init__.py @@ -26,7 +26,7 @@ action_types.update(action_classes) signal_analyzer = None gps = None -discovered_actions = {} +actions = {} if settings.MOCK_SIGAN or settings.RUNNING_TESTS: for name, action in test_actions.items(): logger.debug("test_action: " + name + "=" + str(action)) @@ -38,7 +38,7 @@ if hasattr(discover, "actions"): for name, action in discover.actions.items(): logger.debug("action: " + name + "=" + str(action)) - discovered_actions[name] = action + actions[name] = action if hasattr(discover, "action_types") and discover.action_types is not None: action_types.update(discover.action_types) logger.debug("Checking for signal analyzer.") @@ -54,7 +54,7 @@ logger.error("No signal analyzer found.") #Ensure all actions have a sigan logger.debug("Ensuring actions have signal analyzer.") -for name, action in discovered_actions.items(): +for name, action in actions.items(): if action.signal_analyzer is None: logger.debug(f"Setting signal analyzer for {name}") action.signal_analyzer = signal_analyzer @@ -65,11 +65,6 @@ logger.debug(f"Loading actions in {settings.ACTIONS_DIR}") yaml_actions, yaml_test_actions = init(sigan=signal_analyzer, action_classes = action_types, yaml_dir=settings.ACTIONS_DIR) -discovered_actions.update(yaml_actions) - -for name, action in discovered_actions.items(): - logger.debug("action: " + name + "=" + str(action)) - register_action.send(sender=__name__, action=action) - +actions.update(yaml_actions) logger.debug("Finished loading and registering actions") diff --git a/src/capabilities/__init__.py b/src/capabilities/__init__.py index 63909f42..c14d6b24 100644 --- a/src/capabilities/__init__.py +++ b/src/capabilities/__init__.py @@ -1,10 +1,10 @@ import logging from scos_actions.capabilities import capabilities +from actions import actions -from utils.action_registrar import registered_actions logger = logging.getLogger(__name__) logger.debug("********** Initializing capabilities **********") -actions_by_name = registered_actions +actions_by_name = actions sensor_capabilities = capabilities diff --git a/src/schedule/__init__.py b/src/schedule/__init__.py index da07553f..7f6f150a 100644 --- a/src/schedule/__init__.py +++ b/src/schedule/__init__.py @@ -1,7 +1,7 @@ import logging from utils import get_summary -from utils.action_registrar import registered_actions +from actions import actions def get_action_with_summary(action): diff --git a/src/scheduler/tests/utils.py b/src/scheduler/tests/utils.py index d5e4ef10..5e09c3f5 100644 --- a/src/scheduler/tests/utils.py +++ b/src/scheduler/tests/utils.py @@ -8,7 +8,7 @@ from schedule.models import Request, ScheduleEntry from scheduler.scheduler import Scheduler from sensor import V1 -from utils.action_registrar import registered_actions +from actions import actions BAD_ACTION_STR = "testing expected failure" @@ -122,7 +122,7 @@ def create_bad_action(): def bad_action(schedule_entry_json, task_id): raise Exception(BAD_ACTION_STR) - registered_actions["bad_action"] = bad_action + actions["bad_action"] = bad_action return bad_action diff --git a/src/tasks/__init__.py b/src/tasks/__init__.py index a4143415..8302e1a6 100644 --- a/src/tasks/__init__.py +++ b/src/tasks/__init__.py @@ -1,7 +1,6 @@ import logging -from utils.action_registrar import registered_actions logger = logging.getLogger(__name__) logger.debug("********** Initializing tasks **********") -actions = registered_actions + diff --git a/src/tasks/models/task.py b/src/tasks/models/task.py index f5394224..06118339 100644 --- a/src/tasks/models/task.py +++ b/src/tasks/models/task.py @@ -7,7 +7,7 @@ import logging from collections import namedtuple -from tasks import actions +from actions import actions logger = logging.getLogger(__name__) From c9495ae7246b3e0c8533f8e182fb3ac21d045040 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 12 Jan 2024 09:26:31 -0700 Subject: [PATCH 063/255] remove register_action --- src/actions/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/actions/__init__.py b/src/actions/__init__.py index 8c72f51b..352f50b8 100644 --- a/src/actions/__init__.py +++ b/src/actions/__init__.py @@ -4,7 +4,6 @@ from scos_actions.actions import action_classes from scos_actions.discover import test_actions -from scos_actions.signals import register_action from scos_actions.discover import init @@ -30,7 +29,6 @@ if settings.MOCK_SIGAN or settings.RUNNING_TESTS: for name, action in test_actions.items(): logger.debug("test_action: " + name + "=" + str(action)) - register_action.send(sender=__name__, action=action) else: for name, module in discovered_plugins.items(): logger.debug("Looking for actions in " + name + ": " + str(module)) From b1cd58079cc868daccd0912c04db5a8cab29186f Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 12 Jan 2024 09:36:53 -0700 Subject: [PATCH 064/255] remove checking for sigan and gps from actions/__init__.py --- src/actions/__init__.py | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/src/actions/__init__.py b/src/actions/__init__.py index 352f50b8..ab21435c 100644 --- a/src/actions/__init__.py +++ b/src/actions/__init__.py @@ -23,8 +23,6 @@ logger.debug(discovered_plugins) action_types = {} action_types.update(action_classes) -signal_analyzer = None -gps = None actions = {} if settings.MOCK_SIGAN or settings.RUNNING_TESTS: for name, action in test_actions.items(): @@ -39,30 +37,10 @@ actions[name] = action if hasattr(discover, "action_types") and discover.action_types is not None: action_types.update(discover.action_types) - logger.debug("Checking for signal analyzer.") - if hasattr(discover, "signal_analyzer") and discover.signal_analyzer is not None: - if signal_analyzer is not None: - raise Exception("Multiple signal analyzers discovered.") - logger.debug("signal analyzer found") - signal_analyzer = discover.signal_analyzer - if hasattr(discover, "gps") and discover.gps is not None: - gps = discover.gps - -if signal_analyzer is None: - logger.error("No signal analyzer found.") -#Ensure all actions have a sigan -logger.debug("Ensuring actions have signal analyzer.") -for name, action in actions.items(): - if action.signal_analyzer is None: - logger.debug(f"Setting signal analyzer for {name}") - action.signal_analyzer = signal_analyzer - if gps is not None and action.gps is None: - logger.debug(f"Setting gps for {name}") - action.gps = gps logger.debug(f"Loading actions in {settings.ACTIONS_DIR}") -yaml_actions, yaml_test_actions = init(sigan=signal_analyzer, action_classes = action_types, yaml_dir=settings.ACTIONS_DIR) +yaml_actions, yaml_test_actions = init(action_classes = action_types, yaml_dir=settings.ACTIONS_DIR) actions.update(yaml_actions) logger.debug("Finished loading and registering actions") From f9cf7c22c3f93642c2fc61e08b98a940de601bc2 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 12 Jan 2024 09:46:50 -0700 Subject: [PATCH 065/255] import from actions in schedule. --- src/schedule/__init__.py | 4 ++-- src/schedule/models/schedule_entry.py | 2 +- src/schedule/serializers.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/schedule/__init__.py b/src/schedule/__init__.py index 7f6f150a..f825c3f1 100644 --- a/src/schedule/__init__.py +++ b/src/schedule/__init__.py @@ -6,7 +6,7 @@ def get_action_with_summary(action): """Given an action, return the string 'action_name - summary'.""" - action_fn = registered_actions[action] + action_fn = actions[action] summary = get_summary(action_fn) action_with_summary = action if summary: @@ -17,4 +17,4 @@ def get_action_with_summary(action): logger = logging.getLogger(__name__) logger.debug("********** Initializing schedule **********") -actions = registered_actions + diff --git a/src/schedule/models/schedule_entry.py b/src/schedule/models/schedule_entry.py index a9e6bb1b..47efe181 100644 --- a/src/schedule/models/schedule_entry.py +++ b/src/schedule/models/schedule_entry.py @@ -6,7 +6,7 @@ from django.db import models from constants import MAX_ACTION_LENGTH -from schedule import actions +from actions import actions from scheduler import utils logger = logging.getLogger(__name__) diff --git a/src/schedule/serializers.py b/src/schedule/serializers.py index 27788b73..8d4b5e97 100644 --- a/src/schedule/serializers.py +++ b/src/schedule/serializers.py @@ -6,17 +6,17 @@ convert_datetime_to_millisecond_iso_format, parse_datetime_iso_format_str, ) - +from actions import actions from sensor import V1 from sensor.utils import get_datetime_from_timestamp, get_timestamp_from_datetime -from . import get_action_with_summary, registered_actions +from . import get_action_with_summary from .models import DEFAULT_PRIORITY, ScheduleEntry action_help = "[Required] The name of the action to be scheduled" priority_help = f"Lower number is higher priority (default={DEFAULT_PRIORITY})" CHOICES = [] -actions = sorted(registered_actions.keys()) +actions = sorted(actions.keys()) for action in actions: CHOICES.append((action, get_action_with_summary(action))) From 45911e9b5d953601f6af9824f426557967b7222e Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 12 Jan 2024 09:53:45 -0700 Subject: [PATCH 066/255] import environs in config.py --- gunicorn/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gunicorn/config.py b/gunicorn/config.py index c6b2cc49..82379473 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -2,6 +2,7 @@ import logging import os import sys +from environs import Env from multiprocessing import cpu_count From ec84466cec0fd6ad4930ff7914f2b799ebef9686 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 12 Jan 2024 10:01:16 -0700 Subject: [PATCH 067/255] fix logging error. --- gunicorn/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index 82379473..fadb7c2e 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -35,7 +35,7 @@ def post_worker_init(worker): from scheduler import scheduler sigan_module_setting = env("SIGAN_MODULE") sigan_module = importlib.import_module(sigan_module_setting) - logger.info("Creating " + env("SIGAN_CLASS") + " from " + sigan_module) + logger.info("Creating " + env("SIGAN_CLASS") + " from " + env("SIGAN_MODULE")) sigan_constructor = getattr(sigan_module, env("SIGAN_CLASS")) sigan = sigan_constructor() scheduler.signal_analyzer = sigan From 1592ea5d92088e39569e5dd00af47b1ad62b3ef5 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 12 Jan 2024 10:15:22 -0700 Subject: [PATCH 068/255] register sigan as status provider. Correct default sigan module. --- gunicorn/config.py | 3 ++- src/sensor/settings.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index fadb7c2e..069981e1 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -4,7 +4,7 @@ import sys from environs import Env from multiprocessing import cpu_count - +from scos_actions.signals import register_component_with_status bind = ":8000" workers = 1 @@ -38,6 +38,7 @@ def post_worker_init(worker): logger.info("Creating " + env("SIGAN_CLASS") + " from " + env("SIGAN_MODULE")) sigan_constructor = getattr(sigan_module, env("SIGAN_CLASS")) sigan = sigan_constructor() + register_component_with_status.send(__name__, component=sigan) scheduler.signal_analyzer = sigan scheduler.thread.start() diff --git a/src/sensor/settings.py b/src/sensor/settings.py index ad56f181..f954be82 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -435,5 +435,5 @@ SIGAN_POWER_SWITCH = env("SIGAN_POWER_SWITCH", default=None) MAX_FAILURES = env("MAX_FAILURES", default=2) DEVICE_MODEL = env("DEVICE_MODEL", default="RSA507A") -SIGAN_MODULE = env.str("SIGAN_MDOULE", default="scos_tekrsa.tekrsa_sigan") +SIGAN_MODULE = env.str("SIGAN_MDOULE", default="scos_tekrsa.hardware.tekrsa_sigan") SIGAN_CLASS = env.str("SIGAN_CLASS", default="TekRSASigan") \ No newline at end of file From 549dd16d76400b4282f9302d07e49212365b62d9 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 12 Jan 2024 10:38:34 -0700 Subject: [PATCH 069/255] typo fix. --- gunicorn/config.py | 2 +- src/scheduler/scheduler.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index 069981e1..533f4aef 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -38,7 +38,7 @@ def post_worker_init(worker): logger.info("Creating " + env("SIGAN_CLASS") + " from " + env("SIGAN_MODULE")) sigan_constructor = getattr(sigan_module, env("SIGAN_CLASS")) sigan = sigan_constructor() - register_component_with_status.send(__name__, component=sigan) + register_component_with_status.send(sigan, component=sigan) scheduler.signal_analyzer = sigan scheduler.thread.start() diff --git a/src/scheduler/scheduler.py b/src/scheduler/scheduler.py index 65651b14..4961ad2c 100644 --- a/src/scheduler/scheduler.py +++ b/src/scheduler/scheduler.py @@ -54,7 +54,8 @@ def signal_analyzer(self): return self._signal_analyzer @signal_analyzer.setter - def signal_analzyer(self, sigan): + def signal_analyzer(self, sigan): + logger.debug(f"Set scheduler sigan to {sigan}") self._signal_analyzer = sigan @property @@ -180,8 +181,8 @@ def _call_task_action(self): schedule_entry_json["id"] = entry_name try: - logger.debug(f"running task {entry_name}/{task_id}") - detail = self.task.action_caller(self.signal_analzyer, self.gps, schedule_entry_json, task_id) + logger.debug(f"running task {entry_name}/{task_id} with sigan: {self.signal_analyzer}") + detail = self.task.action_caller(self.signal_analyzer, self.gps, schedule_entry_json, task_id) self.delayfn(0) # let other threads run status = "success" if not isinstance(detail, str): From b57e87d5bbc2479a3bd72a96176683f893feeae1 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 12 Jan 2024 10:45:21 -0700 Subject: [PATCH 070/255] fix setting scheduler sigan. --- gunicorn/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index 533f4aef..885b50b5 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -39,7 +39,7 @@ def post_worker_init(worker): sigan_constructor = getattr(sigan_module, env("SIGAN_CLASS")) sigan = sigan_constructor() register_component_with_status.send(sigan, component=sigan) - scheduler.signal_analyzer = sigan + scheduler.thread.signal_analyzer = sigan scheduler.thread.start() From d71762ff3fbc7af0c3fc792c55e3fe3ced6324db Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 12:32:20 -0700 Subject: [PATCH 071/255] add initializatino package and create Sensor in gunicorn/config.py --- docs/openapi.json | 13 +--- files/README.md | 6 -- gunicorn/config.py | 29 ++++++++- initialization/__init__.py | 93 +++++++++++++++++++++++++++ initialization/tests/__init__.py | 0 src/actions/__init__.py | 92 +++++++++++++------------- src/actions/apps.py | 22 +++---- src/capabilities/__init__.py | 4 +- src/conftest.py | 2 + src/schedule/__init__.py | 3 +- src/schedule/models/schedule_entry.py | 3 +- src/schedule/serializers.py | 3 +- src/scheduler/tests/utils.py | 9 ++- src/sensor/migration_settings.py | 3 + src/sensor/runtime_settings.py | 4 +- src/sensor/settings.py | 14 +++- src/sensor/utils.py | 32 --------- src/sensor/wsgi.py | 15 +++++ src/tasks/models/task.py | 4 +- src/tasks/models/task_result.py | 4 +- src/tox.ini | 2 + 21 files changed, 233 insertions(+), 124 deletions(-) delete mode 100644 files/README.md create mode 100644 initialization/__init__.py create mode 100644 initialization/tests/__init__.py diff --git a/docs/openapi.json b/docs/openapi.json index 270b63cc..bfab7a52 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -1636,16 +1636,7 @@ "title": "Action", "description": "[Required] The name of the action to be scheduled", "type": "string", - "enum": [ - "logger", - "test_monitor_sigan", - "test_multi_frequency_iq_action", - "test_nasctn_sea_data_product", - "test_single_frequency_iq_action", - "test_single_frequency_m4s_action", - "test_survey_iq_action", - "test_sync_gps" - ] + "enum": [] }, "priority": { "title": "Priority", @@ -1941,4 +1932,4 @@ } } } -} +} \ No newline at end of file diff --git a/files/README.md b/files/README.md deleted file mode 100644 index dbb246aa..00000000 --- a/files/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# SCOS Sensor Files Directory - -This directory is mounted as a Docker volume to `/files` in the scos-sensor Docker -container. - -SCOS Sensor stores task result files in this directory on the host (outside the container). diff --git a/gunicorn/config.py b/gunicorn/config.py index 885b50b5..8eb34666 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -3,9 +3,18 @@ import os import sys from environs import Env +from initialization import ( + load_preselector, + load_switches, + load_capabilities, + +) from multiprocessing import cpu_count +from scos_actions.hardware.sensor import Sensor +from scos_actions.metadata.utils import construct_geojson_point from scos_actions.signals import register_component_with_status + bind = ":8000" workers = 1 worker_class = "gthread" @@ -39,7 +48,25 @@ def post_worker_init(worker): sigan_constructor = getattr(sigan_module, env("SIGAN_CLASS")) sigan = sigan_constructor() register_component_with_status.send(sigan, component=sigan) - scheduler.thread.signal_analyzer = sigan + capabilities = load_capabilities(env("SENSOR_DEFINITION_FILE")) + switches = load_switches(env("SWITCH_CONFIGS_DIR")) + for key, switch in switches: + register_component_with_status(switch, component=switch) + preselector = load_preselector(env("PRESELECTOR_CONFIG"), env("PRESELEDTOR_MODULE"), env("PRESELECTOR_CLASS"), capabilities["sensor"]) + register_component_with_status(preselector, component=preselector) + if "location" in capabilities["sensor"]: + try: + sensor_loc = capabilities["sensor"].pop("location") + location = construct_geojson_point( + sensor_loc["x"], + sensor_loc["y"], + sensor_loc["z"] if "z" in sensor_loc else None, + ) + except: + logger.exception("Failed to get sensor location from sensor definition.") + + sensor = Sensor(signal_analyzer=sigan, preselector = preselector, switches = switches, capabilities = capabilities, location = location) + scheduler.thread.sensor = sensor scheduler.thread.start() diff --git a/initialization/__init__.py b/initialization/__init__.py new file mode 100644 index 00000000..bd5ebad3 --- /dev/null +++ b/initialization/__init__.py @@ -0,0 +1,93 @@ +import hashlib +import importlib +import json +import logging +from pathlib import Path +from from django.conf import settings +from its_preselector.configuration_exception import ConfigurationException +from its_preselector.controlbyweb_web_relay import ControlByWebWebRelay + +from scos_actions import utils +from scos_actions.metadata.utils import construct_geojson_point +from scos_actions.signals import register_component_with_status + +logger = logging.getLogger(__name__) + +def load_switches(switch_dir: Path) -> dict: + switch_dict = {} + if switch_dir is not None and switch_dir.is_dir(): + for f in switch_dir.iterdir(): + file_path = f.resolve() + logger.debug(f"loading switch config {file_path}") + conf = utils.load_from_json(file_path) + try: + switch = ControlByWebWebRelay(conf) + logger.debug(f"Adding {switch.id}") + + switch_dict[switch.id] = switch + logger.debug(f"Registering switch status for {switch.name}") + register_component_with_status.send(__name__, component=switch) + except ConfigurationException: + logger.error(f"Unable to configure switch defined in: {file_path}") + + return switch_dict + + +def load_preselector_from_file(preselector_config_file: Path): + if preselector_config_file is None: + return None + else: + try: + preselector_config = utils.load_from_json(preselector_config_file) + return load_preselector( + preselector_config, settings.PRESELECTOR_MODULE, settings.PRESELECTOR_CLASS + ) + except ConfigurationException: + logger.exception( + f"Unable to create preselector defined in: {preselector_config_file}" + ) + return None + + +def load_preselector(preselector_config, module, preselector_class_name, sensor_definition): + if module is not None and preselector_class_name is not None: + preselector_module = importlib.import_module(module) + preselector_constructor = getattr(preselector_module, preselector_class_name) + ps = preselector_constructor(sensor_definition, preselector_config) + if ps and ps.name: + logger.debug(f"Registering {ps.name} as status provider") + register_component_with_status.send(__name__, component=ps) + else: + ps = None + return ps + + +def load_capabilities(sensor_definition_file): + capabilities = {} + SENSOR_DEFINITION_HASH = None + SENSOR_LOCATION = None + + logger.debug(f"Loading {sensor_definition_file}") + try: + capabilities["sensor"] = utils.load_from_json(sensor_definition_file) + except Exception as e: + logger.warning( + f"Failed to load sensor definition file: {sensor_definition_file}" + + "\nAn empty sensor definition will be used" + ) + capabilities["sensor"] = {"sensor_spec": {"id": "unknown"}} + capabilities["sensor"]["sensor_sha512"] = "UNKNOWN SENSOR DEFINITION" + + # Generate sensor definition file hash (SHA 512) + try: + if "sensor_sha512" not in capabilities["sensor"]: + sensor_def = json.dumps(capabilities["sensor"], sort_keys=True) + SENSOR_DEFINITION_HASH = hashlib.sha512(sensor_def.encode("UTF-8")).hexdigest() + capabilities["sensor"]["sensor_sha512"] = SENSOR_DEFINITION_HASH + except: + capabilities["sensor"]["sensor_sha512"] = "ERROR GENERATING HASH" + # SENSOR_DEFINITION_HASH is None, do not raise Exception + logger.exception(f"Unable to generate sensor definition hash") + + return capabilities + diff --git a/initialization/tests/__init__.py b/initialization/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/actions/__init__.py b/src/actions/__init__.py index ab21435c..268ac39a 100644 --- a/src/actions/__init__.py +++ b/src/actions/__init__.py @@ -1,46 +1,46 @@ -import importlib -import logging -import pkgutil - -from scos_actions.actions import action_classes -from scos_actions.discover import test_actions -from scos_actions.discover import init - - -from sensor import settings -from sensor.utils import copy_driver_files - -logger = logging.getLogger(__name__) -logger.debug("********** Initializing actions **********") - -copy_driver_files() # copy driver files before loading plugins - -discovered_plugins = { - name: importlib.import_module(name) - for finder, name, ispkg in pkgutil.iter_modules() - if name.startswith("scos_") and name != "scos_actions" -} -logger.debug(discovered_plugins) -action_types = {} -action_types.update(action_classes) -actions = {} -if settings.MOCK_SIGAN or settings.RUNNING_TESTS: - for name, action in test_actions.items(): - logger.debug("test_action: " + name + "=" + str(action)) -else: - for name, module in discovered_plugins.items(): - logger.debug("Looking for actions in " + name + ": " + str(module)) - discover = importlib.import_module(name + ".discover") - if hasattr(discover, "actions"): - for name, action in discover.actions.items(): - logger.debug("action: " + name + "=" + str(action)) - actions[name] = action - if hasattr(discover, "action_types") and discover.action_types is not None: - action_types.update(discover.action_types) - - -logger.debug(f"Loading actions in {settings.ACTIONS_DIR}") -yaml_actions, yaml_test_actions = init(action_classes = action_types, yaml_dir=settings.ACTIONS_DIR) -actions.update(yaml_actions) -logger.debug("Finished loading and registering actions") - +# import importlib +# import logging +# import pkgutil +# +# from scos_actions.actions import action_classes +# from scos_actions.discover import test_actions +# from scos_actions.discover import init +# +# +# from sensor import settings +# from sensor.utils import copy_driver_files +# +# logger = logging.getLogger(__name__) +# logger.debug("********** Initializing actions **********") +# +# copy_driver_files() # copy driver files before loading plugins +# +# discovered_plugins = { +# name: importlib.import_module(name) +# for finder, name, ispkg in pkgutil.iter_modules() +# if name.startswith("scos_") and name != "scos_actions" +# } +# logger.debug(discovered_plugins) +# action_types = {} +# action_types.update(action_classes) +# actions = {} +# if settings.MOCK_SIGAN or settings.RUNNING_TESTS: +# for name, action in test_actions.items(): +# logger.debug("test_action: " + name + "=" + str(action)) +# else: +# for name, module in discovered_plugins.items(): +# logger.debug("Looking for actions in " + name + ": " + str(module)) +# discover = importlib.import_module(name + ".discover") +# if hasattr(discover, "actions"): +# for name, action in discover.actions.items(): +# logger.debug("action: " + name + "=" + str(action)) +# actions[name] = action +# if hasattr(discover, "action_types") and discover.action_types is not None: +# action_types.update(discover.action_types) +# +# +# logger.debug(f"Loading actions in {settings.ACTIONS_DIR}") +# yaml_actions, yaml_test_actions = init(action_classes = action_types, yaml_dir=settings.ACTIONS_DIR) +# actions.update(yaml_actions) +# logger.debug("Finished loading and registering actions") +# diff --git a/src/actions/apps.py b/src/actions/apps.py index a9f2db23..d5319dee 100644 --- a/src/actions/apps.py +++ b/src/actions/apps.py @@ -1,11 +1,11 @@ -import importlib -import logging -import pkgutil - -from django.apps import AppConfig - -logger = logging.getLogger(__name__) - - -class ActionsConfig(AppConfig): - name = "actions" +# import importlib +# import logging +# import pkgutil +# +# from django.apps import AppConfig +# +# logger = logging.getLogger(__name__) +# +# +# class ActionsConfig(AppConfig): +# name = "actions" diff --git a/src/capabilities/__init__.py b/src/capabilities/__init__.py index c14d6b24..eb84cf44 100644 --- a/src/capabilities/__init__.py +++ b/src/capabilities/__init__.py @@ -1,10 +1,10 @@ import logging from scos_actions.capabilities import capabilities -from actions import actions +from django.conf import settings logger = logging.getLogger(__name__) logger.debug("********** Initializing capabilities **********") -actions_by_name = actions +actions_by_name = settings.actions sensor_capabilities = capabilities diff --git a/src/conftest.py b/src/conftest.py index af8f4f52..845f24ce 100644 --- a/src/conftest.py +++ b/src/conftest.py @@ -10,6 +10,7 @@ from authentication.models import User from authentication.tests.utils import get_test_public_private_key from tasks.models import TaskResult +from scos_actions.hardware.mocks.mock_sigan import MockSignalAnalyzer PRIVATE_KEY, PUBLIC_KEY = get_test_public_private_key() Keys = namedtuple("KEYS", ["private_key", "public_key"]) @@ -42,6 +43,7 @@ def testclock(): def test_scheduler(rf, testclock): """Instantiate test scheduler with fake request context and testclock.""" s = scheduler.scheduler.Scheduler() + s.signal_analyzer = MockSignalAnalyzer() s.request = rf.post("mock://cburl/schedule") return s diff --git a/src/schedule/__init__.py b/src/schedule/__init__.py index f825c3f1..5835226f 100644 --- a/src/schedule/__init__.py +++ b/src/schedule/__init__.py @@ -1,8 +1,9 @@ import logging from utils import get_summary -from actions import actions +from django.conf import settings +actions = settings.actions def get_action_with_summary(action): """Given an action, return the string 'action_name - summary'.""" diff --git a/src/schedule/models/schedule_entry.py b/src/schedule/models/schedule_entry.py index 47efe181..18002763 100644 --- a/src/schedule/models/schedule_entry.py +++ b/src/schedule/models/schedule_entry.py @@ -6,9 +6,10 @@ from django.db import models from constants import MAX_ACTION_LENGTH -from actions import actions +from django.conf import settings from scheduler import utils +actions = settings.actions logger = logging.getLogger(__name__) logger.debug( "************** scos-sensor/schedule/models/schedule_entry.py *****************" diff --git a/src/schedule/serializers.py b/src/schedule/serializers.py index 8d4b5e97..34f0a72a 100644 --- a/src/schedule/serializers.py +++ b/src/schedule/serializers.py @@ -6,13 +6,14 @@ convert_datetime_to_millisecond_iso_format, parse_datetime_iso_format_str, ) -from actions import actions +from django.conf import settings from sensor import V1 from sensor.utils import get_datetime_from_timestamp, get_timestamp_from_datetime from . import get_action_with_summary from .models import DEFAULT_PRIORITY, ScheduleEntry +actions = settings.actions action_help = "[Required] The name of the action to be scheduled" priority_help = f"Lower number is higher priority (default={DEFAULT_PRIORITY})" CHOICES = [] diff --git a/src/scheduler/tests/utils.py b/src/scheduler/tests/utils.py index 5e09c3f5..7efb01f4 100644 --- a/src/scheduler/tests/utils.py +++ b/src/scheduler/tests/utils.py @@ -8,8 +8,10 @@ from schedule.models import Request, ScheduleEntry from scheduler.scheduler import Scheduler from sensor import V1 -from actions import actions +from django.conf import settings +from scos_actions.hardware.mocks.mock_sigan import MockSignalAnalyzer +actions = settings.actions BAD_ACTION_STR = "testing expected failure" logger = logging.getLogger(__name__) @@ -60,6 +62,7 @@ def advance_testclock(iterator, n): def simulate_scheduler_run(n=1): s = Scheduler() + s.signal_analyzer = MockSignalAnalyzer() for _ in range(n): advance_testclock(s.timefn, 1) s.run(blocking=False) @@ -109,7 +112,7 @@ def cb(schedule_entry_json, task_id): return "set flag" cb.__name__ = "testcb" + str(create_action.counter) - registered_actions[cb.__name__] = cb + actions[cb.__name__] = cb create_action.counter += 1 return cb, flag @@ -119,7 +122,7 @@ def cb(schedule_entry_json, task_id): def create_bad_action(): - def bad_action(schedule_entry_json, task_id): + def bad_action(sigan, gps, schedule_entry_json, task_id): raise Exception(BAD_ACTION_STR) actions["bad_action"] = bad_action diff --git a/src/sensor/migration_settings.py b/src/sensor/migration_settings.py index 3b9e2233..22be3b05 100644 --- a/src/sensor/migration_settings.py +++ b/src/sensor/migration_settings.py @@ -22,6 +22,8 @@ from django.core.management.utils import get_random_secret_key from environs import Env +from src.sensor.action_loader import load_actions + env = Env() # See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ @@ -422,3 +424,4 @@ SIGAN_POWER_CYCLE_STATES = env("SIGAN_POWER_CYCLE_STATES", default=None) SIGAN_POWER_SWITCH = env("SIGAN_POWER_SWITCH", default=None) MAX_FAILURES = env("MAX_FAILURES", default=2) +actions = load_actions(MOCK_SIGAN, RUNNING_TESTS, DRIVERS_DIR, ACTIONS_DIR) \ No newline at end of file diff --git a/src/sensor/runtime_settings.py b/src/sensor/runtime_settings.py index 28491ca3..0bb0d272 100644 --- a/src/sensor/runtime_settings.py +++ b/src/sensor/runtime_settings.py @@ -16,7 +16,7 @@ import os import sys from os import path - +from action_loader import load_actions from cryptography.fernet import Fernet from django.core.management.utils import get_random_secret_key from environs import Env @@ -196,7 +196,6 @@ "scheduler.apps.SchedulerConfig", "status.apps.StatusConfig", "sensor.apps.SensorConfig", # global settings/utils, etc - "actions.apps.ActionsConfig", ] MIDDLEWARE = [ @@ -432,3 +431,4 @@ SIGAN_POWER_CYCLE_STATES = env("SIGAN_POWER_CYCLE_STATES", default=None) SIGAN_POWER_SWITCH = env("SIGAN_POWER_SWITCH", default=None) MAX_FAILURES = env("MAX_FAILURES", default=2) +actions = load_actions(MOCK_SIGAN, RUNNING_TESTS, DRIVERS_DIR, ACTIONS_DIR) \ No newline at end of file diff --git a/src/sensor/settings.py b/src/sensor/settings.py index f954be82..a3fc082e 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -326,6 +326,9 @@ "NAME": "test.db", # temporary workaround for https://github.com/pytest-dev/pytest-django/issues/783 } } + DEVICE_MODEL = env("DEVICE_MODEL", default="RSA507A") + SIGAN_MODULE ="scos_actions.hardware.mocks.mock_sigan" + SIGAN_CLASS = "MockSignalAnalyzer" else: DATABASES = { "default": { @@ -337,6 +340,9 @@ "PORT": "5432", } } + DEVICE_MODEL = env("DEVICE_MODEL", default="RSA507A") + SIGAN_MODULE = env.str("SIGAN_MDOULE", default="scos_tekrsa.hardware.tekrsa_sigan") + SIGAN_CLASS = env.str("SIGAN_CLASS", default="TekRSASigan") if not IN_DOCKER: DATABASES["default"]["HOST"] = "localhost" @@ -434,6 +440,8 @@ SIGAN_POWER_CYCLE_STATES = env("SIGAN_POWER_CYCLE_STATES", default=None) SIGAN_POWER_SWITCH = env("SIGAN_POWER_SWITCH", default=None) MAX_FAILURES = env("MAX_FAILURES", default=2) -DEVICE_MODEL = env("DEVICE_MODEL", default="RSA507A") -SIGAN_MODULE = env.str("SIGAN_MDOULE", default="scos_tekrsa.hardware.tekrsa_sigan") -SIGAN_CLASS = env.str("SIGAN_CLASS", default="TekRSASigan") \ No newline at end of file + + + + + diff --git a/src/sensor/utils.py b/src/sensor/utils.py index af07d057..3b2cd92d 100644 --- a/src/sensor/utils.py +++ b/src/sensor/utils.py @@ -1,7 +1,4 @@ -import json import logging -import os -import shutil from datetime import datetime from django.conf import settings @@ -22,32 +19,3 @@ def parse_datetime_str(d: str) -> datetime: return datetime.strptime(d, settings.DATETIME_FORMAT) -def copy_driver_files(): - """Copy driver files where they need to go""" - for root, dirs, files in os.walk(settings.DRIVERS_DIR): - for filename in files: - name_without_ext, ext = os.path.splitext(filename) - if ext.lower() == ".json": - json_data = {} - file_path = os.path.join(root, filename) - with open(file_path) as json_file: - json_data = json.load(json_file) - if type(json_data) == dict and "scos_files" in json_data: - scos_files = json_data["scos_files"] - for scos_file in scos_files: - source_path = os.path.join( - settings.DRIVERS_DIR, scos_file["source_path"] - ) - if not os.path.isfile(source_path): - logger.error(f"Unable to find file at {source_path}") - continue - dest_path = scos_file["dest_path"] - dest_dir = os.path.dirname(dest_path) - try: - if not os.path.isdir(dest_dir): - os.makedirs(dest_dir) - logger.debug(f"copying {source_path} to {dest_path}") - shutil.copyfile(source_path, dest_path) - except Exception as e: - logger.error(f"Failed to copy {source_path} to {dest_path}") - logger.error(e) diff --git a/src/sensor/wsgi.py b/src/sensor/wsgi.py index 703f9628..8f6d64d3 100644 --- a/src/sensor/wsgi.py +++ b/src/sensor/wsgi.py @@ -12,7 +12,11 @@ import os import django +import importlib +import logging from django.core.wsgi import get_wsgi_application +from environs import Env +from scos_actions.signals import register_component_with_status os.environ.setdefault("DJANGO_SETTINGS_MODULE", "sensor.settings") django.setup() # this is necessary because we need to handle our own thread @@ -30,4 +34,15 @@ if not settings.IN_DOCKER: # Normally scheduler is started by gunicorn worker process + env = Env() + sigan_module_setting = env("SIGAN_MODULE") + sigan_module = importlib.import_module(sigan_module_setting) + logger = logging.getLogger(__name__) + logger.info("Creating " + env("SIGAN_CLASS") + " from " + env("SIGAN_MODULE")) + sigan_constructor = getattr(sigan_module, env("SIGAN_CLASS")) + sigan = sigan_constructor() + register_component_with_status.send(sigan, component=sigan) + scheduler.thread.signal_analyzer = sigan scheduler.thread.start() + scheduler.thread.start() + diff --git a/src/tasks/models/task.py b/src/tasks/models/task.py index 06118339..9fc9546a 100644 --- a/src/tasks/models/task.py +++ b/src/tasks/models/task.py @@ -6,9 +6,9 @@ import logging from collections import namedtuple +from django.conf import settings -from actions import actions - +actions = settings.actions logger = logging.getLogger(__name__) logger.debug("*********** scos-sensor/models/task.py ****************") diff --git a/src/tasks/models/task_result.py b/src/tasks/models/task_result.py index 9fc1595f..b004bc69 100644 --- a/src/tasks/models/task_result.py +++ b/src/tasks/models/task_result.py @@ -7,7 +7,7 @@ from django.utils import timezone from schedule.models import ScheduleEntry -from sensor.settings import MAX_DISK_USAGE +from django.conf import settings from tasks.consts import MAX_DETAIL_LEN UTC = timezone.timezone.utc @@ -66,7 +66,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # Allow Swapping max_disk_usage for testing - self.max_disk_usage = MAX_DISK_USAGE + self.max_disk_usage = settings.MAX_DISK_USAGE def save(self): """Limit disk usage to MAX_DISK_USAGE by removing oldest result.""" diff --git a/src/tox.ini b/src/tox.ini index 73313df3..fe7f83ce 100644 --- a/src/tox.ini +++ b/src/tox.ini @@ -23,6 +23,8 @@ setenv = PATH_TO_CLIENT_CERT=test/sensor01.pem PATH_TO_VERIFY_CERT=test/scos_test_ca.crt SWITCH_CONFIGS_DIR=../configs/switches + SIGAN_MODULE=scos_actions.hardware.mocks.mock_sigan + SIGAN_CLASS=MockSignalAnalyzer [testenv:coverage] description = Run tests with pytest and generate coverage report From 50d61ad3895e6d27ce154a19bf33bfa0863aa9a6 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 12:40:30 -0700 Subject: [PATCH 072/255] add logging to settings. Fix actions_loader import. --- initialization/tests/test_initialization.py | 9 +++ src/sensor/action_loader.py | 78 +++++++++++++++++++++ src/sensor/migration_settings.py | 1 + src/sensor/runtime_settings.py | 2 +- src/sensor/settings.py | 5 ++ 5 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 initialization/tests/test_initialization.py create mode 100644 src/sensor/action_loader.py diff --git a/initialization/tests/test_initialization.py b/initialization/tests/test_initialization.py new file mode 100644 index 00000000..9d1deb32 --- /dev/null +++ b/initialization/tests/test_initialization.py @@ -0,0 +1,9 @@ +from initialization import load_preselector + +def test_load_preselector(): + preselector = load_preselector( + {"name": "test", "base_url": "http://127.0.0.1"}, + "its_preselector.web_relay_preselector", + "WebRelayPreselector", + ) + assert preselector is not None \ No newline at end of file diff --git a/src/sensor/action_loader.py b/src/sensor/action_loader.py new file mode 100644 index 00000000..eb4f405e --- /dev/null +++ b/src/sensor/action_loader.py @@ -0,0 +1,78 @@ +import importlib +import json +import logging +import pkgutil +import os +import shutil +from django.conf import settings +from scos_actions.actions import action_classes +from scos_actions.discover import test_actions +from scos_actions.discover import init + +logger = logging.getLogger(__name__) + +def copy_driver_files(driver_dir): + """Copy driver files where they need to go""" + for root, dirs, files in os.walk(driver_dir): + for filename in files: + name_without_ext, ext = os.path.splitext(filename) + if ext.lower() == ".json": + json_data = {} + file_path = os.path.join(root, filename) + with open(file_path) as json_file: + json_data = json.load(json_file) + if type(json_data) == dict and "scos_files" in json_data: + scos_files = json_data["scos_files"] + for scos_file in scos_files: + source_path = os.path.join( + driver_dir, scos_file["source_path"] + ) + if not os.path.isfile(source_path): + logger.error(f"Unable to find file at {source_path}") + continue + dest_path = scos_file["dest_path"] + dest_dir = os.path.dirname(dest_path) + try: + if not os.path.isdir(dest_dir): + os.makedirs(dest_dir) + logger.debug(f"copying {source_path} to {dest_path}") + shutil.copyfile(source_path, dest_path) + except Exception as e: + logger.error(f"Failed to copy {source_path} to {dest_path}") + logger.error(e) + + +def load_actions(mock_sigan, running_tests, driver_dir, action_dir): + logger.debug("********** Initializing actions **********") + + copy_driver_files(driver_dir) # copy driver files before loading plugins + + discovered_plugins = { + name: importlib.import_module(name) + for finder, name, ispkg in pkgutil.iter_modules() + if name.startswith("scos_") and name != "scos_actions" + } + logger.debug(discovered_plugins) + action_types = {} + action_types.update(action_classes) + actions = {} + if mock_sigan or running_tests: + for name, action in test_actions.items(): + logger.debug("test_action: " + name + "=" + str(action)) + else: + for name, module in discovered_plugins.items(): + logger.debug("Looking for actions in " + name + ": " + str(module)) + discover = importlib.import_module(name + ".discover") + if hasattr(discover, "actions"): + for name, action in discover.actions.items(): + logger.debug("action: " + name + "=" + str(action)) + actions[name] = action + if hasattr(discover, "action_types") and discover.action_types is not None: + action_types.update(discover.action_types) + + logger.debug(f"Loading actions in {action_dir}") + yaml_actions, yaml_test_actions = init(action_classes = action_types, yaml_dir=settings.ACTIONS_DIR) + actions.update(yaml_actions) + logger.debug("Finished loading and registering actions") + return actions + diff --git a/src/sensor/migration_settings.py b/src/sensor/migration_settings.py index 22be3b05..d9f50961 100644 --- a/src/sensor/migration_settings.py +++ b/src/sensor/migration_settings.py @@ -22,6 +22,7 @@ from django.core.management.utils import get_random_secret_key from environs import Env +from .action_loader import load_actions from src.sensor.action_loader import load_actions env = Env() diff --git a/src/sensor/runtime_settings.py b/src/sensor/runtime_settings.py index 0bb0d272..3bbeaa47 100644 --- a/src/sensor/runtime_settings.py +++ b/src/sensor/runtime_settings.py @@ -16,7 +16,7 @@ import os import sys from os import path -from action_loader import load_actions +from .action_loader import load_actions from cryptography.fernet import Fernet from django.core.management.utils import get_random_secret_key from environs import Env diff --git a/src/sensor/settings.py b/src/sensor/settings.py index a3fc082e..d81fd9eb 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -14,14 +14,18 @@ """ +import logging import os import sys from os import path +from .action_loader import load_actions from cryptography.fernet import Fernet from django.core.management.utils import get_random_secret_key from environs import Env +logger = logging.getLogger(__name__) +logger.debug("Initializing scos-sensor settings.") env = Env() # See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ @@ -440,6 +444,7 @@ SIGAN_POWER_CYCLE_STATES = env("SIGAN_POWER_CYCLE_STATES", default=None) SIGAN_POWER_SWITCH = env("SIGAN_POWER_SWITCH", default=None) MAX_FAILURES = env("MAX_FAILURES", default=2) +actions = load_actions(MOCK_SIGAN, RUNNING_TESTS, DRIVERS_DIR, ACTIONS_DIR) From cb610235c43c631d4da552a7cf61f935cb4dfdb6 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 12:51:11 -0700 Subject: [PATCH 073/255] move action loading functions into settings. --- src/sensor/action_loader.py | 67 ------------------------------- src/sensor/settings.py | 80 ++++++++++++++++++++++++++++++++++++- 2 files changed, 78 insertions(+), 69 deletions(-) diff --git a/src/sensor/action_loader.py b/src/sensor/action_loader.py index eb4f405e..db26b9b2 100644 --- a/src/sensor/action_loader.py +++ b/src/sensor/action_loader.py @@ -4,75 +4,8 @@ import pkgutil import os import shutil -from django.conf import settings from scos_actions.actions import action_classes from scos_actions.discover import test_actions from scos_actions.discover import init logger = logging.getLogger(__name__) - -def copy_driver_files(driver_dir): - """Copy driver files where they need to go""" - for root, dirs, files in os.walk(driver_dir): - for filename in files: - name_without_ext, ext = os.path.splitext(filename) - if ext.lower() == ".json": - json_data = {} - file_path = os.path.join(root, filename) - with open(file_path) as json_file: - json_data = json.load(json_file) - if type(json_data) == dict and "scos_files" in json_data: - scos_files = json_data["scos_files"] - for scos_file in scos_files: - source_path = os.path.join( - driver_dir, scos_file["source_path"] - ) - if not os.path.isfile(source_path): - logger.error(f"Unable to find file at {source_path}") - continue - dest_path = scos_file["dest_path"] - dest_dir = os.path.dirname(dest_path) - try: - if not os.path.isdir(dest_dir): - os.makedirs(dest_dir) - logger.debug(f"copying {source_path} to {dest_path}") - shutil.copyfile(source_path, dest_path) - except Exception as e: - logger.error(f"Failed to copy {source_path} to {dest_path}") - logger.error(e) - - -def load_actions(mock_sigan, running_tests, driver_dir, action_dir): - logger.debug("********** Initializing actions **********") - - copy_driver_files(driver_dir) # copy driver files before loading plugins - - discovered_plugins = { - name: importlib.import_module(name) - for finder, name, ispkg in pkgutil.iter_modules() - if name.startswith("scos_") and name != "scos_actions" - } - logger.debug(discovered_plugins) - action_types = {} - action_types.update(action_classes) - actions = {} - if mock_sigan or running_tests: - for name, action in test_actions.items(): - logger.debug("test_action: " + name + "=" + str(action)) - else: - for name, module in discovered_plugins.items(): - logger.debug("Looking for actions in " + name + ": " + str(module)) - discover = importlib.import_module(name + ".discover") - if hasattr(discover, "actions"): - for name, action in discover.actions.items(): - logger.debug("action: " + name + "=" + str(action)) - actions[name] = action - if hasattr(discover, "action_types") and discover.action_types is not None: - action_types.update(discover.action_types) - - logger.debug(f"Loading actions in {action_dir}") - yaml_actions, yaml_test_actions = init(action_classes = action_types, yaml_dir=settings.ACTIONS_DIR) - actions.update(yaml_actions) - logger.debug("Finished loading and registering actions") - return actions - diff --git a/src/sensor/settings.py b/src/sensor/settings.py index d81fd9eb..b0009c49 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -13,17 +13,25 @@ Make sure runtime_settings.py and this stay in sync as needed. See entrypoints/api_entrypoints.sh """ - +import importlib +import json import logging import os +import pkgutil +import shutil import sys from os import path -from .action_loader import load_actions + from cryptography.fernet import Fernet from django.core.management.utils import get_random_secret_key from environs import Env + +from scos_actions.actions import action_classes +from scos_actions.discover import test_actions +from scos_actions.discover import init + logger = logging.getLogger(__name__) logger.debug("Initializing scos-sensor settings.") env = Env() @@ -444,6 +452,74 @@ SIGAN_POWER_CYCLE_STATES = env("SIGAN_POWER_CYCLE_STATES", default=None) SIGAN_POWER_SWITCH = env("SIGAN_POWER_SWITCH", default=None) MAX_FAILURES = env("MAX_FAILURES", default=2) + + +def copy_driver_files(driver_dir): + """Copy driver files where they need to go""" + for root, dirs, files in os.walk(driver_dir): + for filename in files: + name_without_ext, ext = os.path.splitext(filename) + if ext.lower() == ".json": + json_data = {} + file_path = os.path.join(root, filename) + with open(file_path) as json_file: + json_data = json.load(json_file) + if type(json_data) == dict and "scos_files" in json_data: + scos_files = json_data["scos_files"] + for scos_file in scos_files: + source_path = os.path.join( + driver_dir, scos_file["source_path"] + ) + if not os.path.isfile(source_path): + logger.error(f"Unable to find file at {source_path}") + continue + dest_path = scos_file["dest_path"] + dest_dir = os.path.dirname(dest_path) + try: + if not os.path.isdir(dest_dir): + os.makedirs(dest_dir) + logger.debug(f"copying {source_path} to {dest_path}") + shutil.copyfile(source_path, dest_path) + except Exception as e: + logger.error(f"Failed to copy {source_path} to {dest_path}") + logger.error(e) + + +def load_actions(mock_sigan, running_tests, driver_dir, action_dir): + logger.debug("********** Initializing actions **********") + + copy_driver_files(driver_dir) # copy driver files before loading plugins + + discovered_plugins = { + name: importlib.import_module(name) + for finder, name, ispkg in pkgutil.iter_modules() + if name.startswith("scos_") and name != "scos_actions" + } + logger.debug(discovered_plugins) + action_types = {} + action_types.update(action_classes) + actions = {} + if mock_sigan or running_tests: + for name, action in test_actions.items(): + logger.debug("test_action: " + name + "=" + str(action)) + else: + for name, module in discovered_plugins.items(): + logger.debug("Looking for actions in " + name + ": " + str(module)) + discover = importlib.import_module(name + ".discover") + if hasattr(discover, "actions"): + for name, action in discover.actions.items(): + logger.debug("action: " + name + "=" + str(action)) + actions[name] = action + if hasattr(discover, "action_types") and discover.action_types is not None: + action_types.update(discover.action_types) + + logger.debug(f"Loading actions in {action_dir}") + yaml_actions, yaml_test_actions = init(action_classes = action_types, yaml_dir=settings.ACTIONS_DIR) + actions.update(yaml_actions) + logger.debug("Finished loading and registering actions") + return actions + + actions = load_actions(MOCK_SIGAN, RUNNING_TESTS, DRIVERS_DIR, ACTIONS_DIR) From 2ea680b42bb1145414d1079020911595b6e9d236 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 12:55:31 -0700 Subject: [PATCH 074/255] uppercase settings ACTIONS. --- src/capabilities/__init__.py | 2 +- src/schedule/__init__.py | 2 +- src/schedule/models/schedule_entry.py | 2 +- src/schedule/serializers.py | 2 +- src/scheduler/tests/utils.py | 2 +- src/sensor/settings.py | 2 +- src/tasks/models/task.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/capabilities/__init__.py b/src/capabilities/__init__.py index eb84cf44..bfd5d3da 100644 --- a/src/capabilities/__init__.py +++ b/src/capabilities/__init__.py @@ -6,5 +6,5 @@ logger = logging.getLogger(__name__) logger.debug("********** Initializing capabilities **********") -actions_by_name = settings.actions +actions_by_name = settings.ACTIONS sensor_capabilities = capabilities diff --git a/src/schedule/__init__.py b/src/schedule/__init__.py index 5835226f..f1e4ac17 100644 --- a/src/schedule/__init__.py +++ b/src/schedule/__init__.py @@ -3,7 +3,7 @@ from utils import get_summary from django.conf import settings -actions = settings.actions +actions = settings.ACTIONS def get_action_with_summary(action): """Given an action, return the string 'action_name - summary'.""" diff --git a/src/schedule/models/schedule_entry.py b/src/schedule/models/schedule_entry.py index 18002763..f977d6b8 100644 --- a/src/schedule/models/schedule_entry.py +++ b/src/schedule/models/schedule_entry.py @@ -9,7 +9,7 @@ from django.conf import settings from scheduler import utils -actions = settings.actions +actions = settings.ACTIONS logger = logging.getLogger(__name__) logger.debug( "************** scos-sensor/schedule/models/schedule_entry.py *****************" diff --git a/src/schedule/serializers.py b/src/schedule/serializers.py index 34f0a72a..9a2bd09d 100644 --- a/src/schedule/serializers.py +++ b/src/schedule/serializers.py @@ -13,7 +13,7 @@ from . import get_action_with_summary from .models import DEFAULT_PRIORITY, ScheduleEntry -actions = settings.actions +actions = settings.ACTIONS action_help = "[Required] The name of the action to be scheduled" priority_help = f"Lower number is higher priority (default={DEFAULT_PRIORITY})" CHOICES = [] diff --git a/src/scheduler/tests/utils.py b/src/scheduler/tests/utils.py index 7efb01f4..203a7a33 100644 --- a/src/scheduler/tests/utils.py +++ b/src/scheduler/tests/utils.py @@ -11,7 +11,7 @@ from django.conf import settings from scos_actions.hardware.mocks.mock_sigan import MockSignalAnalyzer -actions = settings.actions +actions = settings.ACTIONS BAD_ACTION_STR = "testing expected failure" logger = logging.getLogger(__name__) diff --git a/src/sensor/settings.py b/src/sensor/settings.py index b0009c49..32e18a78 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -520,7 +520,7 @@ def load_actions(mock_sigan, running_tests, driver_dir, action_dir): return actions -actions = load_actions(MOCK_SIGAN, RUNNING_TESTS, DRIVERS_DIR, ACTIONS_DIR) +ACTIONS = load_actions(MOCK_SIGAN, RUNNING_TESTS, DRIVERS_DIR, ACTIONS_DIR) diff --git a/src/tasks/models/task.py b/src/tasks/models/task.py index 9fc9546a..cded5af9 100644 --- a/src/tasks/models/task.py +++ b/src/tasks/models/task.py @@ -8,7 +8,7 @@ from collections import namedtuple from django.conf import settings -actions = settings.actions +actions = settings.ACTIONS logger = logging.getLogger(__name__) logger.debug("*********** scos-sensor/models/task.py ****************") From 5bdc21da27d7ff21a13cebbd63cb26f3307859c5 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 13:19:24 -0700 Subject: [PATCH 075/255] move initialization utilities. --- gunicorn/config.py | 11 +++++------ {initialization/tests => src}/__init__.py | 0 {initialization => src/initialization}/__init__.py | 1 - src/initialization/tests/__init__.py | 0 .../initialization}/tests/test_initialization.py | 2 +- src/sensor/settings.py | 2 -- 6 files changed, 6 insertions(+), 10 deletions(-) rename {initialization/tests => src}/__init__.py (100%) rename {initialization => src/initialization}/__init__.py (98%) create mode 100644 src/initialization/tests/__init__.py rename {initialization => src/initialization}/tests/test_initialization.py (83%) diff --git a/gunicorn/config.py b/gunicorn/config.py index 8eb34666..5583951b 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -3,12 +3,6 @@ import os import sys from environs import Env -from initialization import ( - load_preselector, - load_switches, - load_capabilities, - -) from multiprocessing import cpu_count from scos_actions.hardware.sensor import Sensor from scos_actions.metadata.utils import construct_geojson_point @@ -42,6 +36,11 @@ def post_worker_init(worker): django.setup() env = Env() from scheduler import scheduler + from initialization import ( + load_preselector, + load_switches, + load_capabilities, + ) sigan_module_setting = env("SIGAN_MODULE") sigan_module = importlib.import_module(sigan_module_setting) logger.info("Creating " + env("SIGAN_CLASS") + " from " + env("SIGAN_MODULE")) diff --git a/initialization/tests/__init__.py b/src/__init__.py similarity index 100% rename from initialization/tests/__init__.py rename to src/__init__.py diff --git a/initialization/__init__.py b/src/initialization/__init__.py similarity index 98% rename from initialization/__init__.py rename to src/initialization/__init__.py index bd5ebad3..2b3d19ad 100644 --- a/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -8,7 +8,6 @@ from its_preselector.controlbyweb_web_relay import ControlByWebWebRelay from scos_actions import utils -from scos_actions.metadata.utils import construct_geojson_point from scos_actions.signals import register_component_with_status logger = logging.getLogger(__name__) diff --git a/src/initialization/tests/__init__.py b/src/initialization/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/initialization/tests/test_initialization.py b/src/initialization/tests/test_initialization.py similarity index 83% rename from initialization/tests/test_initialization.py rename to src/initialization/tests/test_initialization.py index 9d1deb32..0225ab39 100644 --- a/initialization/tests/test_initialization.py +++ b/src/initialization/tests/test_initialization.py @@ -1,4 +1,4 @@ -from initialization import load_preselector +from src.initialization import load_preselector def test_load_preselector(): preselector = load_preselector( diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 32e18a78..d6323591 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -26,8 +26,6 @@ from cryptography.fernet import Fernet from django.core.management.utils import get_random_secret_key from environs import Env - - from scos_actions.actions import action_classes from scos_actions.discover import test_actions from scos_actions.discover import init From fb1efc92043b760c9a0124f057da2cfb13190301 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 13:26:27 -0700 Subject: [PATCH 076/255] remove actions app from settings and don't import django setting in intitialization code. --- src/initialization/__init__.py | 1 - src/sensor/settings.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 2b3d19ad..a86468dd 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -3,7 +3,6 @@ import json import logging from pathlib import Path -from from django.conf import settings from its_preselector.configuration_exception import ConfigurationException from its_preselector.controlbyweb_web_relay import ControlByWebWebRelay diff --git a/src/sensor/settings.py b/src/sensor/settings.py index d6323591..4d483621 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -208,7 +208,6 @@ "scheduler.apps.SchedulerConfig", "status.apps.StatusConfig", "sensor.apps.SensorConfig", # global settings/utils, etc - "actions.apps.ActionsConfig", ] MIDDLEWARE = [ @@ -512,7 +511,7 @@ def load_actions(mock_sigan, running_tests, driver_dir, action_dir): action_types.update(discover.action_types) logger.debug(f"Loading actions in {action_dir}") - yaml_actions, yaml_test_actions = init(action_classes = action_types, yaml_dir=settings.ACTIONS_DIR) + yaml_actions, yaml_test_actions = init(action_classes = action_types, yaml_dir=action_dir) actions.update(yaml_actions) logger.debug("Finished loading and registering actions") return actions From fe0c73aa34aee31b0fc91f675375c32e76006687 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 13:34:23 -0700 Subject: [PATCH 077/255] sync settings files. --- src/conftest.py | 5 +- src/sensor/migration_settings.py | 84 +++++++++++++++++++++++++++++--- src/sensor/runtime_settings.py | 84 ++++++++++++++++++++++++++++++-- 3 files changed, 162 insertions(+), 11 deletions(-) diff --git a/src/conftest.py b/src/conftest.py index 845f24ce..dc9f6995 100644 --- a/src/conftest.py +++ b/src/conftest.py @@ -10,6 +10,7 @@ from authentication.models import User from authentication.tests.utils import get_test_public_private_key from tasks.models import TaskResult +from scos_actions.hardware.sensor import Sensor from scos_actions.hardware.mocks.mock_sigan import MockSignalAnalyzer PRIVATE_KEY, PUBLIC_KEY = get_test_public_private_key() @@ -43,7 +44,9 @@ def testclock(): def test_scheduler(rf, testclock): """Instantiate test scheduler with fake request context and testclock.""" s = scheduler.scheduler.Scheduler() - s.signal_analyzer = MockSignalAnalyzer() + mock_sigan = MockSignalAnalyzer() + sensor = Sensor(signal_analyzer=mock_sigan) + s.sensor = sensor s.request = rf.post("mock://cburl/schedule") return s diff --git a/src/sensor/migration_settings.py b/src/sensor/migration_settings.py index d9f50961..8b1114ea 100644 --- a/src/sensor/migration_settings.py +++ b/src/sensor/migration_settings.py @@ -14,17 +14,22 @@ """ +import importlib +import json +import logging import os +import pkgutil +import shutil import sys from os import path - -from cryptography.fernet import Fernet from django.core.management.utils import get_random_secret_key from environs import Env +from scos_actions.actions import action_classes +from scos_actions.discover import test_actions +from scos_actions.discover import init -from .action_loader import load_actions -from src.sensor.action_loader import load_actions - +logger = logging.getLogger(__name__) +logger.debug("Initializing scos-sensor settings.") env = Env() # See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ @@ -425,4 +430,71 @@ SIGAN_POWER_CYCLE_STATES = env("SIGAN_POWER_CYCLE_STATES", default=None) SIGAN_POWER_SWITCH = env("SIGAN_POWER_SWITCH", default=None) MAX_FAILURES = env("MAX_FAILURES", default=2) -actions = load_actions(MOCK_SIGAN, RUNNING_TESTS, DRIVERS_DIR, ACTIONS_DIR) \ No newline at end of file + +def copy_driver_files(driver_dir): + """Copy driver files where they need to go""" + for root, dirs, files in os.walk(driver_dir): + for filename in files: + name_without_ext, ext = os.path.splitext(filename) + if ext.lower() == ".json": + json_data = {} + file_path = os.path.join(root, filename) + with open(file_path) as json_file: + json_data = json.load(json_file) + if type(json_data) == dict and "scos_files" in json_data: + scos_files = json_data["scos_files"] + for scos_file in scos_files: + source_path = os.path.join( + driver_dir, scos_file["source_path"] + ) + if not os.path.isfile(source_path): + logger.error(f"Unable to find file at {source_path}") + continue + dest_path = scos_file["dest_path"] + dest_dir = os.path.dirname(dest_path) + try: + if not os.path.isdir(dest_dir): + os.makedirs(dest_dir) + logger.debug(f"copying {source_path} to {dest_path}") + shutil.copyfile(source_path, dest_path) + except Exception as e: + logger.error(f"Failed to copy {source_path} to {dest_path}") + logger.error(e) + + +def load_actions(mock_sigan, running_tests, driver_dir, action_dir): + logger.debug("********** Initializing actions **********") + + copy_driver_files(driver_dir) # copy driver files before loading plugins + + discovered_plugins = { + name: importlib.import_module(name) + for finder, name, ispkg in pkgutil.iter_modules() + if name.startswith("scos_") and name != "scos_actions" + } + logger.debug(discovered_plugins) + action_types = {} + action_types.update(action_classes) + actions = {} + if mock_sigan or running_tests: + for name, action in test_actions.items(): + logger.debug("test_action: " + name + "=" + str(action)) + else: + for name, module in discovered_plugins.items(): + logger.debug("Looking for actions in " + name + ": " + str(module)) + discover = importlib.import_module(name + ".discover") + if hasattr(discover, "actions"): + for name, action in discover.actions.items(): + logger.debug("action: " + name + "=" + str(action)) + actions[name] = action + if hasattr(discover, "action_types") and discover.action_types is not None: + action_types.update(discover.action_types) + + logger.debug(f"Loading actions in {action_dir}") + yaml_actions, yaml_test_actions = init(action_classes = action_types, yaml_dir=action_dir) + actions.update(yaml_actions) + logger.debug("Finished loading and registering actions") + return actions + + +ACTIONS = load_actions(MOCK_SIGAN, RUNNING_TESTS, DRIVERS_DIR, ACTIONS_DIR) \ No newline at end of file diff --git a/src/sensor/runtime_settings.py b/src/sensor/runtime_settings.py index 3bbeaa47..cf9a77e0 100644 --- a/src/sensor/runtime_settings.py +++ b/src/sensor/runtime_settings.py @@ -12,17 +12,26 @@ run in docker. Make sure migration_settings.py and this stay in sync as needed. See entrypoints/api_entrypoints.sh """ - +import importlib +import json +import logging import os +import pkgutil +import shutil import sys from os import path -from .action_loader import load_actions + + from cryptography.fernet import Fernet from django.core.management.utils import get_random_secret_key from environs import Env +from scos_actions.actions import action_classes +from scos_actions.discover import test_actions +from scos_actions.discover import init +logger = logging.getLogger(__name__) +logger.debug("Initializing scos-sensor settings.") env = Env() - # See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ # Build paths inside the project like this: path.join(BASE_DIR, ...) @@ -431,4 +440,71 @@ SIGAN_POWER_CYCLE_STATES = env("SIGAN_POWER_CYCLE_STATES", default=None) SIGAN_POWER_SWITCH = env("SIGAN_POWER_SWITCH", default=None) MAX_FAILURES = env("MAX_FAILURES", default=2) -actions = load_actions(MOCK_SIGAN, RUNNING_TESTS, DRIVERS_DIR, ACTIONS_DIR) \ No newline at end of file + +def copy_driver_files(driver_dir): + """Copy driver files where they need to go""" + for root, dirs, files in os.walk(driver_dir): + for filename in files: + name_without_ext, ext = os.path.splitext(filename) + if ext.lower() == ".json": + json_data = {} + file_path = os.path.join(root, filename) + with open(file_path) as json_file: + json_data = json.load(json_file) + if type(json_data) == dict and "scos_files" in json_data: + scos_files = json_data["scos_files"] + for scos_file in scos_files: + source_path = os.path.join( + driver_dir, scos_file["source_path"] + ) + if not os.path.isfile(source_path): + logger.error(f"Unable to find file at {source_path}") + continue + dest_path = scos_file["dest_path"] + dest_dir = os.path.dirname(dest_path) + try: + if not os.path.isdir(dest_dir): + os.makedirs(dest_dir) + logger.debug(f"copying {source_path} to {dest_path}") + shutil.copyfile(source_path, dest_path) + except Exception as e: + logger.error(f"Failed to copy {source_path} to {dest_path}") + logger.error(e) + + +def load_actions(mock_sigan, running_tests, driver_dir, action_dir): + logger.debug("********** Initializing actions **********") + + copy_driver_files(driver_dir) # copy driver files before loading plugins + + discovered_plugins = { + name: importlib.import_module(name) + for finder, name, ispkg in pkgutil.iter_modules() + if name.startswith("scos_") and name != "scos_actions" + } + logger.debug(discovered_plugins) + action_types = {} + action_types.update(action_classes) + actions = {} + if mock_sigan or running_tests: + for name, action in test_actions.items(): + logger.debug("test_action: " + name + "=" + str(action)) + else: + for name, module in discovered_plugins.items(): + logger.debug("Looking for actions in " + name + ": " + str(module)) + discover = importlib.import_module(name + ".discover") + if hasattr(discover, "actions"): + for name, action in discover.actions.items(): + logger.debug("action: " + name + "=" + str(action)) + actions[name] = action + if hasattr(discover, "action_types") and discover.action_types is not None: + action_types.update(discover.action_types) + + logger.debug(f"Loading actions in {action_dir}") + yaml_actions, yaml_test_actions = init(action_classes = action_types, yaml_dir=action_dir) + actions.update(yaml_actions) + logger.debug("Finished loading and registering actions") + return actions + + +ACTIONS = load_actions(MOCK_SIGAN, RUNNING_TESTS, DRIVERS_DIR, ACTIONS_DIR) \ No newline at end of file From 65b5744dcf76b6a6e14a8f32691266bb332bfe65 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 13:52:15 -0700 Subject: [PATCH 078/255] move capabilities into settings. --- gunicorn/config.py | 3 +-- src/capabilities/__init__.py | 3 +-- src/initialization/__init__.py | 28 ---------------------------- src/sensor/migration_settings.py | 31 ++++++++++++++++++++++++++++++- src/sensor/runtime_settings.py | 31 ++++++++++++++++++++++++++++++- src/sensor/settings.py | 31 ++++++++++++++++++++++++++++++- 6 files changed, 92 insertions(+), 35 deletions(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index 5583951b..a6f9a2d8 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -39,7 +39,6 @@ def post_worker_init(worker): from initialization import ( load_preselector, load_switches, - load_capabilities, ) sigan_module_setting = env("SIGAN_MODULE") sigan_module = importlib.import_module(sigan_module_setting) @@ -47,10 +46,10 @@ def post_worker_init(worker): sigan_constructor = getattr(sigan_module, env("SIGAN_CLASS")) sigan = sigan_constructor() register_component_with_status.send(sigan, component=sigan) - capabilities = load_capabilities(env("SENSOR_DEFINITION_FILE")) switches = load_switches(env("SWITCH_CONFIGS_DIR")) for key, switch in switches: register_component_with_status(switch, component=switch) + capabilities = env("CAPABILITIES") preselector = load_preselector(env("PRESELECTOR_CONFIG"), env("PRESELEDTOR_MODULE"), env("PRESELECTOR_CLASS"), capabilities["sensor"]) register_component_with_status(preselector, component=preselector) if "location" in capabilities["sensor"]: diff --git a/src/capabilities/__init__.py b/src/capabilities/__init__.py index bfd5d3da..ded0dd98 100644 --- a/src/capabilities/__init__.py +++ b/src/capabilities/__init__.py @@ -1,10 +1,9 @@ import logging -from scos_actions.capabilities import capabilities from django.conf import settings logger = logging.getLogger(__name__) logger.debug("********** Initializing capabilities **********") actions_by_name = settings.ACTIONS -sensor_capabilities = capabilities +sensor_capabilities = settings.CAPABILITIES diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index a86468dd..32d929ce 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -60,32 +60,4 @@ def load_preselector(preselector_config, module, preselector_class_name, sensor_ return ps -def load_capabilities(sensor_definition_file): - capabilities = {} - SENSOR_DEFINITION_HASH = None - SENSOR_LOCATION = None - - logger.debug(f"Loading {sensor_definition_file}") - try: - capabilities["sensor"] = utils.load_from_json(sensor_definition_file) - except Exception as e: - logger.warning( - f"Failed to load sensor definition file: {sensor_definition_file}" - + "\nAn empty sensor definition will be used" - ) - capabilities["sensor"] = {"sensor_spec": {"id": "unknown"}} - capabilities["sensor"]["sensor_sha512"] = "UNKNOWN SENSOR DEFINITION" - - # Generate sensor definition file hash (SHA 512) - try: - if "sensor_sha512" not in capabilities["sensor"]: - sensor_def = json.dumps(capabilities["sensor"], sort_keys=True) - SENSOR_DEFINITION_HASH = hashlib.sha512(sensor_def.encode("UTF-8")).hexdigest() - capabilities["sensor"]["sensor_sha512"] = SENSOR_DEFINITION_HASH - except: - capabilities["sensor"]["sensor_sha512"] = "ERROR GENERATING HASH" - # SENSOR_DEFINITION_HASH is None, do not raise Exception - logger.exception(f"Unable to generate sensor definition hash") - - return capabilities diff --git a/src/sensor/migration_settings.py b/src/sensor/migration_settings.py index 8b1114ea..93925979 100644 --- a/src/sensor/migration_settings.py +++ b/src/sensor/migration_settings.py @@ -497,4 +497,33 @@ def load_actions(mock_sigan, running_tests, driver_dir, action_dir): return actions -ACTIONS = load_actions(MOCK_SIGAN, RUNNING_TESTS, DRIVERS_DIR, ACTIONS_DIR) \ No newline at end of file +def load_capabilities(sensor_definition_file): + capabilities = {} + SENSOR_DEFINITION_HASH = None + SENSOR_LOCATION = None + + logger.debug(f"Loading {sensor_definition_file}") + try: + capabilities["sensor"] = utils.load_from_json(sensor_definition_file) + except Exception as e: + logger.warning( + f"Failed to load sensor definition file: {sensor_definition_file}" + + "\nAn empty sensor definition will be used" + ) + capabilities["sensor"] = {"sensor_spec": {"id": "unknown"}} + capabilities["sensor"]["sensor_sha512"] = "UNKNOWN SENSOR DEFINITION" + + # Generate sensor definition file hash (SHA 512) + try: + if "sensor_sha512" not in capabilities["sensor"]: + sensor_def = json.dumps(capabilities["sensor"], sort_keys=True) + SENSOR_DEFINITION_HASH = hashlib.sha512(sensor_def.encode("UTF-8")).hexdigest() + capabilities["sensor"]["sensor_sha512"] = SENSOR_DEFINITION_HASH + except: + capabilities["sensor"]["sensor_sha512"] = "ERROR GENERATING HASH" + # SENSOR_DEFINITION_HASH is None, do not raise Exception + logger.exception(f"Unable to generate sensor definition hash") + + return capabilities + +ACTIONS = load_actions(MOCK_SIGAN, RUNNING_TESTS, DRIVERS_DIR, ACTIONS_DIR) diff --git a/src/sensor/runtime_settings.py b/src/sensor/runtime_settings.py index cf9a77e0..b7a88b6a 100644 --- a/src/sensor/runtime_settings.py +++ b/src/sensor/runtime_settings.py @@ -507,4 +507,33 @@ def load_actions(mock_sigan, running_tests, driver_dir, action_dir): return actions -ACTIONS = load_actions(MOCK_SIGAN, RUNNING_TESTS, DRIVERS_DIR, ACTIONS_DIR) \ No newline at end of file +def load_capabilities(sensor_definition_file): + capabilities = {} + SENSOR_DEFINITION_HASH = None + SENSOR_LOCATION = None + + logger.debug(f"Loading {sensor_definition_file}") + try: + capabilities["sensor"] = utils.load_from_json(sensor_definition_file) + except Exception as e: + logger.warning( + f"Failed to load sensor definition file: {sensor_definition_file}" + + "\nAn empty sensor definition will be used" + ) + capabilities["sensor"] = {"sensor_spec": {"id": "unknown"}} + capabilities["sensor"]["sensor_sha512"] = "UNKNOWN SENSOR DEFINITION" + + # Generate sensor definition file hash (SHA 512) + try: + if "sensor_sha512" not in capabilities["sensor"]: + sensor_def = json.dumps(capabilities["sensor"], sort_keys=True) + SENSOR_DEFINITION_HASH = hashlib.sha512(sensor_def.encode("UTF-8")).hexdigest() + capabilities["sensor"]["sensor_sha512"] = SENSOR_DEFINITION_HASH + except: + capabilities["sensor"]["sensor_sha512"] = "ERROR GENERATING HASH" + # SENSOR_DEFINITION_HASH is None, do not raise Exception + logger.exception(f"Unable to generate sensor definition hash") + + return capabilities + +ACTIONS = load_actions(MOCK_SIGAN, RUNNING_TESTS, DRIVERS_DIR, ACTIONS_DIR) diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 4d483621..fddf0551 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -517,8 +517,37 @@ def load_actions(mock_sigan, running_tests, driver_dir, action_dir): return actions -ACTIONS = load_actions(MOCK_SIGAN, RUNNING_TESTS, DRIVERS_DIR, ACTIONS_DIR) +def load_capabilities(sensor_definition_file): + capabilities = {} + SENSOR_DEFINITION_HASH = None + SENSOR_LOCATION = None + + logger.debug(f"Loading {sensor_definition_file}") + try: + capabilities["sensor"] = utils.load_from_json(sensor_definition_file) + except Exception as e: + logger.warning( + f"Failed to load sensor definition file: {sensor_definition_file}" + + "\nAn empty sensor definition will be used" + ) + capabilities["sensor"] = {"sensor_spec": {"id": "unknown"}} + capabilities["sensor"]["sensor_sha512"] = "UNKNOWN SENSOR DEFINITION" + + # Generate sensor definition file hash (SHA 512) + try: + if "sensor_sha512" not in capabilities["sensor"]: + sensor_def = json.dumps(capabilities["sensor"], sort_keys=True) + SENSOR_DEFINITION_HASH = hashlib.sha512(sensor_def.encode("UTF-8")).hexdigest() + capabilities["sensor"]["sensor_sha512"] = SENSOR_DEFINITION_HASH + except: + capabilities["sensor"]["sensor_sha512"] = "ERROR GENERATING HASH" + # SENSOR_DEFINITION_HASH is None, do not raise Exception + logger.exception(f"Unable to generate sensor definition hash") + + return capabilities +ACTIONS = load_actions(MOCK_SIGAN, RUNNING_TESTS, DRIVERS_DIR, ACTIONS_DIR) +CAPABILITIES = load_capabilities(SENSOR_DEFINITION_FILE) From 4d5d3ccbd342314478a3a7799eb1b848c0fac4bd Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 13:57:28 -0700 Subject: [PATCH 079/255] add missing capabilities in settings files. --- src/sensor/migration_settings.py | 1 + src/sensor/runtime_settings.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/sensor/migration_settings.py b/src/sensor/migration_settings.py index 93925979..38855b9c 100644 --- a/src/sensor/migration_settings.py +++ b/src/sensor/migration_settings.py @@ -527,3 +527,4 @@ def load_capabilities(sensor_definition_file): return capabilities ACTIONS = load_actions(MOCK_SIGAN, RUNNING_TESTS, DRIVERS_DIR, ACTIONS_DIR) +CAPABILITIES = load_capabilities(SENSOR_DEFINITION_FILE) \ No newline at end of file diff --git a/src/sensor/runtime_settings.py b/src/sensor/runtime_settings.py index b7a88b6a..3c0229eb 100644 --- a/src/sensor/runtime_settings.py +++ b/src/sensor/runtime_settings.py @@ -537,3 +537,4 @@ def load_capabilities(sensor_definition_file): return capabilities ACTIONS = load_actions(MOCK_SIGAN, RUNNING_TESTS, DRIVERS_DIR, ACTIONS_DIR) +CAPABILITIES = load_capabilities(SENSOR_DEFINITION_FILE) \ No newline at end of file From 87bf2e93d559ce25c07d274751df29fb1f7bf25d Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 14:18:00 -0700 Subject: [PATCH 080/255] remove refs to scos_actions.capabilities --- src/handlers/location_handler.py | 7 ++-- src/handlers/tests/test_handlers.py | 64 ++++++++++++++--------------- src/sensor/settings.py | 2 + 3 files changed, 37 insertions(+), 36 deletions(-) diff --git a/src/handlers/location_handler.py b/src/handlers/location_handler.py index e3a8d8cf..28808952 100644 --- a/src/handlers/location_handler.py +++ b/src/handlers/location_handler.py @@ -1,7 +1,5 @@ -from scos_actions.capabilities import capabilities - from status.models import GPS_LOCATION_DESCRIPTION, Location - +from django.conf import settings def location_action_completed_callback(sender, **kwargs): """Update database and capabilities when GPS is synced or database is updated""" @@ -31,12 +29,13 @@ def location_action_completed_callback(sender, **kwargs): def db_location_updated(sender, **kwargs): instance = kwargs["instance"] + capabilities = settings.CAPABILITIES if isinstance(instance, Location) and instance.active: if ( "location" not in capabilities["sensor"] or capabilities["sensor"]["location"] is None ): - capabilities["sensor"]["location"] = {} + capabilities["sensor"]["location"] = {} capabilities["sensor"]["location"]["x"] = instance.longitude capabilities["sensor"]["location"]["y"] = instance.latitude capabilities["sensor"]["location"]["z"] = instance.height diff --git a/src/handlers/tests/test_handlers.py b/src/handlers/tests/test_handlers.py index 1f02e981..35449db9 100644 --- a/src/handlers/tests/test_handlers.py +++ b/src/handlers/tests/test_handlers.py @@ -1,14 +1,14 @@ import pytest from django import conf -from scos_actions.capabilities import capabilities +from django.conf.settings import CAPABILITIES from status.models import Location @pytest.mark.django_db def test_db_location_update_handler(): - capabilities["sensor"] = {} - capabilities["sensor"]["location"] = {} + CAPABILITIES["sensor"] = {} + CAPABILITIES["sensor"]["location"] = {} location = Location() location.gps = False location.height = 10 @@ -17,15 +17,15 @@ def test_db_location_update_handler(): location.description = "test" location.active = True location.save() - assert capabilities["sensor"]["location"]["x"] == 100 - assert capabilities["sensor"]["location"]["y"] == -1 - assert capabilities["sensor"]["location"]["z"] == 10 - assert capabilities["sensor"]["location"]["description"] == "test" + assert CAPABILITIES["sensor"]["location"]["x"] == 100 + assert CAPABILITIES["sensor"]["location"]["y"] == -1 + assert CAPABILITIES["sensor"]["location"]["z"] == 10 + assert CAPABILITIES["sensor"]["location"]["description"] == "test" @pytest.mark.django_db def test_db_location_update_handler_current_location_none(): - capabilities["sensor"] = {} + CAPABILITIES["sensor"] = {} capabilities["sensor"]["location"] = None location = Location() location.gps = False @@ -53,29 +53,29 @@ def test_db_location_update_handler_not_active(): location.active = False location.description = "test" location.save() - assert len(capabilities["sensor"]["location"]) == 0 + assert len(CAPABILITIES["sensor"]["location"]) == 0 @pytest.mark.django_db def test_db_location_update_handler_no_description(): - capabilities["sensor"] = {} - capabilities["sensor"]["location"] = {} + CAPABILITIES["sensor"] = {} + CAPABILITIES["sensor"]["location"] = {} location = Location() location.gps = False location.height = 10 location.longitude = 100 location.latitude = -1 location.save() - assert capabilities["sensor"]["location"]["x"] == 100 - assert capabilities["sensor"]["location"]["y"] == -1 - assert capabilities["sensor"]["location"]["z"] == 10 - assert capabilities["sensor"]["location"]["description"] == "" + assert CAPABILITIES["sensor"]["location"]["x"] == 100 + assert CAPABILITIES["sensor"]["location"]["y"] == -1 + assert CAPABILITIES["sensor"]["location"]["z"] == 10 + assert CAPABILITIES["sensor"]["location"]["description"] == "" @pytest.mark.django_db def test_db_location_deleted_handler(): - capabilities["sensor"] = {} - capabilities["sensor"]["location"] = {} + CAPABILITIES["sensor"] = {} + CAPABILITIES["sensor"]["location"] = {} location = Location() location.gps = False location.height = 10 @@ -84,18 +84,18 @@ def test_db_location_deleted_handler(): location.description = "test" location.active = True location.save() - assert capabilities["sensor"]["location"]["x"] == 100 - assert capabilities["sensor"]["location"]["y"] == -1 - assert capabilities["sensor"]["location"]["z"] == 10 - assert capabilities["sensor"]["location"]["description"] == "test" + assert CAPABILITIES["sensor"]["location"]["x"] == 100 + assert CAPABILITIES["sensor"]["location"]["y"] == -1 + assert CAPABILITIES["sensor"]["location"]["z"] == 10 + assert CAPABILITIES["sensor"]["location"]["description"] == "test" location.delete() - assert capabilities["sensor"]["location"] is None + assert CAPABILITIES["sensor"]["location"] is None @pytest.mark.django_db def test_db_location_deleted_inactive_handler(): - capabilities["sensor"] = {} - capabilities["sensor"]["location"] = {} + CAPABILITIES["sensor"] = {} + CAPABILITIES["sensor"]["location"] = {} location = Location() location.gps = False location.height = 10 @@ -104,13 +104,13 @@ def test_db_location_deleted_inactive_handler(): location.description = "test" location.active = True location.save() - assert capabilities["sensor"]["location"]["x"] == 100 - assert capabilities["sensor"]["location"]["y"] == -1 - assert capabilities["sensor"]["location"]["z"] == 10 - assert capabilities["sensor"]["location"]["description"] == "test" + assert CAPABILITIES["sensor"]["location"]["x"] == 100 + assert CAPABILITIES["sensor"]["location"]["y"] == -1 + assert CAPABILITIES["sensor"]["location"]["z"] == 10 + assert CAPABILITIES["sensor"]["location"]["description"] == "test" location.active = False location.delete() - assert capabilities["sensor"]["location"]["x"] == 100 - assert capabilities["sensor"]["location"]["y"] == -1 - assert capabilities["sensor"]["location"]["z"] == 10 - assert capabilities["sensor"]["location"]["description"] == "test" + assert CAPABILITIES["sensor"]["location"]["x"] == 100 + assert CAPABILITIES["sensor"]["location"]["y"] == -1 + assert CAPABILITIES["sensor"]["location"]["z"] == 10 + assert CAPABILITIES["sensor"]["location"]["description"] == "test" diff --git a/src/sensor/settings.py b/src/sensor/settings.py index fddf0551..196d4199 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -14,6 +14,7 @@ """ import importlib +import hashlib import json import logging import os @@ -548,6 +549,7 @@ def load_capabilities(sensor_definition_file): ACTIONS = load_actions(MOCK_SIGAN, RUNNING_TESTS, DRIVERS_DIR, ACTIONS_DIR) CAPABILITIES = load_capabilities(SENSOR_DEFINITION_FILE) +SENSOR_DEFINITION_HASH = CAPABILITIES["sensor"]["sensor_sha512"] From 25185fd34991351b9f84ce18f2e513df51de2460 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 14:25:57 -0700 Subject: [PATCH 081/255] syntax. --- src/handlers/location_handler.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/handlers/location_handler.py b/src/handlers/location_handler.py index 28808952..bc3f2e7a 100644 --- a/src/handlers/location_handler.py +++ b/src/handlers/location_handler.py @@ -35,16 +35,16 @@ def db_location_updated(sender, **kwargs): "location" not in capabilities["sensor"] or capabilities["sensor"]["location"] is None ): - capabilities["sensor"]["location"] = {} - capabilities["sensor"]["location"]["x"] = instance.longitude - capabilities["sensor"]["location"]["y"] = instance.latitude - capabilities["sensor"]["location"]["z"] = instance.height - capabilities["sensor"]["location"]["gps"] = instance.gps - capabilities["sensor"]["location"]["description"] = instance.description + capabilities["sensor"]["location"] = {} + capabilities["sensor"]["location"]["x"] = instance.longitude + capabilities["sensor"]["location"]["y"] = instance.latitude + capabilities["sensor"]["location"]["z"] = instance.height + capabilities["sensor"]["location"]["gps"] = instance.gps + capabilities["sensor"]["location"]["description"] = instance.description def db_location_deleted(sender, **kwargs): instance = kwargs["instance"] if isinstance(instance, Location): - if "location" in capabilities["sensor"] and instance.active: - capabilities["sensor"]["location"] = None + if "location" in settings.CAPABILITIES["sensor"] and instance.active: + settings.CAPABILITIES["sensor"]["location"] = None From d0a5a239c6cd0e1353c413f41c78f8a2d11134a0 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 14:42:24 -0700 Subject: [PATCH 082/255] import django settings in gunicorn/config.py --- gunicorn/config.py | 3 ++- src/sensor/settings.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index a6f9a2d8..5b8a49b4 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -40,13 +40,14 @@ def post_worker_init(worker): load_preselector, load_switches, ) + from django.conf import settings sigan_module_setting = env("SIGAN_MODULE") sigan_module = importlib.import_module(sigan_module_setting) logger.info("Creating " + env("SIGAN_CLASS") + " from " + env("SIGAN_MODULE")) sigan_constructor = getattr(sigan_module, env("SIGAN_CLASS")) sigan = sigan_constructor() register_component_with_status.send(sigan, component=sigan) - switches = load_switches(env("SWITCH_CONFIGS_DIR")) + switches = load_switches(settings.SWITCH_CONFIGS_DIR) for key, switch in switches: register_component_with_status(switch, component=switch) capabilities = env("CAPABILITIES") diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 196d4199..23ab24ff 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -451,7 +451,6 @@ SIGAN_POWER_SWITCH = env("SIGAN_POWER_SWITCH", default=None) MAX_FAILURES = env("MAX_FAILURES", default=2) - def copy_driver_files(driver_dir): """Copy driver files where they need to go""" for root, dirs, files in os.walk(driver_dir): From 3fb91ff7b8905f73a37835973cf5c1e77048e33e Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 14:52:41 -0700 Subject: [PATCH 083/255] remove migration and runtime settings. Make switch_configs_dir a Path. --- entrypoints/api_entrypoint.sh | 5 - src/sensor/migration_settings.py | 530 ------------------------------ src/sensor/runtime_settings.py | 540 ------------------------------- src/sensor/settings.py | 10 +- 4 files changed, 3 insertions(+), 1082 deletions(-) delete mode 100644 src/sensor/migration_settings.py delete mode 100644 src/sensor/runtime_settings.py diff --git a/entrypoints/api_entrypoint.sh b/entrypoints/api_entrypoint.sh index d7c771cc..0fa7f5e0 100644 --- a/entrypoints/api_entrypoint.sh +++ b/entrypoints/api_entrypoint.sh @@ -10,16 +10,11 @@ function cleanup_demodb { trap cleanup_demodb SIGTERM trap cleanup_demodb SIGINT -# This is done to avoid loading actions and connecting to the sigan when migrations are applied and when -# the super user is created. -cp sensor/migration_settings.py sensor/settings.py echo "Starting Migrations" python3.8 manage.py migrate - echo "Creating superuser (if managed)" python3.8 /scripts/create_superuser.py -cp sensor/runtime_settings.py sensor/settings.py echo "Starting Gunicorn" exec gunicorn sensor.wsgi -c ../gunicorn/config.py & diff --git a/src/sensor/migration_settings.py b/src/sensor/migration_settings.py deleted file mode 100644 index 38855b9c..00000000 --- a/src/sensor/migration_settings.py +++ /dev/null @@ -1,530 +0,0 @@ -"""Django settings for scos-sensor project. - -Generated by 'django-admin startproject' using Django 1.11.3. - -For more information on this file, see -https://docs.djangoproject.com/en/1.11/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/1.11/ref/settings/ - -!!!!!!NOTE!!!!!: This file is used when scos-sensor runs migrations prior to running. runtime_settings.py is copied over -this file when scos sensor is run. Make sure runtime_settings.py and this stay in sync as needed. -See entrypoints/api_entrypoints.sh - -""" - -import importlib -import json -import logging -import os -import pkgutil -import shutil -import sys -from os import path -from django.core.management.utils import get_random_secret_key -from environs import Env -from scos_actions.actions import action_classes -from scos_actions.discover import test_actions -from scos_actions.discover import init - -logger = logging.getLogger(__name__) -logger.debug("Initializing scos-sensor settings.") -env = Env() - -# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ - -# Build paths inside the project like this: path.join(BASE_DIR, ...) -BASE_DIR = path.dirname(path.dirname(path.abspath(__file__))) -REPO_ROOT = path.dirname(BASE_DIR) -# REPO_ROOT = env("APP_ROOT", default=None) -# if not REPO_ROOT: -# REPO_ROOT = path.dirname(BASE_DIR) - -FQDN = env("FQDN", "fqdn.unset") - -DOCKER_TAG = env("DOCKER_TAG", default=None) -GIT_BRANCH = env("GIT_BRANCH", default=None) -SCOS_SENSOR_GIT_TAG = env("SCOS_SENSOR_GIT_TAG", default="Unknown") - -if not DOCKER_TAG or DOCKER_TAG == "latest": - VERSION_STRING = GIT_BRANCH -else: - VERSION_STRING = DOCKER_TAG - if VERSION_STRING.startswith("v"): - VERSION_STRING = VERSION_STRING[1:] - -STATIC_ROOT = path.join(BASE_DIR, "static") -STATIC_URL = "/static/" - -__cmd = path.split(sys.argv[0])[-1] -IN_DOCKER = env.bool("IN_DOCKER", default=False) -RUNNING_TESTS = "test" in __cmd -RUNNING_DEMO = env.bool("DEMO", default=False) -MOCK_SIGAN = env.bool("MOCK_SIGAN", default=False) or RUNNING_DEMO or RUNNING_TESTS -MOCK_SIGAN_RANDOM = env.bool("MOCK_SIGAN_RANDOM", default=False) - - -# Healthchecks - the existance of any of these indicates an unhealthy state -SDR_HEALTHCHECK_FILE = path.join(REPO_ROOT, "sdr_unhealthy") -SCHEDULER_HEALTHCHECK_FILE = path.join(REPO_ROOT, "scheduler_dead") - -LICENSE_URL = "https://github.com/NTIA/scos-sensor/blob/master/LICENSE.md" - -OPENAPI_FILE = path.join(REPO_ROOT, "docs", "openapi.json") - -CONFIG_DIR = path.join(REPO_ROOT, "configs") -ACTIONS_DIR = path.join(CONFIG_DIR, "actions") -DRIVERS_DIR = path.join(REPO_ROOT, "drivers") - -DEFAULT_CALIBRATION_FILE= path.join(CONFIG_DIR, "default_calibration.json") -# JSON configs -if path.exists(path.join(CONFIG_DIR, "sensor_calibration.json")): - SENSOR_CALIBRATION_FILE = path.join(CONFIG_DIR, "sensor_calibration.json") -else: - SENSOR_CALIBRATION_FILE = DEFAULT_CALIBRATION_FILE -if path.exists(path.join(CONFIG_DIR, "sigan_calibration.json")): - SIGAN_CALIBRATION_FILE = path.join(CONFIG_DIR, "sigan_calibration.json") -else: - SIGAN_CALIBRATION_FILE = DEFAULT_CALIBRATION_FILE - -if path.exists(path.join(CONFIG_DIR, "sensor_definition.json")): - SENSOR_DEFINITION_FILE = path.join(CONFIG_DIR, "sensor_definition.json") -MEDIA_ROOT = path.join(REPO_ROOT, "files") -PRESELECTOR_CONFIG = path.join(CONFIG_DIR, "preselector_config.json") - -# Cleanup any existing healtcheck files -try: - os.remove(SDR_HEALTHCHECK_FILE) -except OSError: - pass - -# As defined in SigMF -DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" - -# https://docs.djangoproject.com/en/2.2/ref/settings/#internal-ips If -# IN_DOCKER, the IP address that needs to go here to enable the debugging -# toolbar can change each time the bridge network is brought down. It's -# possible to extract the correct address from an incoming request, so if -# IN_DOCKER and DEBUG=true, then the `api_v1_root` view will insert the correct -# IP when the first request comes in. -INTERNAL_IPS = ["127.0.0.1"] - - -# See /env.template -if not IN_DOCKER or RUNNING_TESTS: - SECRET_KEY = get_random_secret_key() - DEBUG = True - ALLOWED_HOSTS = [] -else: - SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") - SECRET_KEY = env.str("SECRET_KEY") - DEBUG = env.bool("DEBUG", default=False) - ALLOWED_HOSTS = env.str("DOMAINS").split() + env.str("IPS").split() - POSTGRES_PASSWORD = env("POSTGRES_PASSWORD") - -SESSION_COOKIE_SECURE = IN_DOCKER -CSRF_COOKIE_SECURE = IN_DOCKER - -SESSION_COOKIE_AGE = 900 # seconds -SESSION_EXPIRE_SECONDS = 900 # seconds -SESSION_EXPIRE_AFTER_LAST_ACTIVITY = True -SESSION_TIMEOUT_REDIRECT = "/api/auth/logout/?next=/api/v1/" - -# Application definition - -API_TITLE = "SCOS Sensor API" - -API_DESCRIPTION = """A RESTful API for controlling a SCOS-compatible sensor. - -# Errors - -The API uses standard HTTP status codes to indicate the success or failure of -the API call. The body of the response will be JSON in the following format: - -## 400 Bad Request (Parse Error) - -```json -{ - "field_name": [ - "description of first error", - "description of second error", - ... - ] -} -``` - -## 400 Bad Request (Protected Error) - -```json -{ - "detail": "description of error", - "protected_objects": [ - "url_to_protected_item_1", - "url_to_protected_item_2", - ... - ] -} -``` - -## 409 Conflict (DB Integrity Error) - -```json -{ - "detail": "description of error" -} -``` - -""" - -INSTALLED_APPS = [ - "django.contrib.admin", - "django.contrib.auth", - "django.contrib.contenttypes", - "django.contrib.sessions", - "django.contrib.messages", - "django.contrib.staticfiles", - "rest_framework", - "rest_framework.authtoken", - "drf_yasg", # OpenAPI generator - # project-local apps - "authentication.apps.AuthenticationConfig", - "capabilities.apps.CapabilitiesConfig", - "handlers.apps.HandlersConfig", - "tasks.apps.TasksConfig", - "schedule.apps.ScheduleConfig", - "scheduler.apps.SchedulerConfig", - "status.apps.StatusConfig", - "sensor.apps.SensorConfig", # global settings/utils, etc -] - -MIDDLEWARE = [ - "django.middleware.security.SecurityMiddleware", - "django.contrib.sessions.middleware.SessionMiddleware", - "django_session_timeout.middleware.SessionTimeoutMiddleware", - "django.middleware.common.CommonMiddleware", - "django.middleware.csrf.CsrfViewMiddleware", - "django.contrib.auth.middleware.AuthenticationMiddleware", - "django.contrib.messages.middleware.MessageMiddleware", - "django.middleware.clickjacking.XFrameOptionsMiddleware", -] - - -ROOT_URLCONF = "sensor.urls" - -TEMPLATES = [ - { - "BACKEND": "django.template.backends.django.DjangoTemplates", - "DIRS": [path.join(BASE_DIR, "templates")], - "APP_DIRS": True, - "OPTIONS": { - "context_processors": [ - "django.template.context_processors.debug", - "django.contrib.auth.context_processors.auth", - "django.contrib.messages.context_processors.messages", - "django.template.context_processors.request", - ], - "builtins": ["sensor.templatetags.sensor_tags"], - }, - } -] - -WSGI_APPLICATION = "sensor.wsgi.application" - -# Django Rest Framework -# http://www.django-rest-framework.org/ - -REST_FRAMEWORK = { - "EXCEPTION_HANDLER": "sensor.exceptions.exception_handler", - "DEFAULT_AUTHENTICATION_CLASSES": ( - "rest_framework.authentication.SessionAuthentication", - ), - "DEFAULT_PERMISSION_CLASSES": ( - "rest_framework.permissions.IsAuthenticated", - "authentication.permissions.RequiredJWTRolePermissionOrIsSuperuser", - ), - "DEFAULT_RENDERER_CLASSES": ( - "rest_framework.renderers.JSONRenderer", - "rest_framework.renderers.BrowsableAPIRenderer", - ), - "DEFAULT_VERSIONING_CLASS": "rest_framework.versioning.URLPathVersioning", - "DEFAULT_VERSION": "v1", # this should always point to latest stable api - "ALLOWED_VERSIONS": ("v1",), - "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.LimitOffsetPagination", - "PAGE_SIZE": 10, - "DATETIME_FORMAT": DATETIME_FORMAT, - "DATETIME_INPUT_FORMATS": ("iso-8601",), - "COERCE_DECIMAL_TO_STRING": False, # DecimalField should return floats - "URL_FIELD_NAME": "self", # RFC 42867 -} - -AUTHENTICATION = env("AUTHENTICATION", default="") -if AUTHENTICATION == "JWT": - REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"] = ( - "authentication.auth.OAuthJWTAuthentication", - "rest_framework.authentication.SessionAuthentication", - ) -else: - REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"] = ( - "rest_framework.authentication.TokenAuthentication", - "rest_framework.authentication.SessionAuthentication", - ) - - -# https://drf-yasg.readthedocs.io/en/stable/settings.html -SWAGGER_SETTINGS = { - "SECURITY_DEFINITIONS": {}, - "APIS_SORTER": "alpha", - "OPERATIONS_SORTER": "method", - "VALIDATOR_URL": None, -} - -if AUTHENTICATION == "JWT": - SWAGGER_SETTINGS["SECURITY_DEFINITIONS"]["oAuth2JWT"] = { - "type": "oauth2", - "description": ( - "OAuth2 authentication using resource owner password flow." - "This is done by verifing JWT bearer tokens signed with RS256 algorithm." - "The JWT_PUBLIC_KEY_FILE setting controls the public key used for signature verification." - "Only authorizes users who have an authority matching the REQUIRED_ROLE setting." - "For more information, see https://tools.ietf.org/html/rfc6749#section-4.3." - ), - "flows": {"password": {"scopes": {}}}, # scopes are not used - } -else: - SWAGGER_SETTINGS["SECURITY_DEFINITIONS"]["token"] = { - "type": "apiKey", - "description": ( - "Tokens are automatically generated for all users. You can " - "view yours by going to your User Details view in the " - "browsable API at `/api/v1/users/me` and looking for the " - "`auth_token` key. New user accounts do not initially " - "have a password and so can not log in to the browsable API. " - "To set a password for a user (for testing purposes), an " - "admin can do that in the Sensor Configuration Portal, but " - "only the account's token should be stored and used for " - "general purpose API access. " - 'Example cURL call: `curl -kLsS -H "Authorization: Token' - ' 529c30e6e04b3b546f2e073e879b75fdfa147c15" ' - "https://localhost/api/v1`" - ), - "name": "Token", - "in": "header", - } - -# Database -# https://docs.djangoproject.com/en/1.11/ref/settings/#databases - -if RUNNING_TESTS or RUNNING_DEMO: - DATABASES = { - "default": { - "ENGINE": "django.db.backends.sqlite3", - # "NAME": ":memory:" - "NAME": "test.db", # temporary workaround for https://github.com/pytest-dev/pytest-django/issues/783 - } - } -else: - DATABASES = { - "default": { - "ENGINE": "django.db.backends.postgresql", - "NAME": "postgres", - "USER": "postgres", - "PASSWORD": env("POSTGRES_PASSWORD"), - "HOST": "db", - "PORT": "5432", - } - } - -if not IN_DOCKER: - DATABASES["default"]["HOST"] = "localhost" - -# Delete oldest TaskResult (and related acquisitions) of current ScheduleEntry if MAX_DISK_USAGE exceeded -MAX_DISK_USAGE = env.int("MAX_DISK_USAGE", default=85) # percent -# Display at most MAX_TASK_QUEUE upcoming tasks in /tasks/upcoming -MAX_TASK_QUEUE = 50 - -# Password validation -# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators - -AUTH_PASSWORD_VALIDATORS = [ - { - "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator" - }, - {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"}, - {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"}, - {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"}, -] - -AUTH_USER_MODEL = "authentication.User" - -# Internationalization -# https://docs.djangoproject.com/en/1.11/topics/i18n/ - -LANGUAGE_CODE = "en-us" -TIME_ZONE = "UTC" -USE_I18N = True -USE_L10N = True -USE_TZ = True - -LOGLEVEL = "DEBUG" if DEBUG else "INFO" - -LOGGING = { - "version": 1, - "disable_existing_loggers": False, - "formatters": {"simple": {"format": "[%(asctime)s] [%(levelname)s] %(message)s"}}, - "filters": {"require_debug_true": {"()": "django.utils.log.RequireDebugTrue"}}, - "handlers": {"console": {"class": "logging.StreamHandler", "formatter": "simple"}}, - "loggers": { - "actions": {"handlers": ["console"], "level": LOGLEVEL}, - "authentication": {"handlers": ["console"], "level": LOGLEVEL}, - "capabilities": {"handlers": ["console"], "level": LOGLEVEL}, - "handlers": {"handlers": ["console"], "level": LOGLEVEL}, - "schedule": {"handlers": ["console"], "level": LOGLEVEL}, - "scheduler": {"handlers": ["console"], "level": LOGLEVEL}, - "sensor": {"handlers": ["console"], "level": LOGLEVEL}, - "status": {"handlers": ["console"], "level": LOGLEVEL}, - "tasks": {"handlers": ["console"], "level": LOGLEVEL}, - "scos_actions": {"handlers": ["console"], "level": LOGLEVEL}, - "scos_usrp": {"handlers": ["console"], "level": LOGLEVEL}, - "scos_sensor_keysight": {"handlers": ["console"], "level": LOGLEVEL}, - "scos_tekrsa": {"handlers": ["console"], "level": LOGLEVEL}, - }, -} - - -CALLBACK_SSL_VERIFICATION = env.bool("CALLBACK_SSL_VERIFICATION", default=True) -# OAuth Password Flow Authentication -CALLBACK_AUTHENTICATION = env("CALLBACK_AUTHENTICATION", default="") -CLIENT_ID = env("CLIENT_ID", default="") -CLIENT_SECRET = env("CLIENT_SECRET", default="") -USER_NAME = CLIENT_ID -PASSWORD = CLIENT_SECRET - -OAUTH_TOKEN_URL = env("OAUTH_TOKEN_URL", default="") -CERTS_DIR = path.join(CONFIG_DIR, "certs") -# Sensor certificate with private key used as client cert -PATH_TO_CLIENT_CERT = env("PATH_TO_CLIENT_CERT", default="") -if PATH_TO_CLIENT_CERT != "": - PATH_TO_CLIENT_CERT = path.join(CERTS_DIR, PATH_TO_CLIENT_CERT) -# Trusted Certificate Authority certificate to verify authserver and callback URL server certificate -PATH_TO_VERIFY_CERT = env("PATH_TO_VERIFY_CERT", default="") -if PATH_TO_VERIFY_CERT != "": - PATH_TO_VERIFY_CERT = path.join(CERTS_DIR, PATH_TO_VERIFY_CERT) -# Public key to verify JWT token -PATH_TO_JWT_PUBLIC_KEY = env.str("PATH_TO_JWT_PUBLIC_KEY", default="") -if PATH_TO_JWT_PUBLIC_KEY != "": - PATH_TO_JWT_PUBLIC_KEY = path.join(CERTS_DIR, PATH_TO_JWT_PUBLIC_KEY) -# Required role from JWT token to access API -REQUIRED_ROLE = "ROLE_MANAGER" - -PRESELECTOR_CONFIG = env.str( - "PRESELECTOR_CONFIG", default=path.join(CONFIG_DIR, "preselector_config.json") -) -PRESELECTOR_MODULE = env.str( - "PRESELECTOR_MODULE", default="its_preselector.web_relay_preselector" -) -PRESELECTOR_CLASS = env.str("PRESELECTOR_CLASS", default="WebRelayPreselector") -SWITCH_CONFIGS_DIR = env.str( - "SWITCH_CONFIGS_DIR", default=path.join(CONFIG_DIR, "switches") -) -SIGAN_POWER_CYCLE_STATES = env("SIGAN_POWER_CYCLE_STATES", default=None) -SIGAN_POWER_SWITCH = env("SIGAN_POWER_SWITCH", default=None) -MAX_FAILURES = env("MAX_FAILURES", default=2) - -def copy_driver_files(driver_dir): - """Copy driver files where they need to go""" - for root, dirs, files in os.walk(driver_dir): - for filename in files: - name_without_ext, ext = os.path.splitext(filename) - if ext.lower() == ".json": - json_data = {} - file_path = os.path.join(root, filename) - with open(file_path) as json_file: - json_data = json.load(json_file) - if type(json_data) == dict and "scos_files" in json_data: - scos_files = json_data["scos_files"] - for scos_file in scos_files: - source_path = os.path.join( - driver_dir, scos_file["source_path"] - ) - if not os.path.isfile(source_path): - logger.error(f"Unable to find file at {source_path}") - continue - dest_path = scos_file["dest_path"] - dest_dir = os.path.dirname(dest_path) - try: - if not os.path.isdir(dest_dir): - os.makedirs(dest_dir) - logger.debug(f"copying {source_path} to {dest_path}") - shutil.copyfile(source_path, dest_path) - except Exception as e: - logger.error(f"Failed to copy {source_path} to {dest_path}") - logger.error(e) - - -def load_actions(mock_sigan, running_tests, driver_dir, action_dir): - logger.debug("********** Initializing actions **********") - - copy_driver_files(driver_dir) # copy driver files before loading plugins - - discovered_plugins = { - name: importlib.import_module(name) - for finder, name, ispkg in pkgutil.iter_modules() - if name.startswith("scos_") and name != "scos_actions" - } - logger.debug(discovered_plugins) - action_types = {} - action_types.update(action_classes) - actions = {} - if mock_sigan or running_tests: - for name, action in test_actions.items(): - logger.debug("test_action: " + name + "=" + str(action)) - else: - for name, module in discovered_plugins.items(): - logger.debug("Looking for actions in " + name + ": " + str(module)) - discover = importlib.import_module(name + ".discover") - if hasattr(discover, "actions"): - for name, action in discover.actions.items(): - logger.debug("action: " + name + "=" + str(action)) - actions[name] = action - if hasattr(discover, "action_types") and discover.action_types is not None: - action_types.update(discover.action_types) - - logger.debug(f"Loading actions in {action_dir}") - yaml_actions, yaml_test_actions = init(action_classes = action_types, yaml_dir=action_dir) - actions.update(yaml_actions) - logger.debug("Finished loading and registering actions") - return actions - - -def load_capabilities(sensor_definition_file): - capabilities = {} - SENSOR_DEFINITION_HASH = None - SENSOR_LOCATION = None - - logger.debug(f"Loading {sensor_definition_file}") - try: - capabilities["sensor"] = utils.load_from_json(sensor_definition_file) - except Exception as e: - logger.warning( - f"Failed to load sensor definition file: {sensor_definition_file}" - + "\nAn empty sensor definition will be used" - ) - capabilities["sensor"] = {"sensor_spec": {"id": "unknown"}} - capabilities["sensor"]["sensor_sha512"] = "UNKNOWN SENSOR DEFINITION" - - # Generate sensor definition file hash (SHA 512) - try: - if "sensor_sha512" not in capabilities["sensor"]: - sensor_def = json.dumps(capabilities["sensor"], sort_keys=True) - SENSOR_DEFINITION_HASH = hashlib.sha512(sensor_def.encode("UTF-8")).hexdigest() - capabilities["sensor"]["sensor_sha512"] = SENSOR_DEFINITION_HASH - except: - capabilities["sensor"]["sensor_sha512"] = "ERROR GENERATING HASH" - # SENSOR_DEFINITION_HASH is None, do not raise Exception - logger.exception(f"Unable to generate sensor definition hash") - - return capabilities - -ACTIONS = load_actions(MOCK_SIGAN, RUNNING_TESTS, DRIVERS_DIR, ACTIONS_DIR) -CAPABILITIES = load_capabilities(SENSOR_DEFINITION_FILE) \ No newline at end of file diff --git a/src/sensor/runtime_settings.py b/src/sensor/runtime_settings.py deleted file mode 100644 index 3c0229eb..00000000 --- a/src/sensor/runtime_settings.py +++ /dev/null @@ -1,540 +0,0 @@ -"""Django settings for scos-sensor project. - -Generated by 'django-admin startproject' using Django 1.11.3. - -For more information on this file, see -https://docs.djangoproject.com/en/1.11/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/1.11/ref/settings/ - -!!!!!!NOTE!!!!!: This file is used when scos-sensor runs in docker. migration_settings.py is used when migrations are -run in docker. Make sure migration_settings.py and this stay in sync as needed. See entrypoints/api_entrypoints.sh - -""" -import importlib -import json -import logging -import os -import pkgutil -import shutil -import sys -from os import path - - -from cryptography.fernet import Fernet -from django.core.management.utils import get_random_secret_key -from environs import Env -from scos_actions.actions import action_classes -from scos_actions.discover import test_actions -from scos_actions.discover import init - -logger = logging.getLogger(__name__) -logger.debug("Initializing scos-sensor settings.") -env = Env() -# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ - -# Build paths inside the project like this: path.join(BASE_DIR, ...) -BASE_DIR = path.dirname(path.dirname(path.abspath(__file__))) -REPO_ROOT = path.dirname(BASE_DIR) -# REPO_ROOT = env("APP_ROOT", default=None) -# if not REPO_ROOT: -# REPO_ROOT = path.dirname(BASE_DIR) - -FQDN = env("FQDN", "fqdn.unset") - -DOCKER_TAG = env("DOCKER_TAG", default=None) -GIT_BRANCH = env("GIT_BRANCH", default=None) -SCOS_SENSOR_GIT_TAG = env("SCOS_SENSOR_GIT_TAG", default="Unknown") - -if not DOCKER_TAG or DOCKER_TAG == "latest": - VERSION_STRING = GIT_BRANCH -else: - VERSION_STRING = DOCKER_TAG - if VERSION_STRING.startswith("v"): - VERSION_STRING = VERSION_STRING[1:] - -STATIC_ROOT = path.join(BASE_DIR, "static") -STATIC_URL = "/static/" - -__cmd = path.split(sys.argv[0])[-1] -IN_DOCKER = env.bool("IN_DOCKER", default=False) -RUNNING_TESTS = "test" in __cmd -RUNNING_DEMO = env.bool("DEMO", default=False) -MOCK_SIGAN = env.bool("MOCK_SIGAN", default=False) or RUNNING_DEMO or RUNNING_TESTS -MOCK_SIGAN_RANDOM = env.bool("MOCK_SIGAN_RANDOM", default=False) - - -# Healthchecks - the existance of any of these indicates an unhealthy state -SDR_HEALTHCHECK_FILE = path.join(REPO_ROOT, "sdr_unhealthy") -SCHEDULER_HEALTHCHECK_FILE = path.join(REPO_ROOT, "scheduler_dead") - -LICENSE_URL = "https://github.com/NTIA/scos-sensor/blob/master/LICENSE.md" - -OPENAPI_FILE = path.join(REPO_ROOT, "docs", "openapi.json") - -CONFIG_DIR = path.join(REPO_ROOT, "configs") -ACTIONS_DIR = path.join(CONFIG_DIR, "actions") -DRIVERS_DIR = path.join(REPO_ROOT, "drivers") - -DEFAULT_CALIBRATION_FILE= path.join(CONFIG_DIR, "default_calibration.json") -# JSON configs -if path.exists(path.join(CONFIG_DIR, "sensor_calibration.json")): - SENSOR_CALIBRATION_FILE = path.join(CONFIG_DIR, "sensor_calibration.json") -else: - SENSOR_CALIBRATION_FILE = DEFAULT_CALIBRATION_FILE -if path.exists(path.join(CONFIG_DIR, "sigan_calibration.json")): - SIGAN_CALIBRATION_FILE = path.join(CONFIG_DIR, "sigan_calibration.json") -else: - SIGAN_CALIBRATION_FILE = DEFAULT_CALIBRATION_FILE - -if path.exists(path.join(CONFIG_DIR, "sensor_definition.json")): - SENSOR_DEFINITION_FILE = path.join(CONFIG_DIR, "sensor_definition.json") -MEDIA_ROOT = path.join(REPO_ROOT, "files") -PRESELECTOR_CONFIG = path.join(CONFIG_DIR, "preselector_config.json") - -# Cleanup any existing healtcheck files -try: - os.remove(SDR_HEALTHCHECK_FILE) -except OSError: - pass - -# As defined in SigMF -DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" - -# https://docs.djangoproject.com/en/2.2/ref/settings/#internal-ips If -# IN_DOCKER, the IP address that needs to go here to enable the debugging -# toolbar can change each time the bridge network is brought down. It's -# possible to extract the correct address from an incoming request, so if -# IN_DOCKER and DEBUG=true, then the `api_v1_root` view will insert the correct -# IP when the first request comes in. -INTERNAL_IPS = ["127.0.0.1"] - -ENCRYPT_DATA_FILES = env.bool("ENCRYPT_DATA_FILES", default=True) - -# See /env.template -if not IN_DOCKER or RUNNING_TESTS: - SECRET_KEY = get_random_secret_key() - DEBUG = True - ALLOWED_HOSTS = [] - ENCRYPTION_KEY = Fernet.generate_key() - ASYNC_CALLBACK = False -else: - SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") - SECRET_KEY = env.str("SECRET_KEY") - DEBUG = env.bool("DEBUG", default=False) - ALLOWED_HOSTS = env.str("DOMAINS").split() + env.str("IPS").split() - POSTGRES_PASSWORD = env("POSTGRES_PASSWORD") - ENCRYPTION_KEY = env.str("ENCRYPTION_KEY") - ASYNC_CALLBACK = env.bool("ASYNC_CALLBACK", default=True) - -SESSION_COOKIE_SECURE = IN_DOCKER -CSRF_COOKIE_SECURE = IN_DOCKER -if IN_DOCKER: - SCOS_TMP = env.str("SCOS_TMP", default="/scos_tmp") -else: - SCOS_TMP = None - -SESSION_COOKIE_AGE = 900 # seconds -SESSION_EXPIRE_SECONDS = 900 # seconds -SESSION_EXPIRE_AFTER_LAST_ACTIVITY = True -SESSION_TIMEOUT_REDIRECT = "/api/auth/logout/?next=/api/v1/" - -# Application definition - -API_TITLE = "SCOS Sensor API" - -API_DESCRIPTION = """A RESTful API for controlling a SCOS-compatible sensor. - -# Errors - -The API uses standard HTTP status codes to indicate the success or failure of -the API call. The body of the response will be JSON in the following format: - -## 400 Bad Request (Parse Error) - -```json -{ - "field_name": [ - "description of first error", - "description of second error", - ... - ] -} -``` - -## 400 Bad Request (Protected Error) - -```json -{ - "detail": "description of error", - "protected_objects": [ - "url_to_protected_item_1", - "url_to_protected_item_2", - ... - ] -} -``` - -## 409 Conflict (DB Integrity Error) - -```json -{ - "detail": "description of error" -} -``` - -""" - -INSTALLED_APPS = [ - "django.contrib.admin", - "django.contrib.auth", - "django.contrib.contenttypes", - "django.contrib.sessions", - "django.contrib.messages", - "django.contrib.staticfiles", - "rest_framework", - "rest_framework.authtoken", - "drf_yasg", # OpenAPI generator - # project-local apps - "authentication.apps.AuthenticationConfig", - "capabilities.apps.CapabilitiesConfig", - "handlers.apps.HandlersConfig", - "tasks.apps.TasksConfig", - "schedule.apps.ScheduleConfig", - "scheduler.apps.SchedulerConfig", - "status.apps.StatusConfig", - "sensor.apps.SensorConfig", # global settings/utils, etc -] - -MIDDLEWARE = [ - "django.middleware.security.SecurityMiddleware", - "django.contrib.sessions.middleware.SessionMiddleware", - "django_session_timeout.middleware.SessionTimeoutMiddleware", - "django.middleware.common.CommonMiddleware", - "django.middleware.csrf.CsrfViewMiddleware", - "django.contrib.auth.middleware.AuthenticationMiddleware", - "django.contrib.messages.middleware.MessageMiddleware", - "django.middleware.clickjacking.XFrameOptionsMiddleware", -] - - -ROOT_URLCONF = "sensor.urls" - -TEMPLATES = [ - { - "BACKEND": "django.template.backends.django.DjangoTemplates", - "DIRS": [path.join(BASE_DIR, "templates")], - "APP_DIRS": True, - "OPTIONS": { - "context_processors": [ - "django.template.context_processors.debug", - "django.contrib.auth.context_processors.auth", - "django.contrib.messages.context_processors.messages", - "django.template.context_processors.request", - ], - "builtins": ["sensor.templatetags.sensor_tags"], - }, - } -] - -WSGI_APPLICATION = "sensor.wsgi.application" - -# Django Rest Framework -# http://www.django-rest-framework.org/ - -REST_FRAMEWORK = { - "EXCEPTION_HANDLER": "sensor.exceptions.exception_handler", - "DEFAULT_AUTHENTICATION_CLASSES": ( - "rest_framework.authentication.SessionAuthentication", - ), - "DEFAULT_PERMISSION_CLASSES": ( - "rest_framework.permissions.IsAuthenticated", - "authentication.permissions.RequiredJWTRolePermissionOrIsSuperuser", - ), - "DEFAULT_RENDERER_CLASSES": ( - "rest_framework.renderers.JSONRenderer", - "rest_framework.renderers.BrowsableAPIRenderer", - ), - "DEFAULT_VERSIONING_CLASS": "rest_framework.versioning.URLPathVersioning", - "DEFAULT_VERSION": "v1", # this should always point to latest stable api - "ALLOWED_VERSIONS": ("v1",), - "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.LimitOffsetPagination", - "PAGE_SIZE": 10, - "DATETIME_FORMAT": DATETIME_FORMAT, - "DATETIME_INPUT_FORMATS": ("iso-8601",), - "COERCE_DECIMAL_TO_STRING": False, # DecimalField should return floats - "URL_FIELD_NAME": "self", # RFC 42867 -} - -AUTHENTICATION = env("AUTHENTICATION", default="") -if AUTHENTICATION == "JWT": - REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"] = ( - "authentication.auth.OAuthJWTAuthentication", - "rest_framework.authentication.SessionAuthentication", - ) -else: - REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"] = ( - "rest_framework.authentication.TokenAuthentication", - "rest_framework.authentication.SessionAuthentication", - ) - - -# https://drf-yasg.readthedocs.io/en/stable/settings.html -SWAGGER_SETTINGS = { - "SECURITY_DEFINITIONS": {}, - "APIS_SORTER": "alpha", - "OPERATIONS_SORTER": "method", - "VALIDATOR_URL": None, -} - -if AUTHENTICATION == "JWT": - SWAGGER_SETTINGS["SECURITY_DEFINITIONS"]["oAuth2JWT"] = { - "type": "oauth2", - "description": ( - "OAuth2 authentication using resource owner password flow." - "This is done by verifing JWT bearer tokens signed with RS256 algorithm." - "The JWT_PUBLIC_KEY_FILE setting controls the public key used for signature verification." - "Only authorizes users who have an authority matching the REQUIRED_ROLE setting." - "For more information, see https://tools.ietf.org/html/rfc6749#section-4.3." - ), - "flows": {"password": {"scopes": {}}}, # scopes are not used - } -else: - SWAGGER_SETTINGS["SECURITY_DEFINITIONS"]["token"] = { - "type": "apiKey", - "description": ( - "Tokens are automatically generated for all users. You can " - "view yours by going to your User Details view in the " - "browsable API at `/api/v1/users/me` and looking for the " - "`auth_token` key. New user accounts do not initially " - "have a password and so can not log in to the browsable API. " - "To set a password for a user (for testing purposes), an " - "admin can do that in the Sensor Configuration Portal, but " - "only the account's token should be stored and used for " - "general purpose API access. " - 'Example cURL call: `curl -kLsS -H "Authorization: Token' - ' 529c30e6e04b3b546f2e073e879b75fdfa147c15" ' - "https://localhost/api/v1`" - ), - "name": "Token", - "in": "header", - } - -# Database -# https://docs.djangoproject.com/en/1.11/ref/settings/#databases - -if RUNNING_TESTS or RUNNING_DEMO: - DATABASES = { - "default": { - "ENGINE": "django.db.backends.sqlite3", - # "NAME": ":memory:" - "NAME": "test.db", # temporary workaround for https://github.com/pytest-dev/pytest-django/issues/783 - } - } -else: - DATABASES = { - "default": { - "ENGINE": "django.db.backends.postgresql", - "NAME": "postgres", - "USER": "postgres", - "PASSWORD": env("POSTGRES_PASSWORD"), - "HOST": "db", - "PORT": "5432", - } - } - -if not IN_DOCKER: - DATABASES["default"]["HOST"] = "localhost" - -# Delete oldest TaskResult (and related acquisitions) of current ScheduleEntry if MAX_DISK_USAGE exceeded -MAX_DISK_USAGE = env.int("MAX_DISK_USAGE", default=85) # percent -# Display at most MAX_TASK_QUEUE upcoming tasks in /tasks/upcoming -MAX_TASK_QUEUE = 50 - -# Password validation -# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators - -AUTH_PASSWORD_VALIDATORS = [ - { - "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator" - }, - {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"}, - {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"}, - {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"}, -] - -AUTH_USER_MODEL = "authentication.User" - -# Internationalization -# https://docs.djangoproject.com/en/1.11/topics/i18n/ - -LANGUAGE_CODE = "en-us" -TIME_ZONE = "UTC" -USE_I18N = True -USE_L10N = True -USE_TZ = True - -LOGLEVEL = "DEBUG" if DEBUG else "INFO" - -LOGGING = { - "version": 1, - "disable_existing_loggers": False, - "formatters": {"simple": {"format": "[%(asctime)s] [%(levelname)s] %(message)s"}}, - "filters": {"require_debug_true": {"()": "django.utils.log.RequireDebugTrue"}}, - "handlers": {"console": {"class": "logging.StreamHandler", "formatter": "simple"}}, - "loggers": { - "actions": {"handlers": ["console"], "level": LOGLEVEL}, - "authentication": {"handlers": ["console"], "level": LOGLEVEL}, - "capabilities": {"handlers": ["console"], "level": LOGLEVEL}, - "handlers": {"handlers": ["console"], "level": LOGLEVEL}, - "schedule": {"handlers": ["console"], "level": LOGLEVEL}, - "scheduler": {"handlers": ["console"], "level": LOGLEVEL}, - "sensor": {"handlers": ["console"], "level": LOGLEVEL}, - "status": {"handlers": ["console"], "level": LOGLEVEL}, - "tasks": {"handlers": ["console"], "level": LOGLEVEL}, - "scos_actions": {"handlers": ["console"], "level": LOGLEVEL}, - "scos_usrp": {"handlers": ["console"], "level": LOGLEVEL}, - "scos_sensor_keysight": {"handlers": ["console"], "level": LOGLEVEL}, - "scos_tekrsa": {"handlers": ["console"], "level": LOGLEVEL}, - }, -} - - -CALLBACK_SSL_VERIFICATION = env.bool("CALLBACK_SSL_VERIFICATION", default=True) -# OAuth Password Flow Authentication -CALLBACK_AUTHENTICATION = env("CALLBACK_AUTHENTICATION", default="") -CALLBACK_TIMEOUT = env.int("CALLBACK_TIMEOUT", default=3) -CLIENT_ID = env("CLIENT_ID", default="") -CLIENT_SECRET = env("CLIENT_SECRET", default="") -USER_NAME = CLIENT_ID -PASSWORD = CLIENT_SECRET - -OAUTH_TOKEN_URL = env("OAUTH_TOKEN_URL", default="") -CERTS_DIR = path.join(CONFIG_DIR, "certs") -# Sensor certificate with private key used as client cert -PATH_TO_CLIENT_CERT = env("PATH_TO_CLIENT_CERT", default="") -if PATH_TO_CLIENT_CERT != "": - PATH_TO_CLIENT_CERT = path.join(CERTS_DIR, PATH_TO_CLIENT_CERT) -# Trusted Certificate Authority certificate to verify authserver and callback URL server certificate -PATH_TO_VERIFY_CERT = env("PATH_TO_VERIFY_CERT", default="") -if PATH_TO_VERIFY_CERT != "": - PATH_TO_VERIFY_CERT = path.join(CERTS_DIR, PATH_TO_VERIFY_CERT) -# Public key to verify JWT token -PATH_TO_JWT_PUBLIC_KEY = env.str("PATH_TO_JWT_PUBLIC_KEY", default="") -if PATH_TO_JWT_PUBLIC_KEY != "": - PATH_TO_JWT_PUBLIC_KEY = path.join(CERTS_DIR, PATH_TO_JWT_PUBLIC_KEY) -# Required role from JWT token to access API -REQUIRED_ROLE = "ROLE_MANAGER" - -PRESELECTOR_CONFIG = env.str( - "PRESELECTOR_CONFIG", default=path.join(CONFIG_DIR, "preselector_config.json") -) -PRESELECTOR_MODULE = env.str( - "PRESELECTOR_MODULE", default="its_preselector.web_relay_preselector" -) -PRESELECTOR_CLASS = env.str("PRESELECTOR_CLASS", default="WebRelayPreselector") -SWITCH_CONFIGS_DIR = env.str( - "SWITCH_CONFIGS_DIR", default=path.join(CONFIG_DIR, "switches") -) -SIGAN_POWER_CYCLE_STATES = env("SIGAN_POWER_CYCLE_STATES", default=None) -SIGAN_POWER_SWITCH = env("SIGAN_POWER_SWITCH", default=None) -MAX_FAILURES = env("MAX_FAILURES", default=2) - -def copy_driver_files(driver_dir): - """Copy driver files where they need to go""" - for root, dirs, files in os.walk(driver_dir): - for filename in files: - name_without_ext, ext = os.path.splitext(filename) - if ext.lower() == ".json": - json_data = {} - file_path = os.path.join(root, filename) - with open(file_path) as json_file: - json_data = json.load(json_file) - if type(json_data) == dict and "scos_files" in json_data: - scos_files = json_data["scos_files"] - for scos_file in scos_files: - source_path = os.path.join( - driver_dir, scos_file["source_path"] - ) - if not os.path.isfile(source_path): - logger.error(f"Unable to find file at {source_path}") - continue - dest_path = scos_file["dest_path"] - dest_dir = os.path.dirname(dest_path) - try: - if not os.path.isdir(dest_dir): - os.makedirs(dest_dir) - logger.debug(f"copying {source_path} to {dest_path}") - shutil.copyfile(source_path, dest_path) - except Exception as e: - logger.error(f"Failed to copy {source_path} to {dest_path}") - logger.error(e) - - -def load_actions(mock_sigan, running_tests, driver_dir, action_dir): - logger.debug("********** Initializing actions **********") - - copy_driver_files(driver_dir) # copy driver files before loading plugins - - discovered_plugins = { - name: importlib.import_module(name) - for finder, name, ispkg in pkgutil.iter_modules() - if name.startswith("scos_") and name != "scos_actions" - } - logger.debug(discovered_plugins) - action_types = {} - action_types.update(action_classes) - actions = {} - if mock_sigan or running_tests: - for name, action in test_actions.items(): - logger.debug("test_action: " + name + "=" + str(action)) - else: - for name, module in discovered_plugins.items(): - logger.debug("Looking for actions in " + name + ": " + str(module)) - discover = importlib.import_module(name + ".discover") - if hasattr(discover, "actions"): - for name, action in discover.actions.items(): - logger.debug("action: " + name + "=" + str(action)) - actions[name] = action - if hasattr(discover, "action_types") and discover.action_types is not None: - action_types.update(discover.action_types) - - logger.debug(f"Loading actions in {action_dir}") - yaml_actions, yaml_test_actions = init(action_classes = action_types, yaml_dir=action_dir) - actions.update(yaml_actions) - logger.debug("Finished loading and registering actions") - return actions - - -def load_capabilities(sensor_definition_file): - capabilities = {} - SENSOR_DEFINITION_HASH = None - SENSOR_LOCATION = None - - logger.debug(f"Loading {sensor_definition_file}") - try: - capabilities["sensor"] = utils.load_from_json(sensor_definition_file) - except Exception as e: - logger.warning( - f"Failed to load sensor definition file: {sensor_definition_file}" - + "\nAn empty sensor definition will be used" - ) - capabilities["sensor"] = {"sensor_spec": {"id": "unknown"}} - capabilities["sensor"]["sensor_sha512"] = "UNKNOWN SENSOR DEFINITION" - - # Generate sensor definition file hash (SHA 512) - try: - if "sensor_sha512" not in capabilities["sensor"]: - sensor_def = json.dumps(capabilities["sensor"], sort_keys=True) - SENSOR_DEFINITION_HASH = hashlib.sha512(sensor_def.encode("UTF-8")).hexdigest() - capabilities["sensor"]["sensor_sha512"] = SENSOR_DEFINITION_HASH - except: - capabilities["sensor"]["sensor_sha512"] = "ERROR GENERATING HASH" - # SENSOR_DEFINITION_HASH is None, do not raise Exception - logger.exception(f"Unable to generate sensor definition hash") - - return capabilities - -ACTIONS = load_actions(MOCK_SIGAN, RUNNING_TESTS, DRIVERS_DIR, ACTIONS_DIR) -CAPABILITIES = load_capabilities(SENSOR_DEFINITION_FILE) \ No newline at end of file diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 23ab24ff..cccf3ef8 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -8,10 +8,6 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/1.11/ref/settings/ -!!!!!!NOTE!!!!!: This file is replaced when scos-sensor runs in docker. migration_settings.py is used when migrations are -run and runtime_settings is used when scos sensor is run in docker. -Make sure runtime_settings.py and this stay in sync as needed. See entrypoints/api_entrypoints.sh - """ import importlib import hashlib @@ -22,7 +18,7 @@ import shutil import sys from os import path - +from pathlib import Path from cryptography.fernet import Fernet from django.core.management.utils import get_random_secret_key @@ -444,9 +440,9 @@ "PRESELECTOR_MODULE", default="its_preselector.web_relay_preselector" ) PRESELECTOR_CLASS = env.str("PRESELECTOR_CLASS", default="WebRelayPreselector") -SWITCH_CONFIGS_DIR = env.str( +SWITCH_CONFIGS_DIR = Path(env.str( "SWITCH_CONFIGS_DIR", default=path.join(CONFIG_DIR, "switches") -) +)) SIGAN_POWER_CYCLE_STATES = env("SIGAN_POWER_CYCLE_STATES", default=None) SIGAN_POWER_SWITCH = env("SIGAN_POWER_SWITCH", default=None) MAX_FAILURES = env("MAX_FAILURES", default=2) From 7ec6df3d916aca09423206d900f1394a56b4fd21 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 15:33:08 -0700 Subject: [PATCH 084/255] Initialize calibrations and pass to signal analyzer. --- gunicorn/config.py | 19 +++++--- src/initialization/__init__.py | 2 + src/initialization/calibration.py | 73 +++++++++++++++++++++++++++++++ 3 files changed, 88 insertions(+), 6 deletions(-) create mode 100644 src/initialization/calibration.py diff --git a/gunicorn/config.py b/gunicorn/config.py index 5b8a49b4..5b604d5f 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -40,18 +40,24 @@ def post_worker_init(worker): load_preselector, load_switches, ) + from initialization.calibration import ( + get_sensor_calibration, + get_sigan_calibration + ) from django.conf import settings - sigan_module_setting = env("SIGAN_MODULE") + sigan_module_setting = settings.SIGAN_MODULE sigan_module = importlib.import_module(sigan_module_setting) - logger.info("Creating " + env("SIGAN_CLASS") + " from " + env("SIGAN_MODULE")) - sigan_constructor = getattr(sigan_module, env("SIGAN_CLASS")) - sigan = sigan_constructor() + logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) + sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) + sensor_cal = get_sensor_calibration(settings.SENSOR_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) + sigan_cal = get_sigan_calibration(settings.SIGAN_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) + sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal) register_component_with_status.send(sigan, component=sigan) switches = load_switches(settings.SWITCH_CONFIGS_DIR) for key, switch in switches: register_component_with_status(switch, component=switch) - capabilities = env("CAPABILITIES") - preselector = load_preselector(env("PRESELECTOR_CONFIG"), env("PRESELEDTOR_MODULE"), env("PRESELECTOR_CLASS"), capabilities["sensor"]) + capabilities = settings.CAPABILITIES + preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELEDTOR_MODULE, settings.PRESELECTOR_CLASS, capabilities["sensor"]) register_component_with_status(preselector, component=preselector) if "location" in capabilities["sensor"]: try: @@ -64,6 +70,7 @@ def post_worker_init(worker): except: logger.exception("Failed to get sensor location from sensor definition.") + sensor = Sensor(signal_analyzer=sigan, preselector = preselector, switches = switches, capabilities = capabilities, location = location) scheduler.thread.sensor = sensor scheduler.thread.start() diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 32d929ce..bd267aad 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -59,5 +59,7 @@ def load_preselector(preselector_config, module, preselector_class_name, sensor_ ps = None return ps +import logging +from os import path diff --git a/src/initialization/calibration.py b/src/initialization/calibration.py new file mode 100644 index 00000000..45fc752e --- /dev/null +++ b/src/initialization/calibration.py @@ -0,0 +1,73 @@ +from os import path + +from scos_actions.calibration.calibration import Calibration, load_from_json + +logger = logging.getLogger(__name__) + + +def get_sigan_calibration(sigan_cal_file_path: str, default_cal_file_path: str) -> Calibration: + """ + Load signal analyzer calibration data from file. + + :param sigan_cal_file_path: Path to JSON file containing signal + analyzer calibration data. + :param default_cal_file_path: Path to the default cal file. + :return: The signal analyzer ``Calibration`` object. + """ + try: + sigan_cal = None + if sigan_cal_file_path is None or sigan_cal_file_path == "": + logger.warning("No sigan calibration file specified. Not loading calibration file.") + elif not path.exists(sigan_cal_file_path): + logger.warning( + sigan_cal_file_path + " does not exist. Not loading sigan calibration file." + ) + else: + logger.debug(f"Loading sigan cal file: {sigan_cal_file_path}") + check_for_default_calibration(sigan_cal_file_path,default_cal_file_path, "Sigan") + sigan_cal = load_from_json(sigan_cal_file_path) + except Exception: + sigan_cal = None + logger.exception("Unable to load sigan calibration data, reverting to none") + return sigan_cal + + +def get_sensor_calibration(sensor_cal_file_path: str, default_cal_file_path: str) -> Calibration: + """ + Load sensor calibration data from file. + + :param sensor_cal_file_path: Path to JSON file containing sensor + calibration data. + :param default_cal_file_path: Name of the default calibration file. + :return: The sensor ``Calibration`` object. + """ + try: + sensor_cal = None + if sensor_cal_file_path is None or sensor_cal_file_path == "": + logger.warning( + "No sensor calibration file specified. Not loading calibration file." + ) + elif not path.exists(sensor_cal_file_path): + logger.warning( + sensor_cal_file_path + + " does not exist. Not loading sensor calibration file." + ) + else: + logger.debug(f"Loading sensor cal file: {sensor_cal_file_path}") + check_for_default_calibration( + sensor_cal_file_path, default_cal_file_path, "Sensor" + ) + sensor_cal = load_from_json(sensor_cal_file_path) + except Exception: + sensor_cal = None + logger.exception("Unable to load sensor calibration data, reverting to none") + return sensor_cal + + +def check_for_default_calibration(cal_file_path: str,default_cal_path: str, cal_type: str): + default_cal = False + if cal_file_path == default_cal_path: + default_cal = True + logger.warning( + f"***************LOADING DEFAULT {cal_type} CALIBRATION***************" + ) From 2c643dd2178f78ebd44ef1453bfbf5d6c7ac3dc5 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 15:56:05 -0700 Subject: [PATCH 085/255] mark Calibrations as default or not. --- src/initialization/calibration.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/initialization/calibration.py b/src/initialization/calibration.py index 45fc752e..ab85ebc7 100644 --- a/src/initialization/calibration.py +++ b/src/initialization/calibration.py @@ -23,9 +23,10 @@ def get_sigan_calibration(sigan_cal_file_path: str, default_cal_file_path: str) sigan_cal_file_path + " does not exist. Not loading sigan calibration file." ) else: - logger.debug(f"Loading sigan cal file: {sigan_cal_file_path}") - check_for_default_calibration(sigan_cal_file_path,default_cal_file_path, "Sigan") - sigan_cal = load_from_json(sigan_cal_file_path) + logger.debug(f"Loading sigan cal file: {sigan_cal_file_path}") + default = check_for_default_calibration(sigan_cal_file_path,default_cal_file_path, "Sigan") + sigan_cal = load_from_json(sigan_cal_file_path) + sigan_cal.is_default = default except Exception: sigan_cal = None logger.exception("Unable to load sigan calibration data, reverting to none") @@ -54,20 +55,22 @@ def get_sensor_calibration(sensor_cal_file_path: str, default_cal_file_path: str ) else: logger.debug(f"Loading sensor cal file: {sensor_cal_file_path}") - check_for_default_calibration( + default = check_for_default_calibration( sensor_cal_file_path, default_cal_file_path, "Sensor" ) sensor_cal = load_from_json(sensor_cal_file_path) + sensor_cal.is_default = default except Exception: sensor_cal = None logger.exception("Unable to load sensor calibration data, reverting to none") return sensor_cal -def check_for_default_calibration(cal_file_path: str,default_cal_path: str, cal_type: str): +def check_for_default_calibration(cal_file_path: str,default_cal_path: str, cal_type: str) -> bool: default_cal = False if cal_file_path == default_cal_path: default_cal = True logger.warning( f"***************LOADING DEFAULT {cal_type} CALIBRATION***************" ) + return default_cal From af3f5dde4977e19aeb552d05c2d80f93608aec97 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 16:27:28 -0700 Subject: [PATCH 086/255] add status and signal analyzer registration handlers. Don't import cal in status/views. --- src/status/__init__.py | 27 +++++++++++++++++++++++++-- src/status/views.py | 7 ++++--- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/status/__init__.py b/src/status/__init__.py index 8d4f2823..95b99946 100644 --- a/src/status/__init__.py +++ b/src/status/__init__.py @@ -1,9 +1,32 @@ import datetime import logging +from scos_actions.signals import register_component_with_status +from scos_actions.signals import register_signal_analyzer +from scos_actions.status.status_monitor import StatusMonitor -from scos_actions.calibration import sensor_calibration + +signal_analyzers = [] logger = logging.getLogger(__name__) logger.debug("********** Initializing status **********") -sensor_cal = sensor_calibration start_time = datetime.datetime.utcnow() +status_monitor = StatusMonitor() + +def signal_analyzer_registration_handler(sender, **kwargs): + try: + logger.debug(f"Registering {sender} as status provider") + signal_analyzers[0] = kwargs["signal_analyzer"] + except: + logger.exception("Error registering status component") + + +def status_registration_handler(sender, **kwargs): + try: + logger.debug(f"Registering {sender} as status provider") + status_monitor.add_component(kwargs["component"]) + except: + logger.exception("Error registering status component") + +register_component_with_status.connect(status_registration_handler) +register_signal_analyzer.connect(signal_analyzer_registration_handler) + diff --git a/src/status/views.py b/src/status/views.py index 7bb09f93..a1c1fb48 100644 --- a/src/status/views.py +++ b/src/status/views.py @@ -2,12 +2,13 @@ import logging import shutil +from . import status_monitor +from . import signal_analyzers from its_preselector.preselector import Preselector from its_preselector.web_relay import WebRelay from rest_framework.decorators import api_view from rest_framework.response import Response from scos_actions.hardware.sigan_iface import SignalAnalyzerInterface -from scos_actions.status import status_monitor from scos_actions.utils import ( convert_datetime_to_millisecond_iso_format, get_datetime_str_now, @@ -15,7 +16,7 @@ from scheduler import scheduler -from . import sensor_cal, start_time +from . import start_time from .serializers import LocationSerializer from .utils import get_location @@ -57,7 +58,7 @@ def status(request, version, format=None): "location": serialize_location(), "system_time": get_datetime_str_now(), "start_time": convert_datetime_to_millisecond_iso_format(start_time), - "last_calibration_datetime": sensor_cal.last_calibration_datetime, + "last_calibration_datetime": signal_analyzers[0].signal_analyzer.sensor_calibration.last_calibration_datetime, "disk_usage": disk_usage(), "days_up": get_days_up(), } From 6ff799465b10de33dc8fb9a9ddb3707a0735f1e0 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 16:28:58 -0700 Subject: [PATCH 087/255] register sigan. --- gunicorn/config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gunicorn/config.py b/gunicorn/config.py index 5b604d5f..46755f49 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -7,6 +7,7 @@ from scos_actions.hardware.sensor import Sensor from scos_actions.metadata.utils import construct_geojson_point from scos_actions.signals import register_component_with_status +from scos_actions.signals import register_signal_analyzer bind = ":8000" @@ -53,6 +54,8 @@ def post_worker_init(worker): sigan_cal = get_sigan_calibration(settings.SIGAN_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal) register_component_with_status.send(sigan, component=sigan) + register_signal_analyzer.send(sigan, signal_analyzer=sigan) + switches = load_switches(settings.SWITCH_CONFIGS_DIR) for key, switch in switches: register_component_with_status(switch, component=switch) From 3956cefed9f58b73f56c86d452ed7609bccaf5c3 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 16:41:57 -0700 Subject: [PATCH 088/255] import logging. --- gunicorn/config.py | 2 +- src/initialization/calibration.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index 46755f49..d7418925 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -55,7 +55,7 @@ def post_worker_init(worker): sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal) register_component_with_status.send(sigan, component=sigan) register_signal_analyzer.send(sigan, signal_analyzer=sigan) - + switches = load_switches(settings.SWITCH_CONFIGS_DIR) for key, switch in switches: register_component_with_status(switch, component=switch) diff --git a/src/initialization/calibration.py b/src/initialization/calibration.py index ab85ebc7..175ec138 100644 --- a/src/initialization/calibration.py +++ b/src/initialization/calibration.py @@ -1,3 +1,4 @@ +import logging from os import path from scos_actions.calibration.calibration import Calibration, load_from_json From 3c1d1a4c33cf2575c2a002cc00876678f5c8031f Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 16:53:44 -0700 Subject: [PATCH 089/255] import load_from_json --- gunicorn/config.py | 1 - src/sensor/settings.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index d7418925..d060695a 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -35,7 +35,6 @@ def post_worker_init(worker): import django django.setup() - env = Env() from scheduler import scheduler from initialization import ( load_preselector, diff --git a/src/sensor/settings.py b/src/sensor/settings.py index cccf3ef8..10312cf9 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -26,6 +26,7 @@ from scos_actions.actions import action_classes from scos_actions.discover import test_actions from scos_actions.discover import init +from scos_actions.utils import load_from_json logger = logging.getLogger(__name__) logger.debug("Initializing scos-sensor settings.") @@ -520,7 +521,7 @@ def load_capabilities(sensor_definition_file): logger.debug(f"Loading {sensor_definition_file}") try: - capabilities["sensor"] = utils.load_from_json(sensor_definition_file) + capabilities["sensor"] = load_from_json(sensor_definition_file) except Exception as e: logger.warning( f"Failed to load sensor definition file: {sensor_definition_file}" From 6e22f7a9b34c027e019754160c0e18622583d565 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 19:13:57 -0700 Subject: [PATCH 090/255] pass if is_default into loading calibration from json. --- src/initialization/calibration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/initialization/calibration.py b/src/initialization/calibration.py index 175ec138..b10d3350 100644 --- a/src/initialization/calibration.py +++ b/src/initialization/calibration.py @@ -26,7 +26,7 @@ def get_sigan_calibration(sigan_cal_file_path: str, default_cal_file_path: str) else: logger.debug(f"Loading sigan cal file: {sigan_cal_file_path}") default = check_for_default_calibration(sigan_cal_file_path,default_cal_file_path, "Sigan") - sigan_cal = load_from_json(sigan_cal_file_path) + sigan_cal = load_from_json(sigan_cal_file_path, default) sigan_cal.is_default = default except Exception: sigan_cal = None @@ -59,7 +59,7 @@ def get_sensor_calibration(sensor_cal_file_path: str, default_cal_file_path: str default = check_for_default_calibration( sensor_cal_file_path, default_cal_file_path, "Sensor" ) - sensor_cal = load_from_json(sensor_cal_file_path) + sensor_cal = load_from_json(sensor_cal_file_path, default) sensor_cal.is_default = default except Exception: sensor_cal = None From d629cd5a5dcda81c5ef9c43aba0a00d89169db55 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 19:20:34 -0700 Subject: [PATCH 091/255] fix register_sigan handler --- src/status/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/status/__init__.py b/src/status/__init__.py index 95b99946..012d44d1 100644 --- a/src/status/__init__.py +++ b/src/status/__init__.py @@ -14,8 +14,11 @@ def signal_analyzer_registration_handler(sender, **kwargs): try: - logger.debug(f"Registering {sender} as status provider") - signal_analyzers[0] = kwargs["signal_analyzer"] + logger.debug(f"Registering {sender} as signa analyzer") + if len(signal_analyzers) > 0: + signal_analyzers[0] = + else: + signal_analyzers.append( kwargs["signal_analyzer"]) except: logger.exception("Error registering status component") From 3973c12b5905284ac0dca1b126cef00637175210 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 20:12:39 -0700 Subject: [PATCH 092/255] fix mistaken deletion --- src/status/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/status/__init__.py b/src/status/__init__.py index 012d44d1..f55078f5 100644 --- a/src/status/__init__.py +++ b/src/status/__init__.py @@ -16,7 +16,7 @@ def signal_analyzer_registration_handler(sender, **kwargs): try: logger.debug(f"Registering {sender} as signa analyzer") if len(signal_analyzers) > 0: - signal_analyzers[0] = + signal_analyzers[0] = kwargs["signal_analyzer"] else: signal_analyzers.append( kwargs["signal_analyzer"]) except: From d1e5beb52de49b3ca4b03dfabc2af3973450be5c Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 20:22:52 -0700 Subject: [PATCH 093/255] fix loop to register switches. --- gunicorn/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index d060695a..b1c02e3b 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -56,7 +56,7 @@ def post_worker_init(worker): register_signal_analyzer.send(sigan, signal_analyzer=sigan) switches = load_switches(settings.SWITCH_CONFIGS_DIR) - for key, switch in switches: + for key, switch in switches.items(): register_component_with_status(switch, component=switch) capabilities = settings.CAPABILITIES preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELEDTOR_MODULE, settings.PRESELECTOR_CLASS, capabilities["sensor"]) From 15c4f775aa1f0a51cc150e57c3389fec5e693244 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 20:29:56 -0700 Subject: [PATCH 094/255] send signal in config. --- gunicorn/config.py | 2 +- src/initialization/__init__.py | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index b1c02e3b..5bde91d9 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -57,7 +57,7 @@ def post_worker_init(worker): switches = load_switches(settings.SWITCH_CONFIGS_DIR) for key, switch in switches.items(): - register_component_with_status(switch, component=switch) + register_component_with_status.send(switch, component=switch) capabilities = settings.CAPABILITIES preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELEDTOR_MODULE, settings.PRESELECTOR_CLASS, capabilities["sensor"]) register_component_with_status(preselector, component=preselector) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index bd267aad..a119e55d 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -52,14 +52,8 @@ def load_preselector(preselector_config, module, preselector_class_name, sensor_ preselector_module = importlib.import_module(module) preselector_constructor = getattr(preselector_module, preselector_class_name) ps = preselector_constructor(sensor_definition, preselector_config) - if ps and ps.name: - logger.debug(f"Registering {ps.name} as status provider") - register_component_with_status.send(__name__, component=ps) else: ps = None return ps -import logging -from os import path - From 356e1f02997f6d306f120446d81c94e140d8e990 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 20:34:15 -0700 Subject: [PATCH 095/255] typo fix. --- gunicorn/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index 5bde91d9..882f95d2 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -59,7 +59,7 @@ def post_worker_init(worker): for key, switch in switches.items(): register_component_with_status.send(switch, component=switch) capabilities = settings.CAPABILITIES - preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELEDTOR_MODULE, settings.PRESELECTOR_CLASS, capabilities["sensor"]) + preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELECTOR_MODULE, settings.PRESELECTOR_CLASS, capabilities["sensor"]) register_component_with_status(preselector, component=preselector) if "location" in capabilities["sensor"]: try: From e8b6ecc65972722cf021345d38f4a814a956a989 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 20:53:13 -0700 Subject: [PATCH 096/255] debugging. --- src/initialization/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index a119e55d..4eca1fbd 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -48,6 +48,7 @@ def load_preselector_from_file(preselector_config_file: Path): def load_preselector(preselector_config, module, preselector_class_name, sensor_definition): + logger.debug(f"loading {preselector_class_name} from {module} with config: {preselector_config}") if module is not None and preselector_class_name is not None: preselector_module = importlib.import_module(module) preselector_constructor = getattr(preselector_module, preselector_class_name) From 11b5fe73b0ce0ec285bcf17f6aebb1d6c70b8093 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 21:04:37 -0700 Subject: [PATCH 097/255] load preselector config from json. --- src/initialization/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 4eca1fbd..15a357b2 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -52,6 +52,7 @@ def load_preselector(preselector_config, module, preselector_class_name, sensor_ if module is not None and preselector_class_name is not None: preselector_module = importlib.import_module(module) preselector_constructor = getattr(preselector_module, preselector_class_name) + preselector_config = utils.load_from_json(preselector_config) ps = preselector_constructor(sensor_definition, preselector_config) else: ps = None From b9a9d444184e50071d2a5e05271669dfd37407e9 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 21:10:30 -0700 Subject: [PATCH 098/255] send signal in config. add type hints. --- gunicorn/config.py | 2 +- src/initialization/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index 882f95d2..fd88581a 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -60,7 +60,7 @@ def post_worker_init(worker): register_component_with_status.send(switch, component=switch) capabilities = settings.CAPABILITIES preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELECTOR_MODULE, settings.PRESELECTOR_CLASS, capabilities["sensor"]) - register_component_with_status(preselector, component=preselector) + register_component_with_status.send(preselector, component=preselector) if "location" in capabilities["sensor"]: try: sensor_loc = capabilities["sensor"].pop("location") diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 15a357b2..2cef5649 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -47,7 +47,7 @@ def load_preselector_from_file(preselector_config_file: Path): return None -def load_preselector(preselector_config, module, preselector_class_name, sensor_definition): +def load_preselector(preselector_config: str, module: str, preselector_class_name: str, sensor_definition: dict): logger.debug(f"loading {preselector_class_name} from {module} with config: {preselector_config}") if module is not None and preselector_class_name is not None: preselector_module = importlib.import_module(module) From 29bcd32b6263c2e18c782c2efa4373b881ab6803 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 21:20:12 -0700 Subject: [PATCH 099/255] Replace scheduler signal_analyzer with sensor. --- src/scheduler/scheduler.py | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/src/scheduler/scheduler.py b/src/scheduler/scheduler.py index 4961ad2c..79450ca7 100644 --- a/src/scheduler/scheduler.py +++ b/src/scheduler/scheduler.py @@ -8,6 +8,7 @@ import requests from django.utils import timezone +from scos_actions.hardware.sensor import Sensor from scos_actions.signals import trigger_api_restart from authentication import oauth from schedule.models import ScheduleEntry @@ -46,25 +47,16 @@ def __init__(self): self.task = None # Task object describing current task self.last_status = "" self.consecutive_failures = 0 - self._signal_analyzer = None - self._gps = None + self._sensor = None @property - def signal_analyzer(self): - return self._signal_analyzer + def sensor(self): + return self._sensor - @signal_analyzer.setter - def signal_analyzer(self, sigan): - logger.debug(f"Set scheduler sigan to {sigan}") - self._signal_analyzer = sigan - - @property - def gps(self): - return self._gps - - @gps.setter - def gps(self, gps): - self._gps = gps + @sensor.setter + def sensor(self, sensor: Sensor): + logger.debug(f"Set scheduler sigan to {sensor}") + self._sensor = sensor @property def schedule(self): @@ -181,8 +173,8 @@ def _call_task_action(self): schedule_entry_json["id"] = entry_name try: - logger.debug(f"running task {entry_name}/{task_id} with sigan: {self.signal_analyzer}") - detail = self.task.action_caller(self.signal_analyzer, self.gps, schedule_entry_json, task_id) + logger.debug(f"running task {entry_name}/{task_id} with sigan: {self.sensor}") + detail = self.task.action_caller(self.sensor, schedule_entry_json, task_id) self.delayfn(0) # let other threads run status = "success" if not isinstance(detail, str): From 7a4d16c9b3b362c93c61aec1e8c8b81a4b15b6de Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 21:38:47 -0700 Subject: [PATCH 100/255] correct ref to signal_analyzer in status. --- src/status/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/status/views.py b/src/status/views.py index a1c1fb48..7b000ebb 100644 --- a/src/status/views.py +++ b/src/status/views.py @@ -58,7 +58,7 @@ def status(request, version, format=None): "location": serialize_location(), "system_time": get_datetime_str_now(), "start_time": convert_datetime_to_millisecond_iso_format(start_time), - "last_calibration_datetime": signal_analyzers[0].signal_analyzer.sensor_calibration.last_calibration_datetime, + "last_calibration_datetime": signal_analyzers[0].sensor_calibration.last_calibration_datetime, "disk_usage": disk_usage(), "days_up": get_days_up(), } From 17d27af21152dcf83e0f34e5f893349da0935d28 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 13 Jan 2024 21:51:11 -0700 Subject: [PATCH 101/255] move status registration into initializatino methods. --- gunicorn/config.py | 3 --- src/initialization/__init__.py | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index fd88581a..2d86bd4a 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -56,11 +56,8 @@ def post_worker_init(worker): register_signal_analyzer.send(sigan, signal_analyzer=sigan) switches = load_switches(settings.SWITCH_CONFIGS_DIR) - for key, switch in switches.items(): - register_component_with_status.send(switch, component=switch) capabilities = settings.CAPABILITIES preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELECTOR_MODULE, settings.PRESELECTOR_CLASS, capabilities["sensor"]) - register_component_with_status.send(preselector, component=preselector) if "location" in capabilities["sensor"]: try: sensor_loc = capabilities["sensor"].pop("location") diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 2cef5649..54ebc7e9 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -54,6 +54,7 @@ def load_preselector(preselector_config: str, module: str, preselector_class_nam preselector_constructor = getattr(preselector_module, preselector_class_name) preselector_config = utils.load_from_json(preselector_config) ps = preselector_constructor(sensor_definition, preselector_config) + register_component_with_status.send(ps, component=ps) else: ps = None return ps From 16e571e586cbc5c9a9feed4668815007862da3a3 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sun, 14 Jan 2024 11:35:07 -0700 Subject: [PATCH 102/255] Add register sensor handler. Update registered sensors location when location in the db is changed. --- src/handlers/__init__.py | 1 + src/handlers/apps.py | 7 +++++++ src/handlers/location_handler.py | 23 ++++++++++------------- src/handlers/register_sensor_handler.py | 12 ++++++++++++ 4 files changed, 30 insertions(+), 13 deletions(-) create mode 100644 src/handlers/register_sensor_handler.py diff --git a/src/handlers/__init__.py b/src/handlers/__init__.py index 2d4dd503..fb6877a6 100644 --- a/src/handlers/__init__.py +++ b/src/handlers/__init__.py @@ -2,3 +2,4 @@ logger = logging.getLogger(__name__) logger.debug("********** Initializing handlers **********") +sensors = [] \ No newline at end of file diff --git a/src/handlers/apps.py b/src/handlers/apps.py index 3dffb135..8c3a3f6c 100644 --- a/src/handlers/apps.py +++ b/src/handlers/apps.py @@ -6,6 +6,7 @@ location_action_completed, measurement_action_completed, trigger_api_restart, + register_sensor ) logger = logging.getLogger(__name__) @@ -20,8 +21,10 @@ def ready(self): db_location_deleted, db_location_updated, location_action_completed_callback, + ) from handlers.measurement_handler import measurement_action_completed_callback + from handlers.register_sensor_handler import sensor_registered measurement_action_completed.connect(measurement_action_completed_callback) logger.debug( @@ -39,3 +42,7 @@ def ready(self): trigger_api_restart.connect(trigger_api_restart_callback) logger.debug("trigger_api_restart_callback registered to trigger_api_restart") + + register_sensor.connect(sensor_registered) + logger.debug("sensor_registered handler connected to register_sensor signal") + diff --git a/src/handlers/location_handler.py b/src/handlers/location_handler.py index bc3f2e7a..bfd8d03a 100644 --- a/src/handlers/location_handler.py +++ b/src/handlers/location_handler.py @@ -1,9 +1,13 @@ +import logging +from . import sensor from status.models import GPS_LOCATION_DESCRIPTION, Location from django.conf import settings +from scos_actions.metadata.utils import construct_geojson_point + +logger = logging.getLogger(__name__) def location_action_completed_callback(sender, **kwargs): """Update database and capabilities when GPS is synced or database is updated""" - latitude = kwargs["latitude"] if "latitude" in kwargs else None longitude = kwargs["longitude"] if "longitude" in kwargs else None gps = kwargs["gps"] if "gps" in kwargs else None @@ -29,22 +33,15 @@ def location_action_completed_callback(sender, **kwargs): def db_location_updated(sender, **kwargs): instance = kwargs["instance"] - capabilities = settings.CAPABILITIES if isinstance(instance, Location) and instance.active: - if ( - "location" not in capabilities["sensor"] - or capabilities["sensor"]["location"] is None - ): - capabilities["sensor"]["location"] = {} - capabilities["sensor"]["location"]["x"] = instance.longitude - capabilities["sensor"]["location"]["y"] = instance.latitude - capabilities["sensor"]["location"]["z"] = instance.height - capabilities["sensor"]["location"]["gps"] = instance.gps - capabilities["sensor"]["location"]["description"] = instance.description + geojson = construct_geojson_point(longitude = instance.longitude, latitude=instance.latitude, altitude= instance.height) + sensor.location = geojson + logger.debug(f"Updated {sensor} location to {geojson}") + def db_location_deleted(sender, **kwargs): instance = kwargs["instance"] if isinstance(instance, Location): if "location" in settings.CAPABILITIES["sensor"] and instance.active: - settings.CAPABILITIES["sensor"]["location"] = None + sensor.location = None diff --git a/src/handlers/register_sensor_handler.py b/src/handlers/register_sensor_handler.py new file mode 100644 index 00000000..09f139f2 --- /dev/null +++ b/src/handlers/register_sensor_handler.py @@ -0,0 +1,12 @@ +import logging +from . import sensors + +logger = logging.getLogger(__name__) + +def sensor_registered(sender, **kwargs): + sensor = kwargs["sensor"] + if len(sensors) > 0: + sensors[0] = sensor + else: + sensors.append(sensor) + logger.debug(f"Registered sensor {sensor}") \ No newline at end of file From a1838e6018be9353d9bf77fa4d1d4eba0d577304 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sun, 14 Jan 2024 11:45:00 -0700 Subject: [PATCH 103/255] Fix sensor import in location handler. --- src/handlers/location_handler.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/handlers/location_handler.py b/src/handlers/location_handler.py index bfd8d03a..d9d3b02f 100644 --- a/src/handlers/location_handler.py +++ b/src/handlers/location_handler.py @@ -1,5 +1,5 @@ import logging -from . import sensor +from . import sensors from status.models import GPS_LOCATION_DESCRIPTION, Location from django.conf import settings from scos_actions.metadata.utils import construct_geojson_point @@ -35,13 +35,19 @@ def db_location_updated(sender, **kwargs): instance = kwargs["instance"] if isinstance(instance, Location) and instance.active: geojson = construct_geojson_point(longitude = instance.longitude, latitude=instance.latitude, altitude= instance.height) - sensor.location = geojson - logger.debug(f"Updated {sensor} location to {geojson}") - + if len(sensors) > 0: + sensors[0].location = geojson + logger.debug(f"Updated {sensors[0]} location to {geojson}") + else: + logger.warning("No sensor is registered. Unable to update sensor location.") def db_location_deleted(sender, **kwargs): instance = kwargs["instance"] if isinstance(instance, Location): if "location" in settings.CAPABILITIES["sensor"] and instance.active: - sensor.location = None + if len(sensors) >0: + sensors[0].location = None + logger.debug(f"Set {sensors[0]} location to None.") + else: + logger.warning("No sensor registered. Unable to remove sensor location.") From ce50dc4e6585463156858f9098c6c1b049f3b52e Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sun, 14 Jan 2024 11:50:21 -0700 Subject: [PATCH 104/255] send register sensor signal. --- gunicorn/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gunicorn/config.py b/gunicorn/config.py index 2d86bd4a..1fc81c76 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -8,6 +8,7 @@ from scos_actions.metadata.utils import construct_geojson_point from scos_actions.signals import register_component_with_status from scos_actions.signals import register_signal_analyzer +from scos_actions.signals import register_sensor bind = ":8000" @@ -72,6 +73,7 @@ def post_worker_init(worker): sensor = Sensor(signal_analyzer=sigan, preselector = preselector, switches = switches, capabilities = capabilities, location = location) scheduler.thread.sensor = sensor + register_sensor.send(sensor=sensor) scheduler.thread.start() From 1a8abb2ace893bea81dfef5c247aa50b901cf401 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sun, 14 Jan 2024 11:55:01 -0700 Subject: [PATCH 105/255] add sender to register sensor signal send. --- gunicorn/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index 1fc81c76..3009174d 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -73,7 +73,7 @@ def post_worker_init(worker): sensor = Sensor(signal_analyzer=sigan, preselector = preselector, switches = switches, capabilities = capabilities, location = location) scheduler.thread.sensor = sensor - register_sensor.send(sensor=sensor) + register_sensor.send(sensor, sensor=sensor) scheduler.thread.start() From 1b46c918b6d072e17a122bf685c75d8e3a9a57f1 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sun, 14 Jan 2024 13:45:06 -0700 Subject: [PATCH 106/255] If there is an active location in the database, use it over the sensor definition location. If there is no location in the database and there is a location in the sensor def, save it the database as the active location. --- gunicorn/config.py | 25 ++++++++++++++++++++----- src/handlers/location_handler.py | 2 +- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index 3009174d..3c954afb 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -46,6 +46,7 @@ def post_worker_init(worker): get_sigan_calibration ) from django.conf import settings + from status.models import Location sigan_module_setting = settings.SIGAN_MODULE sigan_module = importlib.import_module(sigan_module_setting) logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) @@ -62,11 +63,25 @@ def post_worker_init(worker): if "location" in capabilities["sensor"]: try: sensor_loc = capabilities["sensor"].pop("location") - location = construct_geojson_point( - sensor_loc["x"], - sensor_loc["y"], - sensor_loc["z"] if "z" in sensor_loc else None, - ) + try: + #if there is an active database location, use it over the value in the sensor def. + db_location = Location.objects.get(active=True) + location = construct_geojson_point(db_location.longitude, db_location.latitude, db_location.height) + except Location.DoesNotExist: + #No DB location. Use sensor def location and save to DB. + location = construct_geojson_point( + sensor_loc["x"], + sensor_loc["y"], + sensor_loc["z"] if "z" in sensor_loc else None, + ) + #Save the sensor location from the sensor def to the database + db_location = Location() + db_location.longitude = sensor_loc["x"] + db_location.latitude = sensor_loc["y"] + db_location.height = sensor_loc["z"] + db_location.gps = False + db_location.description = "" + db_location.save() except: logger.exception("Failed to get sensor location from sensor definition.") diff --git a/src/handlers/location_handler.py b/src/handlers/location_handler.py index d9d3b02f..8581cd6c 100644 --- a/src/handlers/location_handler.py +++ b/src/handlers/location_handler.py @@ -7,7 +7,7 @@ logger = logging.getLogger(__name__) def location_action_completed_callback(sender, **kwargs): - """Update database and capabilities when GPS is synced or database is updated""" + """Update database when GPS is synced or database is updated""" latitude = kwargs["latitude"] if "latitude" in kwargs else None longitude = kwargs["longitude"] if "longitude" in kwargs else None gps = kwargs["gps"] if "gps" in kwargs else None From bac837f0b75e77ba3edf652c2254bac6494059c3 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sun, 14 Jan 2024 14:03:06 -0700 Subject: [PATCH 107/255] initialize location to None. --- gunicorn/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gunicorn/config.py b/gunicorn/config.py index 3c954afb..d9af08be 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -60,6 +60,7 @@ def post_worker_init(worker): switches = load_switches(settings.SWITCH_CONFIGS_DIR) capabilities = settings.CAPABILITIES preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELECTOR_MODULE, settings.PRESELECTOR_CLASS, capabilities["sensor"]) + location = None if "location" in capabilities["sensor"]: try: sensor_loc = capabilities["sensor"].pop("location") From 377a94c4001a2f74ec2e16da28009d9a494187c5 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sun, 14 Jan 2024 14:42:36 -0700 Subject: [PATCH 108/255] Set description when saving location from sensor definition file. --- gunicorn/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index d9af08be..4e68cc41 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -81,7 +81,7 @@ def post_worker_init(worker): db_location.latitude = sensor_loc["y"] db_location.height = sensor_loc["z"] db_location.gps = False - db_location.description = "" + db_location.description = "Sensor Definition Location" db_location.save() except: logger.exception("Failed to get sensor location from sensor definition.") From 603fc534bccb528324e01785ed619563c9a0f7a1 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sun, 14 Jan 2024 14:49:25 -0700 Subject: [PATCH 109/255] cleanup --- src/sensor/settings.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 10312cf9..1e3a36b5 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -501,9 +501,8 @@ def load_actions(mock_sigan, running_tests, driver_dir, action_dir): logger.debug("Looking for actions in " + name + ": " + str(module)) discover = importlib.import_module(name + ".discover") if hasattr(discover, "actions"): - for name, action in discover.actions.items(): - logger.debug("action: " + name + "=" + str(action)) - actions[name] = action + logger.debug(f"loading {len(discover.actions)} actions.") + actions.update(discover.actions) if hasattr(discover, "action_types") and discover.action_types is not None: action_types.update(discover.action_types) @@ -516,8 +515,8 @@ def load_actions(mock_sigan, running_tests, driver_dir, action_dir): def load_capabilities(sensor_definition_file): capabilities = {} - SENSOR_DEFINITION_HASH = None - SENSOR_LOCATION = None + sensor_definition_hash = None + sensor_location = None logger.debug(f"Loading {sensor_definition_file}") try: @@ -534,11 +533,11 @@ def load_capabilities(sensor_definition_file): try: if "sensor_sha512" not in capabilities["sensor"]: sensor_def = json.dumps(capabilities["sensor"], sort_keys=True) - SENSOR_DEFINITION_HASH = hashlib.sha512(sensor_def.encode("UTF-8")).hexdigest() - capabilities["sensor"]["sensor_sha512"] = SENSOR_DEFINITION_HASH + sensor_definition_hash = hashlib.sha512(sensor_def.encode("UTF-8")).hexdigest() + capabilities["sensor"]["sensor_sha512"] = sensor_definition_hash except: capabilities["sensor"]["sensor_sha512"] = "ERROR GENERATING HASH" - # SENSOR_DEFINITION_HASH is None, do not raise Exception + # SENSOR_DEFINITION_HASH is None, do not raise Exception, but log it logger.exception(f"Unable to generate sensor definition hash") return capabilities From ad179fc0615ddd993c6e286be4907e4e82557f11 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sun, 14 Jan 2024 15:35:01 -0700 Subject: [PATCH 110/255] log location handler. Correct comment. --- src/handlers/location_handler.py | 3 ++- src/scheduler/scheduler.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/handlers/location_handler.py b/src/handlers/location_handler.py index 8581cd6c..0cbcf595 100644 --- a/src/handlers/location_handler.py +++ b/src/handlers/location_handler.py @@ -8,6 +8,7 @@ def location_action_completed_callback(sender, **kwargs): """Update database when GPS is synced or database is updated""" + logger.debug(f"Updating location from {sender}") latitude = kwargs["latitude"] if "latitude" in kwargs else None longitude = kwargs["longitude"] if "longitude" in kwargs else None gps = kwargs["gps"] if "gps" in kwargs else None @@ -45,7 +46,7 @@ def db_location_updated(sender, **kwargs): def db_location_deleted(sender, **kwargs): instance = kwargs["instance"] if isinstance(instance, Location): - if "location" in settings.CAPABILITIES["sensor"] and instance.active: + if instance.active: if len(sensors) >0: sensors[0].location = None logger.debug(f"Set {sensors[0]} location to None.") diff --git a/src/scheduler/scheduler.py b/src/scheduler/scheduler.py index 79450ca7..0f5a3d1c 100644 --- a/src/scheduler/scheduler.py +++ b/src/scheduler/scheduler.py @@ -55,7 +55,7 @@ def sensor(self): @sensor.setter def sensor(self, sensor: Sensor): - logger.debug(f"Set scheduler sigan to {sensor}") + logger.debug(f"Set scheduler sensor to {sensor}") self._sensor = sensor @property From fc3e6db1c7a7021a469a1d126715365b3b381189 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sun, 14 Jan 2024 16:16:23 -0700 Subject: [PATCH 111/255] Remove actions app. --- src/actions/__init__.py | 46 -------------------------------- src/actions/apps.py | 11 -------- src/handlers/location_handler.py | 1 + src/sensor/wsgi.py | 2 +- 4 files changed, 2 insertions(+), 58 deletions(-) delete mode 100644 src/actions/__init__.py delete mode 100644 src/actions/apps.py diff --git a/src/actions/__init__.py b/src/actions/__init__.py deleted file mode 100644 index 268ac39a..00000000 --- a/src/actions/__init__.py +++ /dev/null @@ -1,46 +0,0 @@ -# import importlib -# import logging -# import pkgutil -# -# from scos_actions.actions import action_classes -# from scos_actions.discover import test_actions -# from scos_actions.discover import init -# -# -# from sensor import settings -# from sensor.utils import copy_driver_files -# -# logger = logging.getLogger(__name__) -# logger.debug("********** Initializing actions **********") -# -# copy_driver_files() # copy driver files before loading plugins -# -# discovered_plugins = { -# name: importlib.import_module(name) -# for finder, name, ispkg in pkgutil.iter_modules() -# if name.startswith("scos_") and name != "scos_actions" -# } -# logger.debug(discovered_plugins) -# action_types = {} -# action_types.update(action_classes) -# actions = {} -# if settings.MOCK_SIGAN or settings.RUNNING_TESTS: -# for name, action in test_actions.items(): -# logger.debug("test_action: " + name + "=" + str(action)) -# else: -# for name, module in discovered_plugins.items(): -# logger.debug("Looking for actions in " + name + ": " + str(module)) -# discover = importlib.import_module(name + ".discover") -# if hasattr(discover, "actions"): -# for name, action in discover.actions.items(): -# logger.debug("action: " + name + "=" + str(action)) -# actions[name] = action -# if hasattr(discover, "action_types") and discover.action_types is not None: -# action_types.update(discover.action_types) -# -# -# logger.debug(f"Loading actions in {settings.ACTIONS_DIR}") -# yaml_actions, yaml_test_actions = init(action_classes = action_types, yaml_dir=settings.ACTIONS_DIR) -# actions.update(yaml_actions) -# logger.debug("Finished loading and registering actions") -# diff --git a/src/actions/apps.py b/src/actions/apps.py deleted file mode 100644 index d5319dee..00000000 --- a/src/actions/apps.py +++ /dev/null @@ -1,11 +0,0 @@ -# import importlib -# import logging -# import pkgutil -# -# from django.apps import AppConfig -# -# logger = logging.getLogger(__name__) -# -# -# class ActionsConfig(AppConfig): -# name = "actions" diff --git a/src/handlers/location_handler.py b/src/handlers/location_handler.py index 0cbcf595..3cfb6d86 100644 --- a/src/handlers/location_handler.py +++ b/src/handlers/location_handler.py @@ -34,6 +34,7 @@ def location_action_completed_callback(sender, **kwargs): def db_location_updated(sender, **kwargs): instance = kwargs["instance"] + logger.debug(f"DB location updated by {sender}") if isinstance(instance, Location) and instance.active: geojson = construct_geojson_point(longitude = instance.longitude, latitude=instance.latitude, altitude= instance.height) if len(sensors) > 0: diff --git a/src/sensor/wsgi.py b/src/sensor/wsgi.py index 8f6d64d3..b725c8e1 100644 --- a/src/sensor/wsgi.py +++ b/src/sensor/wsgi.py @@ -44,5 +44,5 @@ register_component_with_status.send(sigan, component=sigan) scheduler.thread.signal_analyzer = sigan scheduler.thread.start() - scheduler.thread.start() + From 1b0258e464dbe4a6d1b68c3840d9437b4ff4113b Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 15 Jan 2024 12:17:17 -0700 Subject: [PATCH 112/255] use description from sensor def file if saving loc to db. --- gunicorn/config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index 4e68cc41..7fcc9902 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -69,7 +69,8 @@ def post_worker_init(worker): db_location = Location.objects.get(active=True) location = construct_geojson_point(db_location.longitude, db_location.latitude, db_location.height) except Location.DoesNotExist: - #No DB location. Use sensor def location and save to DB. + # This should never occur because status/migrations/0003_auto_20211217_2229.py + # will load the No DB location. Use sensor def location and save to DB. location = construct_geojson_point( sensor_loc["x"], sensor_loc["y"], @@ -81,7 +82,7 @@ def post_worker_init(worker): db_location.latitude = sensor_loc["y"] db_location.height = sensor_loc["z"] db_location.gps = False - db_location.description = "Sensor Definition Location" + db_location.description = sensor_loc["description"] db_location.save() except: logger.exception("Failed to get sensor location from sensor definition.") From 19caf8499dac259835e5cf3240a2c07593db8958 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 15 Jan 2024 13:12:02 -0700 Subject: [PATCH 113/255] Move scos_actions imports so django is configured. --- gunicorn/config.py | 13 +++--- src/handlers/tests/test_handlers.py | 70 ++++++++++++++++------------- 2 files changed, 45 insertions(+), 38 deletions(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index 7fcc9902..8ade4352 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -2,13 +2,8 @@ import logging import os import sys -from environs import Env from multiprocessing import cpu_count -from scos_actions.hardware.sensor import Sensor -from scos_actions.metadata.utils import construct_geojson_point -from scos_actions.signals import register_component_with_status -from scos_actions.signals import register_signal_analyzer -from scos_actions.signals import register_sensor + bind = ":8000" @@ -47,6 +42,12 @@ def post_worker_init(worker): ) from django.conf import settings from status.models import Location + from scos_actions.hardware.sensor import Sensor + from scos_actions.metadata.utils import construct_geojson_point + from scos_actions.signals import register_component_with_status + from scos_actions.signals import register_signal_analyzer + from scos_actions.signals import register_sensor + sigan_module_setting = settings.SIGAN_MODULE sigan_module = importlib.import_module(sigan_module_setting) logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) diff --git a/src/handlers/tests/test_handlers.py b/src/handlers/tests/test_handlers.py index 35449db9..e1a4d792 100644 --- a/src/handlers/tests/test_handlers.py +++ b/src/handlers/tests/test_handlers.py @@ -1,14 +1,15 @@ import pytest from django import conf -from django.conf.settings import CAPABILITIES +from django.conf import settings from status.models import Location @pytest.mark.django_db def test_db_location_update_handler(): - CAPABILITIES["sensor"] = {} - CAPABILITIES["sensor"]["location"] = {} + capabilities = settings.CAPABILITIES + capabilities["sensor"] = {} + capabilities["sensor"]["location"] = {} location = Location() location.gps = False location.height = 10 @@ -17,15 +18,16 @@ def test_db_location_update_handler(): location.description = "test" location.active = True location.save() - assert CAPABILITIES["sensor"]["location"]["x"] == 100 - assert CAPABILITIES["sensor"]["location"]["y"] == -1 - assert CAPABILITIES["sensor"]["location"]["z"] == 10 - assert CAPABILITIES["sensor"]["location"]["description"] == "test" + assert capabilities["sensor"]["location"]["x"] == 100 + assert capabilities["sensor"]["location"]["y"] == -1 + assert capabilities["sensor"]["location"]["z"] == 10 + assert capabilities["sensor"]["location"]["description"] == "test" @pytest.mark.django_db def test_db_location_update_handler_current_location_none(): - CAPABILITIES["sensor"] = {} + capabilities = settings.CAPABILITIES + capabilities["sensor"] = {} capabilities["sensor"]["location"] = None location = Location() location.gps = False @@ -43,6 +45,7 @@ def test_db_location_update_handler_current_location_none(): @pytest.mark.django_db def test_db_location_update_handler_not_active(): + capabilities = settings.CAPABILITIES capabilities["sensor"] = {} capabilities["sensor"]["location"] = {} location = Location() @@ -53,29 +56,31 @@ def test_db_location_update_handler_not_active(): location.active = False location.description = "test" location.save() - assert len(CAPABILITIES["sensor"]["location"]) == 0 + assert len(capabilities["sensor"]["location"]) == 0 @pytest.mark.django_db def test_db_location_update_handler_no_description(): - CAPABILITIES["sensor"] = {} - CAPABILITIES["sensor"]["location"] = {} + capabilities = settings.CAPABILITIES + capabilities["sensor"] = {} + capabilities["sensor"]["location"] = {} location = Location() location.gps = False location.height = 10 location.longitude = 100 location.latitude = -1 location.save() - assert CAPABILITIES["sensor"]["location"]["x"] == 100 - assert CAPABILITIES["sensor"]["location"]["y"] == -1 - assert CAPABILITIES["sensor"]["location"]["z"] == 10 - assert CAPABILITIES["sensor"]["location"]["description"] == "" + assert capabilities["sensor"]["location"]["x"] == 100 + assert capabilities["sensor"]["location"]["y"] == -1 + assert capabilities["sensor"]["location"]["z"] == 10 + assert capabilities["sensor"]["location"]["description"] == "" @pytest.mark.django_db def test_db_location_deleted_handler(): - CAPABILITIES["sensor"] = {} - CAPABILITIES["sensor"]["location"] = {} + capabilities = settings.CAPABILITIES + capabilities["sensor"] = {} + capabilities["sensor"]["location"] = {} location = Location() location.gps = False location.height = 10 @@ -84,18 +89,19 @@ def test_db_location_deleted_handler(): location.description = "test" location.active = True location.save() - assert CAPABILITIES["sensor"]["location"]["x"] == 100 - assert CAPABILITIES["sensor"]["location"]["y"] == -1 - assert CAPABILITIES["sensor"]["location"]["z"] == 10 - assert CAPABILITIES["sensor"]["location"]["description"] == "test" + assert capabilities["sensor"]["location"]["x"] == 100 + assert capabilities["sensor"]["location"]["y"] == -1 + assert capabilities["sensor"]["location"]["z"] == 10 + assert capabilities["sensor"]["location"]["description"] == "test" location.delete() - assert CAPABILITIES["sensor"]["location"] is None + assert capabilities["sensor"]["location"] is None @pytest.mark.django_db def test_db_location_deleted_inactive_handler(): - CAPABILITIES["sensor"] = {} - CAPABILITIES["sensor"]["location"] = {} + capabilities = settings.CAPABILITIES + capabilities["sensor"] = {} + capabilities["sensor"]["location"] = {} location = Location() location.gps = False location.height = 10 @@ -104,13 +110,13 @@ def test_db_location_deleted_inactive_handler(): location.description = "test" location.active = True location.save() - assert CAPABILITIES["sensor"]["location"]["x"] == 100 - assert CAPABILITIES["sensor"]["location"]["y"] == -1 - assert CAPABILITIES["sensor"]["location"]["z"] == 10 - assert CAPABILITIES["sensor"]["location"]["description"] == "test" + assert capabilities["sensor"]["location"]["x"] == 100 + assert capabilities["sensor"]["location"]["y"] == -1 + assert capabilities["sensor"]["location"]["z"] == 10 + assert capabilities["sensor"]["location"]["description"] == "test" location.active = False location.delete() - assert CAPABILITIES["sensor"]["location"]["x"] == 100 - assert CAPABILITIES["sensor"]["location"]["y"] == -1 - assert CAPABILITIES["sensor"]["location"]["z"] == 10 - assert CAPABILITIES["sensor"]["location"]["description"] == "test" + assert capabilities["sensor"]["location"]["x"] == 100 + assert capabilities["sensor"]["location"]["y"] == -1 + assert capabilities["sensor"]["location"]["z"] == 10 + assert capabilities["sensor"]["location"]["description"] == "test" From 861c5d7c1d5266cb74027385a9159f01fded0e77 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 15 Jan 2024 14:04:01 -0700 Subject: [PATCH 114/255] Place all initialization methods in intialization/__init__.py. Defer all initialization until after settings are loaded. Set actions and capabilities in settings after it is initialized. --- gunicorn/config.py | 13 ++- src/initialization/__init__.py | 178 +++++++++++++++++++++++++++++- src/initialization/calibration.py | 77 ------------- src/sensor/settings.py | 105 +----------------- 4 files changed, 189 insertions(+), 184 deletions(-) delete mode 100644 src/initialization/calibration.py diff --git a/gunicorn/config.py b/gunicorn/config.py index 8ade4352..986198d1 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -36,9 +36,12 @@ def post_worker_init(worker): load_preselector, load_switches, ) - from initialization.calibration import ( + from initialization import ( + copy_driver_files, get_sensor_calibration, - get_sigan_calibration + get_sigan_calibration, + load_actions, + load_capabilities ) from django.conf import settings from status.models import Location @@ -47,7 +50,11 @@ def post_worker_init(worker): from scos_actions.signals import register_component_with_status from scos_actions.signals import register_signal_analyzer from scos_actions.signals import register_sensor - + + settings.ACTIONS = load_actions(settings.MOCK_SIGAN, settings.RUNNING_TESTS, settings.DRIVERS_DIR, settings.ACTIONS_DIR) + settings.CAPABILITIES = load_capabilities(settings.SENSOR_DEFINITION_FILE) + settings.SENSOR_DEFINITION_HASH = settings.CAPABILITIES["sensor"]["sensor_sha512"] + sigan_module_setting = settings.SIGAN_MODULE sigan_module = importlib.import_module(sigan_module_setting) logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 54ebc7e9..509d5699 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -2,15 +2,55 @@ import importlib import json import logging +import pkgutil +import shutil +from os import path from pathlib import Path from its_preselector.configuration_exception import ConfigurationException from its_preselector.controlbyweb_web_relay import ControlByWebWebRelay +from scos_actions.actions import action_classes +from scos_actions.discover import test_actions +from scos_actions.discover import init from scos_actions import utils from scos_actions.signals import register_component_with_status +from scos_actions.calibration.calibration import Calibration, load_from_json + logger = logging.getLogger(__name__) + + +def copy_driver_files(driver_dir): + """Copy driver files where they need to go""" + for root, dirs, files in os.walk(driver_dir): + for filename in files: + name_without_ext, ext = os.path.splitext(filename) + if ext.lower() == ".json": + json_data = {} + file_path = os.path.join(root, filename) + with open(file_path) as json_file: + json_data = json.load(json_file) + if type(json_data) == dict and "scos_files" in json_data: + scos_files = json_data["scos_files"] + for scos_file in scos_files: + source_path = os.path.join( + driver_dir, scos_file["source_path"] + ) + if not os.path.isfile(source_path): + logger.error(f"Unable to find file at {source_path}") + continue + dest_path = scos_file["dest_path"] + dest_dir = os.path.dirname(dest_path) + try: + if not os.path.isdir(dest_dir): + os.makedirs(dest_dir) + logger.debug(f"copying {source_path} to {dest_path}") + shutil.copyfile(source_path, dest_path) + except Exception as e: + logger.error(f"Failed to copy {source_path} to {dest_path}") + logger.error(e) + def load_switches(switch_dir: Path) -> dict: switch_dict = {} if switch_dir is not None and switch_dir.is_dir(): @@ -31,14 +71,14 @@ def load_switches(switch_dir: Path) -> dict: return switch_dict -def load_preselector_from_file(preselector_config_file: Path): +def load_preselector_from_file(preselector_module, preselector_class, preselector_config_file: Path): if preselector_config_file is None: return None else: try: preselector_config = utils.load_from_json(preselector_config_file) return load_preselector( - preselector_config, settings.PRESELECTOR_MODULE, settings.PRESELECTOR_CLASS + preselector_config, preselector_module, preselector_class ) except ConfigurationException: logger.exception( @@ -60,3 +100,137 @@ def load_preselector(preselector_config: str, module: str, preselector_class_nam return ps + + + +def get_sigan_calibration(sigan_cal_file_path: str, default_cal_file_path: str) -> Calibration: + """ + Load signal analyzer calibration data from file. + + :param sigan_cal_file_path: Path to JSON file containing signal + analyzer calibration data. + :param default_cal_file_path: Path to the default cal file. + :return: The signal analyzer ``Calibration`` object. + """ + try: + sigan_cal = None + if sigan_cal_file_path is None or sigan_cal_file_path == "": + logger.warning("No sigan calibration file specified. Not loading calibration file.") + elif not path.exists(sigan_cal_file_path): + logger.warning( + sigan_cal_file_path + " does not exist. Not loading sigan calibration file." + ) + else: + logger.debug(f"Loading sigan cal file: {sigan_cal_file_path}") + default = check_for_default_calibration(sigan_cal_file_path,default_cal_file_path, "Sigan") + sigan_cal = load_from_json(sigan_cal_file_path, default) + sigan_cal.is_default = default + except Exception: + sigan_cal = None + logger.exception("Unable to load sigan calibration data, reverting to none") + return sigan_cal + + +def get_sensor_calibration(sensor_cal_file_path: str, default_cal_file_path: str) -> Calibration: + """ + Load sensor calibration data from file. + + :param sensor_cal_file_path: Path to JSON file containing sensor + calibration data. + :param default_cal_file_path: Name of the default calibration file. + :return: The sensor ``Calibration`` object. + """ + try: + sensor_cal = None + if sensor_cal_file_path is None or sensor_cal_file_path == "": + logger.warning( + "No sensor calibration file specified. Not loading calibration file." + ) + elif not path.exists(sensor_cal_file_path): + logger.warning( + sensor_cal_file_path + + " does not exist. Not loading sensor calibration file." + ) + else: + logger.debug(f"Loading sensor cal file: {sensor_cal_file_path}") + default = check_for_default_calibration( + sensor_cal_file_path, default_cal_file_path, "Sensor" + ) + sensor_cal = load_from_json(sensor_cal_file_path, default) + sensor_cal.is_default = default + except Exception: + sensor_cal = None + logger.exception("Unable to load sensor calibration data, reverting to none") + return sensor_cal + + +def check_for_default_calibration(cal_file_path: str,default_cal_path: str, cal_type: str) -> bool: + default_cal = False + if cal_file_path == default_cal_path: + default_cal = True + logger.warning( + f"***************LOADING DEFAULT {cal_type} CALIBRATION***************" + ) + return default_cal + + +def load_capabilities(sensor_definition_file): + capabilities = {} + sensor_definition_hash = None + sensor_location = None + + logger.debug(f"Loading {sensor_definition_file}") + try: + capabilities["sensor"] = load_from_json(sensor_definition_file) + except Exception as e: + logger.warning( + f"Failed to load sensor definition file: {sensor_definition_file}" + + "\nAn empty sensor definition will be used" + ) + capabilities["sensor"] = {"sensor_spec": {"id": "unknown"}} + capabilities["sensor"]["sensor_sha512"] = "UNKNOWN SENSOR DEFINITION" + + # Generate sensor definition file hash (SHA 512) + try: + if "sensor_sha512" not in capabilities["sensor"]: + sensor_def = json.dumps(capabilities["sensor"], sort_keys=True) + sensor_definition_hash = hashlib.sha512(sensor_def.encode("UTF-8")).hexdigest() + capabilities["sensor"]["sensor_sha512"] = sensor_definition_hash + except: + capabilities["sensor"]["sensor_sha512"] = "ERROR GENERATING HASH" + # SENSOR_DEFINITION_HASH is None, do not raise Exception, but log it + logger.exception(f"Unable to generate sensor definition hash") + + return capabilities + +def load_actions(mock_sigan, running_tests, driver_dir, action_dir): + logger.debug("********** Initializing actions **********") + + copy_driver_files(driver_dir) # copy driver files before loading plugins + discovered_plugins = { + name: importlib.import_module(name) + for finder, name, ispkg in pkgutil.iter_modules() + if name.startswith("scos_") and name != "scos_actions" + } + logger.debug(discovered_plugins) + action_types = {} + action_types.update(action_classes) + actions = {} + if mock_sigan or running_tests: + for name, action in test_actions.items(): + logger.debug("test_action: " + name + "=" + str(action)) + else: + for name, module in discovered_plugins.items(): + logger.debug("Looking for actions in " + name + ": " + str(module)) + discover = importlib.import_module(name + ".discover") + if hasattr(discover, "actions"): + logger.debug(f"loading {len(discover.actions)} actions.") + actions.update(discover.actions) + if hasattr(discover, "action_types") and discover.action_types is not None: + action_types.update(discover.action_types) + + logger.debug(f"Loading actions in {action_dir}") + yaml_actions, yaml_test_actions = init(action_classes = action_types, yaml_dir=action_dir) + actions.update(yaml_actions) + logger.debug("Finished loading and registering actions") + return actions \ No newline at end of file diff --git a/src/initialization/calibration.py b/src/initialization/calibration.py deleted file mode 100644 index b10d3350..00000000 --- a/src/initialization/calibration.py +++ /dev/null @@ -1,77 +0,0 @@ -import logging -from os import path - -from scos_actions.calibration.calibration import Calibration, load_from_json - -logger = logging.getLogger(__name__) - - -def get_sigan_calibration(sigan_cal_file_path: str, default_cal_file_path: str) -> Calibration: - """ - Load signal analyzer calibration data from file. - - :param sigan_cal_file_path: Path to JSON file containing signal - analyzer calibration data. - :param default_cal_file_path: Path to the default cal file. - :return: The signal analyzer ``Calibration`` object. - """ - try: - sigan_cal = None - if sigan_cal_file_path is None or sigan_cal_file_path == "": - logger.warning("No sigan calibration file specified. Not loading calibration file.") - elif not path.exists(sigan_cal_file_path): - logger.warning( - sigan_cal_file_path + " does not exist. Not loading sigan calibration file." - ) - else: - logger.debug(f"Loading sigan cal file: {sigan_cal_file_path}") - default = check_for_default_calibration(sigan_cal_file_path,default_cal_file_path, "Sigan") - sigan_cal = load_from_json(sigan_cal_file_path, default) - sigan_cal.is_default = default - except Exception: - sigan_cal = None - logger.exception("Unable to load sigan calibration data, reverting to none") - return sigan_cal - - -def get_sensor_calibration(sensor_cal_file_path: str, default_cal_file_path: str) -> Calibration: - """ - Load sensor calibration data from file. - - :param sensor_cal_file_path: Path to JSON file containing sensor - calibration data. - :param default_cal_file_path: Name of the default calibration file. - :return: The sensor ``Calibration`` object. - """ - try: - sensor_cal = None - if sensor_cal_file_path is None or sensor_cal_file_path == "": - logger.warning( - "No sensor calibration file specified. Not loading calibration file." - ) - elif not path.exists(sensor_cal_file_path): - logger.warning( - sensor_cal_file_path - + " does not exist. Not loading sensor calibration file." - ) - else: - logger.debug(f"Loading sensor cal file: {sensor_cal_file_path}") - default = check_for_default_calibration( - sensor_cal_file_path, default_cal_file_path, "Sensor" - ) - sensor_cal = load_from_json(sensor_cal_file_path, default) - sensor_cal.is_default = default - except Exception: - sensor_cal = None - logger.exception("Unable to load sensor calibration data, reverting to none") - return sensor_cal - - -def check_for_default_calibration(cal_file_path: str,default_cal_path: str, cal_type: str) -> bool: - default_cal = False - if cal_file_path == default_cal_path: - default_cal = True - logger.warning( - f"***************LOADING DEFAULT {cal_type} CALIBRATION***************" - ) - return default_cal diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 1e3a36b5..b21301d1 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -14,8 +14,6 @@ import json import logging import os -import pkgutil -import shutil import sys from os import path from pathlib import Path @@ -23,10 +21,7 @@ from cryptography.fernet import Fernet from django.core.management.utils import get_random_secret_key from environs import Env -from scos_actions.actions import action_classes -from scos_actions.discover import test_actions -from scos_actions.discover import init -from scos_actions.utils import load_from_json + logger = logging.getLogger(__name__) logger.debug("Initializing scos-sensor settings.") @@ -448,102 +443,8 @@ SIGAN_POWER_SWITCH = env("SIGAN_POWER_SWITCH", default=None) MAX_FAILURES = env("MAX_FAILURES", default=2) -def copy_driver_files(driver_dir): - """Copy driver files where they need to go""" - for root, dirs, files in os.walk(driver_dir): - for filename in files: - name_without_ext, ext = os.path.splitext(filename) - if ext.lower() == ".json": - json_data = {} - file_path = os.path.join(root, filename) - with open(file_path) as json_file: - json_data = json.load(json_file) - if type(json_data) == dict and "scos_files" in json_data: - scos_files = json_data["scos_files"] - for scos_file in scos_files: - source_path = os.path.join( - driver_dir, scos_file["source_path"] - ) - if not os.path.isfile(source_path): - logger.error(f"Unable to find file at {source_path}") - continue - dest_path = scos_file["dest_path"] - dest_dir = os.path.dirname(dest_path) - try: - if not os.path.isdir(dest_dir): - os.makedirs(dest_dir) - logger.debug(f"copying {source_path} to {dest_path}") - shutil.copyfile(source_path, dest_path) - except Exception as e: - logger.error(f"Failed to copy {source_path} to {dest_path}") - logger.error(e) - - -def load_actions(mock_sigan, running_tests, driver_dir, action_dir): - logger.debug("********** Initializing actions **********") - - copy_driver_files(driver_dir) # copy driver files before loading plugins - - discovered_plugins = { - name: importlib.import_module(name) - for finder, name, ispkg in pkgutil.iter_modules() - if name.startswith("scos_") and name != "scos_actions" - } - logger.debug(discovered_plugins) - action_types = {} - action_types.update(action_classes) - actions = {} - if mock_sigan or running_tests: - for name, action in test_actions.items(): - logger.debug("test_action: " + name + "=" + str(action)) - else: - for name, module in discovered_plugins.items(): - logger.debug("Looking for actions in " + name + ": " + str(module)) - discover = importlib.import_module(name + ".discover") - if hasattr(discover, "actions"): - logger.debug(f"loading {len(discover.actions)} actions.") - actions.update(discover.actions) - if hasattr(discover, "action_types") and discover.action_types is not None: - action_types.update(discover.action_types) - - logger.debug(f"Loading actions in {action_dir}") - yaml_actions, yaml_test_actions = init(action_classes = action_types, yaml_dir=action_dir) - actions.update(yaml_actions) - logger.debug("Finished loading and registering actions") - return actions - - -def load_capabilities(sensor_definition_file): - capabilities = {} - sensor_definition_hash = None - sensor_location = None - - logger.debug(f"Loading {sensor_definition_file}") - try: - capabilities["sensor"] = load_from_json(sensor_definition_file) - except Exception as e: - logger.warning( - f"Failed to load sensor definition file: {sensor_definition_file}" - + "\nAn empty sensor definition will be used" - ) - capabilities["sensor"] = {"sensor_spec": {"id": "unknown"}} - capabilities["sensor"]["sensor_sha512"] = "UNKNOWN SENSOR DEFINITION" - - # Generate sensor definition file hash (SHA 512) - try: - if "sensor_sha512" not in capabilities["sensor"]: - sensor_def = json.dumps(capabilities["sensor"], sort_keys=True) - sensor_definition_hash = hashlib.sha512(sensor_def.encode("UTF-8")).hexdigest() - capabilities["sensor"]["sensor_sha512"] = sensor_definition_hash - except: - capabilities["sensor"]["sensor_sha512"] = "ERROR GENERATING HASH" - # SENSOR_DEFINITION_HASH is None, do not raise Exception, but log it - logger.exception(f"Unable to generate sensor definition hash") - - return capabilities - -ACTIONS = load_actions(MOCK_SIGAN, RUNNING_TESTS, DRIVERS_DIR, ACTIONS_DIR) -CAPABILITIES = load_capabilities(SENSOR_DEFINITION_FILE) +ACTIONS = {} +CAPABILITIES = {} SENSOR_DEFINITION_HASH = CAPABILITIES["sensor"]["sensor_sha512"] From 36ab6ce0114d67ba45087674037dce60dc7737a5 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 15 Jan 2024 14:13:45 -0700 Subject: [PATCH 115/255] don't set SENSOR_DEFINITION_HASH in settings. --- gunicorn/config.py | 1 - src/initialization/__init__.py | 2 +- src/sensor/settings.py | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index 986198d1..93ed7735 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -53,7 +53,6 @@ def post_worker_init(worker): settings.ACTIONS = load_actions(settings.MOCK_SIGAN, settings.RUNNING_TESTS, settings.DRIVERS_DIR, settings.ACTIONS_DIR) settings.CAPABILITIES = load_capabilities(settings.SENSOR_DEFINITION_FILE) - settings.SENSOR_DEFINITION_HASH = settings.CAPABILITIES["sensor"]["sensor_sha512"] sigan_module_setting = settings.SIGAN_MODULE sigan_module = importlib.import_module(sigan_module_setting) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 509d5699..85ff53b9 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -198,7 +198,7 @@ def load_capabilities(sensor_definition_file): capabilities["sensor"]["sensor_sha512"] = sensor_definition_hash except: capabilities["sensor"]["sensor_sha512"] = "ERROR GENERATING HASH" - # SENSOR_DEFINITION_HASH is None, do not raise Exception, but log it + # sensor_sha512 is None, do not raise Exception, but log it logger.exception(f"Unable to generate sensor definition hash") return capabilities diff --git a/src/sensor/settings.py b/src/sensor/settings.py index b21301d1..b6a6b1d6 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -445,7 +445,6 @@ ACTIONS = {} CAPABILITIES = {} -SENSOR_DEFINITION_HASH = CAPABILITIES["sensor"]["sensor_sha512"] From db179755038f4587b54fff155af3ae8504c00436 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 15 Jan 2024 14:19:37 -0700 Subject: [PATCH 116/255] missing import. --- src/initialization/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 85ff53b9..b7374cc2 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -2,6 +2,7 @@ import importlib import json import logging +import os import pkgutil import shutil from os import path From d04a8259f46616926bff5115752d01406c8f97fd Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 15 Jan 2024 14:50:55 -0700 Subject: [PATCH 117/255] Initialize actions, and capabilities in settings so they are there during migration. --- gunicorn/config.py | 3 - src/initialization/__init__.py | 100 +------------------------------- src/sensor/settings.py | 102 ++++++++++++++++++++++++++++++++- 3 files changed, 101 insertions(+), 104 deletions(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index 93ed7735..5a5bdf20 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -51,9 +51,6 @@ def post_worker_init(worker): from scos_actions.signals import register_signal_analyzer from scos_actions.signals import register_sensor - settings.ACTIONS = load_actions(settings.MOCK_SIGAN, settings.RUNNING_TESTS, settings.DRIVERS_DIR, settings.ACTIONS_DIR) - settings.CAPABILITIES = load_capabilities(settings.SENSOR_DEFINITION_FILE) - sigan_module_setting = settings.SIGAN_MODULE sigan_module = importlib.import_module(sigan_module_setting) logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index b7374cc2..1d6deb05 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -1,18 +1,11 @@ -import hashlib + import importlib -import json import logging -import os -import pkgutil -import shutil from os import path from pathlib import Path from its_preselector.configuration_exception import ConfigurationException from its_preselector.controlbyweb_web_relay import ControlByWebWebRelay -from scos_actions.actions import action_classes -from scos_actions.discover import test_actions -from scos_actions.discover import init from scos_actions import utils from scos_actions.signals import register_component_with_status @@ -21,37 +14,6 @@ logger = logging.getLogger(__name__) - -def copy_driver_files(driver_dir): - """Copy driver files where they need to go""" - for root, dirs, files in os.walk(driver_dir): - for filename in files: - name_without_ext, ext = os.path.splitext(filename) - if ext.lower() == ".json": - json_data = {} - file_path = os.path.join(root, filename) - with open(file_path) as json_file: - json_data = json.load(json_file) - if type(json_data) == dict and "scos_files" in json_data: - scos_files = json_data["scos_files"] - for scos_file in scos_files: - source_path = os.path.join( - driver_dir, scos_file["source_path"] - ) - if not os.path.isfile(source_path): - logger.error(f"Unable to find file at {source_path}") - continue - dest_path = scos_file["dest_path"] - dest_dir = os.path.dirname(dest_path) - try: - if not os.path.isdir(dest_dir): - os.makedirs(dest_dir) - logger.debug(f"copying {source_path} to {dest_path}") - shutil.copyfile(source_path, dest_path) - except Exception as e: - logger.error(f"Failed to copy {source_path} to {dest_path}") - logger.error(e) - def load_switches(switch_dir: Path) -> dict: switch_dict = {} if switch_dir is not None and switch_dir.is_dir(): @@ -175,63 +137,3 @@ def check_for_default_calibration(cal_file_path: str,default_cal_path: str, cal_ return default_cal -def load_capabilities(sensor_definition_file): - capabilities = {} - sensor_definition_hash = None - sensor_location = None - - logger.debug(f"Loading {sensor_definition_file}") - try: - capabilities["sensor"] = load_from_json(sensor_definition_file) - except Exception as e: - logger.warning( - f"Failed to load sensor definition file: {sensor_definition_file}" - + "\nAn empty sensor definition will be used" - ) - capabilities["sensor"] = {"sensor_spec": {"id": "unknown"}} - capabilities["sensor"]["sensor_sha512"] = "UNKNOWN SENSOR DEFINITION" - - # Generate sensor definition file hash (SHA 512) - try: - if "sensor_sha512" not in capabilities["sensor"]: - sensor_def = json.dumps(capabilities["sensor"], sort_keys=True) - sensor_definition_hash = hashlib.sha512(sensor_def.encode("UTF-8")).hexdigest() - capabilities["sensor"]["sensor_sha512"] = sensor_definition_hash - except: - capabilities["sensor"]["sensor_sha512"] = "ERROR GENERATING HASH" - # sensor_sha512 is None, do not raise Exception, but log it - logger.exception(f"Unable to generate sensor definition hash") - - return capabilities - -def load_actions(mock_sigan, running_tests, driver_dir, action_dir): - logger.debug("********** Initializing actions **********") - - copy_driver_files(driver_dir) # copy driver files before loading plugins - discovered_plugins = { - name: importlib.import_module(name) - for finder, name, ispkg in pkgutil.iter_modules() - if name.startswith("scos_") and name != "scos_actions" - } - logger.debug(discovered_plugins) - action_types = {} - action_types.update(action_classes) - actions = {} - if mock_sigan or running_tests: - for name, action in test_actions.items(): - logger.debug("test_action: " + name + "=" + str(action)) - else: - for name, module in discovered_plugins.items(): - logger.debug("Looking for actions in " + name + ": " + str(module)) - discover = importlib.import_module(name + ".discover") - if hasattr(discover, "actions"): - logger.debug(f"loading {len(discover.actions)} actions.") - actions.update(discover.actions) - if hasattr(discover, "action_types") and discover.action_types is not None: - action_types.update(discover.action_types) - - logger.debug(f"Loading actions in {action_dir}") - yaml_actions, yaml_test_actions = init(action_classes = action_types, yaml_dir=action_dir) - actions.update(yaml_actions) - logger.debug("Finished loading and registering actions") - return actions \ No newline at end of file diff --git a/src/sensor/settings.py b/src/sensor/settings.py index b6a6b1d6..893a77ce 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -443,8 +443,106 @@ SIGAN_POWER_SWITCH = env("SIGAN_POWER_SWITCH", default=None) MAX_FAILURES = env("MAX_FAILURES", default=2) -ACTIONS = {} -CAPABILITIES = {} + +def copy_driver_files(driver_dir): + import shutil + """Copy driver files where they need to go""" + for root, dirs, files in os.walk(driver_dir): + for filename in files: + name_without_ext, ext = os.path.splitext(filename) + if ext.lower() == ".json": + json_data = {} + file_path = os.path.join(root, filename) + with open(file_path) as json_file: + json_data = json.load(json_file) + if type(json_data) == dict and "scos_files" in json_data: + scos_files = json_data["scos_files"] + for scos_file in scos_files: + source_path = os.path.join( + driver_dir, scos_file["source_path"] + ) + if not os.path.isfile(source_path): + logger.error(f"Unable to find file at {source_path}") + continue + dest_path = scos_file["dest_path"] + dest_dir = os.path.dirname(dest_path) + try: + if not os.path.isdir(dest_dir): + os.makedirs(dest_dir) + logger.debug(f"copying {source_path} to {dest_path}") + shutil.copyfile(source_path, dest_path) + except Exception as e: + logger.error(f"Failed to copy {source_path} to {dest_path}") + logger.error(e) + +def load_capabilities(sensor_definition_file): + from scos_actions.utils import load_from_json + capabilities = {} + sensor_definition_hash = None + sensor_location = None + + logger.debug(f"Loading {sensor_definition_file}") + try: + capabilities["sensor"] = load_from_json(sensor_definition_file) + except Exception as e: + logger.warning( + f"Failed to load sensor definition file: {sensor_definition_file}" + + "\nAn empty sensor definition will be used" + ) + capabilities["sensor"] = {"sensor_spec": {"id": "unknown"}} + capabilities["sensor"]["sensor_sha512"] = "UNKNOWN SENSOR DEFINITION" + + # Generate sensor definition file hash (SHA 512) + try: + if "sensor_sha512" not in capabilities["sensor"]: + sensor_def = json.dumps(capabilities["sensor"], sort_keys=True) + sensor_definition_hash = hashlib.sha512(sensor_def.encode("UTF-8")).hexdigest() + capabilities["sensor"]["sensor_sha512"] = sensor_definition_hash + except: + capabilities["sensor"]["sensor_sha512"] = "ERROR GENERATING HASH" + # sensor_sha512 is None, do not raise Exception, but log it + logger.exception(f"Unable to generate sensor definition hash") + + return capabilities + +def load_actions(mock_sigan, running_tests, driver_dir, action_dir): + from scos_actions.actions import action_classes + from scos_actions.discover import test_actions + from scos_actions.discover import init + import pkgutil + logger.debug("********** Initializing actions **********") + + copy_driver_files(driver_dir) # copy driver files before loading plugins + discovered_plugins = { + name: importlib.import_module(name) + for finder, name, ispkg in pkgutil.iter_modules() + if name.startswith("scos_") and name != "scos_actions" + } + logger.debug(discovered_plugins) + action_types = {} + action_types.update(action_classes) + actions = {} + if mock_sigan or running_tests: + for name, action in test_actions.items(): + logger.debug("test_action: " + name + "=" + str(action)) + else: + for name, module in discovered_plugins.items(): + logger.debug("Looking for actions in " + name + ": " + str(module)) + discover = importlib.import_module(name + ".discover") + if hasattr(discover, "actions"): + logger.debug(f"loading {len(discover.actions)} actions.") + actions.update(discover.actions) + if hasattr(discover, "action_types") and discover.action_types is not None: + action_types.update(discover.action_types) + + logger.debug(f"Loading actions in {action_dir}") + yaml_actions, yaml_test_actions = init(action_classes = action_types, yaml_dir=action_dir) + actions.update(yaml_actions) + logger.debug("Finished loading and registering actions") + return actions + +ACTIONS = load_actions(MOCK_SIGAN, RUNNING_TESTS, DRIVERS_DIR, ACTIONS_DIR) +CAPABILITIES = load_capabilities(SENSOR_DEFINITION_FILE) From 9b5a8f302fa14d4755c5d36e76a7b030cc26fa82 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 15 Jan 2024 14:56:32 -0700 Subject: [PATCH 118/255] remove invalid imports. --- gunicorn/config.py | 5 +---- src/sensor/settings.py | 2 ++ 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index 5a5bdf20..e210b943 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -37,11 +37,8 @@ def post_worker_init(worker): load_switches, ) from initialization import ( - copy_driver_files, get_sensor_calibration, - get_sigan_calibration, - load_actions, - load_capabilities + get_sigan_calibration ) from django.conf import settings from status.models import Location diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 893a77ce..d55c0ecc 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -78,6 +78,8 @@ SENSOR_CALIBRATION_FILE = path.join(CONFIG_DIR, "sensor_calibration.json") else: SENSOR_CALIBRATION_FILE = DEFAULT_CALIBRATION_FILE + + if path.exists(path.join(CONFIG_DIR, "sigan_calibration.json")): SIGAN_CALIBRATION_FILE = path.join(CONFIG_DIR, "sigan_calibration.json") else: From d6926fca5cbd269b084907173404a538577e1df7 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 15 Jan 2024 15:09:52 -0700 Subject: [PATCH 119/255] set sigan and sensor calibration file environment variables. --- src/sensor/settings.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sensor/settings.py b/src/sensor/settings.py index d55c0ecc..0d5d1eb7 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -69,6 +69,7 @@ OPENAPI_FILE = path.join(REPO_ROOT, "docs", "openapi.json") CONFIG_DIR = path.join(REPO_ROOT, "configs") + ACTIONS_DIR = path.join(CONFIG_DIR, "actions") DRIVERS_DIR = path.join(REPO_ROOT, "drivers") @@ -79,12 +80,15 @@ else: SENSOR_CALIBRATION_FILE = DEFAULT_CALIBRATION_FILE +os.environ["SENSOR_CALIBRATION_FILE"] = SENSOR_CALIBRATION_FILE if path.exists(path.join(CONFIG_DIR, "sigan_calibration.json")): SIGAN_CALIBRATION_FILE = path.join(CONFIG_DIR, "sigan_calibration.json") else: SIGAN_CALIBRATION_FILE = DEFAULT_CALIBRATION_FILE +os.environ["SIGAN_CALIBRATION_FILE"] = SIGAN_CALIBRATION_FILE + if path.exists(path.join(CONFIG_DIR, "sensor_definition.json")): SENSOR_DEFINITION_FILE = path.join(CONFIG_DIR, "sensor_definition.json") MEDIA_ROOT = path.join(REPO_ROOT, "files") From 5f2275dea98591a301e190f20fc1d33e659c06a1 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 15 Jan 2024 16:14:43 -0700 Subject: [PATCH 120/255] Set DEFAULT_CALIBRATION_FILE environment variable. --- src/sensor/settings.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 0d5d1eb7..0cf07b95 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -74,6 +74,7 @@ DRIVERS_DIR = path.join(REPO_ROOT, "drivers") DEFAULT_CALIBRATION_FILE= path.join(CONFIG_DIR, "default_calibration.json") +os.environ["DEFAULT_CALIBRATION_FILE"] = DEFAULT_CALIBRATION_FILE # JSON configs if path.exists(path.join(CONFIG_DIR, "sensor_calibration.json")): SENSOR_CALIBRATION_FILE = path.join(CONFIG_DIR, "sensor_calibration.json") @@ -445,6 +446,7 @@ SWITCH_CONFIGS_DIR = Path(env.str( "SWITCH_CONFIGS_DIR", default=path.join(CONFIG_DIR, "switches") )) +os.environ["SWITCH_CONFIGS_DIR"] = SWITCH_CONFIGS_DIR SIGAN_POWER_CYCLE_STATES = env("SIGAN_POWER_CYCLE_STATES", default=None) SIGAN_POWER_SWITCH = env("SIGAN_POWER_SWITCH", default=None) MAX_FAILURES = env("MAX_FAILURES", default=2) @@ -546,7 +548,7 @@ def load_actions(mock_sigan, running_tests, driver_dir, action_dir): actions.update(yaml_actions) logger.debug("Finished loading and registering actions") return actions - +os.environs["RUNNING_TESTS"] = RUNNING_TESTS ACTIONS = load_actions(MOCK_SIGAN, RUNNING_TESTS, DRIVERS_DIR, ACTIONS_DIR) CAPABILITIES = load_capabilities(SENSOR_DEFINITION_FILE) From 64df810c38d6ab23ac661b12e86a6a52dc746eba Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 15 Jan 2024 16:40:03 -0700 Subject: [PATCH 121/255] Fix env error --- src/sensor/settings.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 0cf07b95..1867ba52 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -74,22 +74,18 @@ DRIVERS_DIR = path.join(REPO_ROOT, "drivers") DEFAULT_CALIBRATION_FILE= path.join(CONFIG_DIR, "default_calibration.json") -os.environ["DEFAULT_CALIBRATION_FILE"] = DEFAULT_CALIBRATION_FILE +os.environ["DEFAULT_CALIBRATION_FILE"] = str(DEFAULT_CALIBRATION_FILE) # JSON configs if path.exists(path.join(CONFIG_DIR, "sensor_calibration.json")): SENSOR_CALIBRATION_FILE = path.join(CONFIG_DIR, "sensor_calibration.json") else: SENSOR_CALIBRATION_FILE = DEFAULT_CALIBRATION_FILE -os.environ["SENSOR_CALIBRATION_FILE"] = SENSOR_CALIBRATION_FILE - if path.exists(path.join(CONFIG_DIR, "sigan_calibration.json")): SIGAN_CALIBRATION_FILE = path.join(CONFIG_DIR, "sigan_calibration.json") else: SIGAN_CALIBRATION_FILE = DEFAULT_CALIBRATION_FILE -os.environ["SIGAN_CALIBRATION_FILE"] = SIGAN_CALIBRATION_FILE - if path.exists(path.join(CONFIG_DIR, "sensor_definition.json")): SENSOR_DEFINITION_FILE = path.join(CONFIG_DIR, "sensor_definition.json") MEDIA_ROOT = path.join(REPO_ROOT, "files") @@ -443,10 +439,11 @@ "PRESELECTOR_MODULE", default="its_preselector.web_relay_preselector" ) PRESELECTOR_CLASS = env.str("PRESELECTOR_CLASS", default="WebRelayPreselector") -SWITCH_CONFIGS_DIR = Path(env.str( - "SWITCH_CONFIGS_DIR", default=path.join(CONFIG_DIR, "switches") -)) -os.environ["SWITCH_CONFIGS_DIR"] = SWITCH_CONFIGS_DIR +SWITCH_CONFIGS_DIR =env.str( + "SWITCH_CONFIGS_DIR", default=str(path.join(CONFIG_DIR, "switches")) +) +os.environ["SWITCH_CONFIGS_DIR"] = str(SWITCH_CONFIGS_DIR) +SWITCH_CONFIGS_DIR = Path(SWITCH_CONFIGS_DIR) SIGAN_POWER_CYCLE_STATES = env("SIGAN_POWER_CYCLE_STATES", default=None) SIGAN_POWER_SWITCH = env("SIGAN_POWER_SWITCH", default=None) MAX_FAILURES = env("MAX_FAILURES", default=2) @@ -548,7 +545,7 @@ def load_actions(mock_sigan, running_tests, driver_dir, action_dir): actions.update(yaml_actions) logger.debug("Finished loading and registering actions") return actions -os.environs["RUNNING_TESTS"] = RUNNING_TESTS +os.environ["RUNNING_TESTS"] = RUNNING_TESTS ACTIONS = load_actions(MOCK_SIGAN, RUNNING_TESTS, DRIVERS_DIR, ACTIONS_DIR) CAPABILITIES = load_capabilities(SENSOR_DEFINITION_FILE) From d28091ad4c8b26aeecd37150574d3cf6a8750efa Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 15 Jan 2024 16:46:14 -0700 Subject: [PATCH 122/255] set env vars as strings. --- src/sensor/settings.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 1867ba52..ecf5d78b 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -57,8 +57,9 @@ RUNNING_TESTS = "test" in __cmd RUNNING_DEMO = env.bool("DEMO", default=False) MOCK_SIGAN = env.bool("MOCK_SIGAN", default=False) or RUNNING_DEMO or RUNNING_TESTS +os.environ["MOCK_SIGAN"] = str(MOCK_SIGAN) MOCK_SIGAN_RANDOM = env.bool("MOCK_SIGAN_RANDOM", default=False) - +os.environ["MOCK_SIGAN_RANDOM"] = str(MOCK_SIGAN_RANDOM) # Healthchecks - the existance of any of these indicates an unhealthy state SDR_HEALTHCHECK_FILE = path.join(REPO_ROOT, "sdr_unhealthy") @@ -545,7 +546,7 @@ def load_actions(mock_sigan, running_tests, driver_dir, action_dir): actions.update(yaml_actions) logger.debug("Finished loading and registering actions") return actions -os.environ["RUNNING_TESTS"] = RUNNING_TESTS +os.environ["RUNNING_TESTS"] = str(RUNNING_TESTS) ACTIONS = load_actions(MOCK_SIGAN, RUNNING_TESTS, DRIVERS_DIR, ACTIONS_DIR) CAPABILITIES = load_capabilities(SENSOR_DEFINITION_FILE) From 6be34cbf5ec6f00e0e19240eaf16fce951ece02a Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 09:16:12 -0700 Subject: [PATCH 123/255] Fix handler tests. Sync sensor/wsgi with gunicorn/config.py --- src/handlers/tests/test_handlers.py | 113 +++++++++--------- .../tests/test_initialization.py | 7 +- src/scheduler/tests/utils.py | 12 +- src/sensor/wsgi.py | 69 +++++++++-- 4 files changed, 128 insertions(+), 73 deletions(-) diff --git a/src/handlers/tests/test_handlers.py b/src/handlers/tests/test_handlers.py index e1a4d792..d4084b92 100644 --- a/src/handlers/tests/test_handlers.py +++ b/src/handlers/tests/test_handlers.py @@ -1,15 +1,22 @@ +import logging import pytest -from django import conf +from handlers import sensors from django.conf import settings - from status.models import Location +from scos_actions.hardware.sensor import Sensor +from scos_actions.metadata.utils import construct_geojson_point +from scos_actions.signals import register_sensor +logger = logging.getLogger(__name__) @pytest.mark.django_db def test_db_location_update_handler(): - capabilities = settings.CAPABILITIES - capabilities["sensor"] = {} - capabilities["sensor"]["location"] = {} + location = construct_geojson_point(-105.7, 40.5, 0) + sensor = Sensor(location = location) + register_sensor.send(sensor, sensor = sensor) + logger.debug(f"len(sensors) sensors registered") + logger.debug(f"sigan: {sensors[0].signal_analyzer}") + logger.debug(f"Registered sigan = {sensors}") location = Location() location.gps = False location.height = 10 @@ -18,17 +25,20 @@ def test_db_location_update_handler(): location.description = "test" location.active = True location.save() - assert capabilities["sensor"]["location"]["x"] == 100 - assert capabilities["sensor"]["location"]["y"] == -1 - assert capabilities["sensor"]["location"]["z"] == 10 - assert capabilities["sensor"]["location"]["description"] == "test" + assert sensor.location is not None + assert sensor.location["coordinates"][0] == 100 + assert sensor.location["coordinates"][1] == -1 + assert sensor.location["coordinates"][2] == 10 + @pytest.mark.django_db def test_db_location_update_handler_current_location_none(): - capabilities = settings.CAPABILITIES - capabilities["sensor"] = {} - capabilities["sensor"]["location"] = None + sensor = Sensor() + register_sensor.send(sensor, sensor = sensor) + logger.debug(f"len(sensors) sensors registered") + logger.debug(f"sigan: {sensors[0].signal_analyzer}") + logger.debug(f"Registered sigan = {sensors}") location = Location() location.gps = False location.height = 10 @@ -37,50 +47,39 @@ def test_db_location_update_handler_current_location_none(): location.description = "test" location.active = True location.save() - assert capabilities["sensor"]["location"]["x"] == 100 - assert capabilities["sensor"]["location"]["y"] == -1 - assert capabilities["sensor"]["location"]["z"] == 10 - assert capabilities["sensor"]["location"]["description"] == "test" + assert sensor.location is not None + assert sensor.location["coordinates"][0] == 100 + assert sensor.location["coordinates"][1] == -1 + assert sensor.location["coordinates"][2] == 10 @pytest.mark.django_db def test_db_location_update_handler_not_active(): - capabilities = settings.CAPABILITIES - capabilities["sensor"] = {} - capabilities["sensor"]["location"] = {} + location = construct_geojson_point(-105.7, 40.5, 0) + sensor = Sensor(location=location) + register_sensor.send(sensor, sensor=sensor) + logger.debug(f"len(sensors) sensors registered") + logger.debug(f"sigan: {sensors[0].signal_analyzer}") + logger.debug(f"Registered sigan = {sensors}") location = Location() location.gps = False location.height = 10 location.longitude = 100 location.latitude = -1 + location.description = "" location.active = False - location.description = "test" - location.save() - assert len(capabilities["sensor"]["location"]) == 0 - - -@pytest.mark.django_db -def test_db_location_update_handler_no_description(): - capabilities = settings.CAPABILITIES - capabilities["sensor"] = {} - capabilities["sensor"]["location"] = {} - location = Location() - location.gps = False - location.height = 10 - location.longitude = 100 - location.latitude = -1 location.save() - assert capabilities["sensor"]["location"]["x"] == 100 - assert capabilities["sensor"]["location"]["y"] == -1 - assert capabilities["sensor"]["location"]["z"] == 10 - assert capabilities["sensor"]["location"]["description"] == "" + assert sensor.location is not None + assert sensor.location["coordinates"][0] == -105.7 + assert sensor.location["coordinates"][1] == 40.5 + assert sensor.location["coordinates"][2] == 0 @pytest.mark.django_db def test_db_location_deleted_handler(): - capabilities = settings.CAPABILITIES - capabilities["sensor"] = {} - capabilities["sensor"]["location"] = {} + location = construct_geojson_point(-105.7, 40.5, 0) + sensor = Sensor(location=location) + register_sensor.send(sensor, sensor=sensor) location = Location() location.gps = False location.height = 10 @@ -89,19 +88,19 @@ def test_db_location_deleted_handler(): location.description = "test" location.active = True location.save() - assert capabilities["sensor"]["location"]["x"] == 100 - assert capabilities["sensor"]["location"]["y"] == -1 - assert capabilities["sensor"]["location"]["z"] == 10 - assert capabilities["sensor"]["location"]["description"] == "test" + assert sensor.location is not None + assert sensor.location["coordinates"][0] == 100 + assert sensor.location["coordinates"][1] == -1 + assert sensor.location["coordinates"][2] == 10 location.delete() - assert capabilities["sensor"]["location"] is None + assert sensor.location is None @pytest.mark.django_db def test_db_location_deleted_inactive_handler(): - capabilities = settings.CAPABILITIES - capabilities["sensor"] = {} - capabilities["sensor"]["location"] = {} + location = construct_geojson_point(-105.7, 40.5, 0) + sensor = Sensor(location=location) + register_sensor.send(sensor, sensor=sensor) location = Location() location.gps = False location.height = 10 @@ -110,13 +109,13 @@ def test_db_location_deleted_inactive_handler(): location.description = "test" location.active = True location.save() - assert capabilities["sensor"]["location"]["x"] == 100 - assert capabilities["sensor"]["location"]["y"] == -1 - assert capabilities["sensor"]["location"]["z"] == 10 - assert capabilities["sensor"]["location"]["description"] == "test" + assert sensor.location is not None + assert sensor.location["coordinates"][0] == 100 + assert sensor.location["coordinates"][1] == -1 + assert sensor.location["coordinates"][2] == 10 location.active = False location.delete() - assert capabilities["sensor"]["location"]["x"] == 100 - assert capabilities["sensor"]["location"]["y"] == -1 - assert capabilities["sensor"]["location"]["z"] == 10 - assert capabilities["sensor"]["location"]["description"] == "test" + assert sensor.location is not None + assert sensor.location["coordinates"][0] == 100 + assert sensor.location["coordinates"][1] == -1 + assert sensor.location["coordinates"][2] == 10 diff --git a/src/initialization/tests/test_initialization.py b/src/initialization/tests/test_initialization.py index 0225ab39..d74b2f41 100644 --- a/src/initialization/tests/test_initialization.py +++ b/src/initialization/tests/test_initialization.py @@ -1,9 +1,10 @@ from src.initialization import load_preselector + def test_load_preselector(): - preselector = load_preselector( + preselector = load_preselector(preselector_config= {"name": "test", "base_url": "http://127.0.0.1"}, - "its_preselector.web_relay_preselector", - "WebRelayPreselector", + module="its_preselector.web_relay_preselector", + preselector_class_name = "WebRelayPreselector", sensor_definition={} ) assert preselector is not None \ No newline at end of file diff --git a/src/scheduler/tests/utils.py b/src/scheduler/tests/utils.py index 203a7a33..a28e2c47 100644 --- a/src/scheduler/tests/utils.py +++ b/src/scheduler/tests/utils.py @@ -11,10 +11,14 @@ from django.conf import settings from scos_actions.hardware.mocks.mock_sigan import MockSignalAnalyzer +logger = logging.getLogger(__name__) actions = settings.ACTIONS +if actions is not None: + logger.debug(f"Have {len(actions)} actions") +else: + logger.warning("Actions is None") BAD_ACTION_STR = "testing expected failure" -logger = logging.getLogger(__name__) logger.debug("*************** scos-sensor/scheduler/test/utils ***********") @@ -62,7 +66,7 @@ def advance_testclock(iterator, n): def simulate_scheduler_run(n=1): s = Scheduler() - s.signal_analyzer = MockSignalAnalyzer() + s.sensor = Sensor(signal_analyzer=MockSignalAnalyzer()) for _ in range(n): advance_testclock(s.timefn, 1) s.run(blocking=False) @@ -107,7 +111,7 @@ def create_action(): """ flag = threading.Event() - def cb(schedule_entry_json, task_id): + def cb(sensor, schedule_entry_json, task_id): flag.set() return "set flag" @@ -122,7 +126,7 @@ def cb(schedule_entry_json, task_id): def create_bad_action(): - def bad_action(sigan, gps, schedule_entry_json, task_id): + def bad_action(sensor, schedule_entry_json, task_id): raise Exception(BAD_ACTION_STR) actions["bad_action"] = bad_action diff --git a/src/sensor/wsgi.py b/src/sensor/wsgi.py index b725c8e1..d7cabd6e 100644 --- a/src/sensor/wsgi.py +++ b/src/sensor/wsgi.py @@ -15,8 +15,6 @@ import importlib import logging from django.core.wsgi import get_wsgi_application -from environs import Env -from scos_actions.signals import register_component_with_status os.environ.setdefault("DJANGO_SETTINGS_MODULE", "sensor.settings") django.setup() # this is necessary because we need to handle our own thread @@ -31,18 +29,71 @@ faulthandler.enable() application = get_wsgi_application() +logger = logging.get(__name__) if not settings.IN_DOCKER: # Normally scheduler is started by gunicorn worker process - env = Env() - sigan_module_setting = env("SIGAN_MODULE") + from scheduler import scheduler + from initialization import ( + load_preselector, + load_switches, + ) + from initialization import ( + get_sensor_calibration, + get_sigan_calibration + ) + from django.conf import settings + from status.models import Location + from scos_actions.hardware.sensor import Sensor + from scos_actions.metadata.utils import construct_geojson_point + from scos_actions.signals import register_component_with_status + from scos_actions.signals import register_signal_analyzer + from scos_actions.signals import register_sensor + + sigan_module_setting = settings.SIGAN_MODULE sigan_module = importlib.import_module(sigan_module_setting) - logger = logging.getLogger(__name__) - logger.info("Creating " + env("SIGAN_CLASS") + " from " + env("SIGAN_MODULE")) - sigan_constructor = getattr(sigan_module, env("SIGAN_CLASS")) - sigan = sigan_constructor() + logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) + sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) + sensor_cal = get_sensor_calibration(settings.SENSOR_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) + sigan_cal = get_sigan_calibration(settings.SIGAN_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) + sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal) register_component_with_status.send(sigan, component=sigan) - scheduler.thread.signal_analyzer = sigan + register_signal_analyzer.send(sigan, signal_analyzer=sigan) + + switches = load_switches(settings.SWITCH_CONFIGS_DIR) + capabilities = settings.CAPABILITIES + preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELECTOR_MODULE, settings.PRESELECTOR_CLASS, capabilities["sensor"]) + location = None + if "location" in capabilities["sensor"]: + try: + sensor_loc = capabilities["sensor"].pop("location") + try: + #if there is an active database location, use it over the value in the sensor def. + db_location = Location.objects.get(active=True) + location = construct_geojson_point(db_location.longitude, db_location.latitude, db_location.height) + except Location.DoesNotExist: + # This should never occur because status/migrations/0003_auto_20211217_2229.py + # will load the No DB location. Use sensor def location and save to DB. + location = construct_geojson_point( + sensor_loc["x"], + sensor_loc["y"], + sensor_loc["z"] if "z" in sensor_loc else None, + ) + #Save the sensor location from the sensor def to the database + db_location = Location() + db_location.longitude = sensor_loc["x"] + db_location.latitude = sensor_loc["y"] + db_location.height = sensor_loc["z"] + db_location.gps = False + db_location.description = sensor_loc["description"] + db_location.save() + except: + logger.exception("Failed to get sensor location from sensor definition.") + + + sensor = Sensor(signal_analyzer=sigan, preselector = preselector, switches = switches, capabilities = capabilities, location = location) + scheduler.thread.sensor = sensor + register_sensor.send(sensor, sensor=sensor) scheduler.thread.start() From cba6b7bfcd0836907acb6b8df9bc606734ede558 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 09:25:27 -0700 Subject: [PATCH 124/255] use action_classes in plugins instead of action_types. --- src/sensor/settings.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/sensor/settings.py b/src/sensor/settings.py index ecf5d78b..6755639b 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -525,8 +525,6 @@ def load_actions(mock_sigan, running_tests, driver_dir, action_dir): if name.startswith("scos_") and name != "scos_actions" } logger.debug(discovered_plugins) - action_types = {} - action_types.update(action_classes) actions = {} if mock_sigan or running_tests: for name, action in test_actions.items(): @@ -538,11 +536,11 @@ def load_actions(mock_sigan, running_tests, driver_dir, action_dir): if hasattr(discover, "actions"): logger.debug(f"loading {len(discover.actions)} actions.") actions.update(discover.actions) - if hasattr(discover, "action_types") and discover.action_types is not None: - action_types.update(discover.action_types) + if hasattr(discover, "action_classes") and discover.action_classes is not None: + action_classes.update(discover.action_classes) logger.debug(f"Loading actions in {action_dir}") - yaml_actions, yaml_test_actions = init(action_classes = action_types, yaml_dir=action_dir) + yaml_actions, yaml_test_actions = init(action_classes = action_classes, yaml_dir=action_dir) actions.update(yaml_actions) logger.debug("Finished loading and registering actions") return actions From 78ab76f8b8e180c43c3084df74698b1e0bc69a2f Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 09:32:04 -0700 Subject: [PATCH 125/255] fix logger init. --- src/sensor/wsgi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sensor/wsgi.py b/src/sensor/wsgi.py index d7cabd6e..b495fd99 100644 --- a/src/sensor/wsgi.py +++ b/src/sensor/wsgi.py @@ -29,7 +29,7 @@ faulthandler.enable() application = get_wsgi_application() -logger = logging.get(__name__) +logger = logging.getLogger(__name__) if not settings.IN_DOCKER: # Normally scheduler is started by gunicorn worker process From 51eb890a6efa4192a6b6557eee515d7dd0d185a6 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 10:04:51 -0700 Subject: [PATCH 126/255] move action loading to actions app. --- src/actions/__init__.py | 77 +++++++++++++++++++++++++++ src/capabilities/__init__.py | 3 +- src/schedule/__init__.py | 2 +- src/schedule/models/schedule_entry.py | 4 +- src/schedule/serializers.py | 3 +- src/scheduler/tests/utils.py | 4 +- src/sensor/settings.py | 67 +---------------------- src/tasks/models/task.py | 5 +- 8 files changed, 89 insertions(+), 76 deletions(-) create mode 100644 src/actions/__init__.py diff --git a/src/actions/__init__.py b/src/actions/__init__.py new file mode 100644 index 00000000..f631b5b8 --- /dev/null +++ b/src/actions/__init__.py @@ -0,0 +1,77 @@ +import importlib +import json +import logging +import os + +from django.conf import settings + +from scos_actions.actions import action_classes +from scos_actions.discover import test_actions +from scos_actions.discover import init + +logger = logging.getLogger(__name__) + +def copy_driver_files(driver_dir): + import shutil + """Copy driver files where they need to go""" + for root, dirs, files in os.walk(driver_dir): + for filename in files: + name_without_ext, ext = os.path.splitext(filename) + if ext.lower() == ".json": + json_data = {} + file_path = os.path.join(root, filename) + with open(file_path) as json_file: + json_data = json.load(json_file) + if type(json_data) == dict and "scos_files" in json_data: + scos_files = json_data["scos_files"] + for scos_file in scos_files: + source_path = os.path.join( + driver_dir, scos_file["source_path"] + ) + if not os.path.isfile(source_path): + logger.error(f"Unable to find file at {source_path}") + continue + dest_path = scos_file["dest_path"] + dest_dir = os.path.dirname(dest_path) + try: + if not os.path.isdir(dest_dir): + os.makedirs(dest_dir) + logger.debug(f"copying {source_path} to {dest_path}") + shutil.copyfile(source_path, dest_path) + except Exception as e: + logger.error(f"Failed to copy {source_path} to {dest_path}") + logger.error(e) + +def load_actions(mock_sigan, running_tests, driver_dir, action_dir): + + import pkgutil + logger.debug("********** Initializing actions **********") + + copy_driver_files(driver_dir) # copy driver files before loading plugins + discovered_plugins = { + name: importlib.import_module(name) + for finder, name, ispkg in pkgutil.iter_modules() + if name.startswith("scos_") and name != "scos_actions" + } + logger.debug(discovered_plugins) + actions = {} + if mock_sigan or running_tests: + for name, action in test_actions.items(): + logger.debug("test_action: " + name + "=" + str(action)) + else: + for name, module in discovered_plugins.items(): + logger.debug("Looking for actions in " + name + ": " + str(module)) + discover = importlib.import_module(name + ".discover") + if hasattr(discover, "actions"): + logger.debug(f"loading {len(discover.actions)} actions.") + actions.update(discover.actions) + if hasattr(discover, "action_classes") and discover.action_classes is not None: + action_classes.update(discover.action_classes) + + logger.debug(f"Loading actions in {action_dir}") + yaml_actions, yaml_test_actions = init(action_classes = action_classes, yaml_dir=action_dir) + actions.update(yaml_actions) + logger.debug("Finished loading and registering actions") + return actions + +actions = load_actions(settings.MOCK_SIGAN, settings.RUNNING_TESTS, settings.DRIVER_DIR, settings.ACTION_DIR) \ No newline at end of file diff --git a/src/capabilities/__init__.py b/src/capabilities/__init__.py index ded0dd98..bbcbceff 100644 --- a/src/capabilities/__init__.py +++ b/src/capabilities/__init__.py @@ -1,9 +1,10 @@ import logging from django.conf import settings +from actions import actions logger = logging.getLogger(__name__) logger.debug("********** Initializing capabilities **********") -actions_by_name = settings.ACTIONS +actions_by_name = actions sensor_capabilities = settings.CAPABILITIES diff --git a/src/schedule/__init__.py b/src/schedule/__init__.py index f1e4ac17..2969790b 100644 --- a/src/schedule/__init__.py +++ b/src/schedule/__init__.py @@ -3,7 +3,7 @@ from utils import get_summary from django.conf import settings -actions = settings.ACTIONS +from actions import actions def get_action_with_summary(action): """Given an action, return the string 'action_name - summary'.""" diff --git a/src/schedule/models/schedule_entry.py b/src/schedule/models/schedule_entry.py index f977d6b8..aafb45e5 100644 --- a/src/schedule/models/schedule_entry.py +++ b/src/schedule/models/schedule_entry.py @@ -6,10 +6,10 @@ from django.db import models from constants import MAX_ACTION_LENGTH -from django.conf import settings +from actions import actions from scheduler import utils -actions = settings.ACTIONS + logger = logging.getLogger(__name__) logger.debug( "************** scos-sensor/schedule/models/schedule_entry.py *****************" diff --git a/src/schedule/serializers.py b/src/schedule/serializers.py index 9a2bd09d..8d4b5e97 100644 --- a/src/schedule/serializers.py +++ b/src/schedule/serializers.py @@ -6,14 +6,13 @@ convert_datetime_to_millisecond_iso_format, parse_datetime_iso_format_str, ) -from django.conf import settings +from actions import actions from sensor import V1 from sensor.utils import get_datetime_from_timestamp, get_timestamp_from_datetime from . import get_action_with_summary from .models import DEFAULT_PRIORITY, ScheduleEntry -actions = settings.ACTIONS action_help = "[Required] The name of the action to be scheduled" priority_help = f"Lower number is higher priority (default={DEFAULT_PRIORITY})" CHOICES = [] diff --git a/src/scheduler/tests/utils.py b/src/scheduler/tests/utils.py index a28e2c47..6101edb5 100644 --- a/src/scheduler/tests/utils.py +++ b/src/scheduler/tests/utils.py @@ -4,15 +4,15 @@ import time from itertools import chain, count, islice +from actions import actions from authentication.models import User from schedule.models import Request, ScheduleEntry from scheduler.scheduler import Scheduler from sensor import V1 -from django.conf import settings from scos_actions.hardware.mocks.mock_sigan import MockSignalAnalyzer logger = logging.getLogger(__name__) -actions = settings.ACTIONS + if actions is not None: logger.debug(f"Have {len(actions)} actions") else: diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 6755639b..5337d2af 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -449,38 +449,6 @@ SIGAN_POWER_SWITCH = env("SIGAN_POWER_SWITCH", default=None) MAX_FAILURES = env("MAX_FAILURES", default=2) - -def copy_driver_files(driver_dir): - import shutil - """Copy driver files where they need to go""" - for root, dirs, files in os.walk(driver_dir): - for filename in files: - name_without_ext, ext = os.path.splitext(filename) - if ext.lower() == ".json": - json_data = {} - file_path = os.path.join(root, filename) - with open(file_path) as json_file: - json_data = json.load(json_file) - if type(json_data) == dict and "scos_files" in json_data: - scos_files = json_data["scos_files"] - for scos_file in scos_files: - source_path = os.path.join( - driver_dir, scos_file["source_path"] - ) - if not os.path.isfile(source_path): - logger.error(f"Unable to find file at {source_path}") - continue - dest_path = scos_file["dest_path"] - dest_dir = os.path.dirname(dest_path) - try: - if not os.path.isdir(dest_dir): - os.makedirs(dest_dir) - logger.debug(f"copying {source_path} to {dest_path}") - shutil.copyfile(source_path, dest_path) - except Exception as e: - logger.error(f"Failed to copy {source_path} to {dest_path}") - logger.error(e) - def load_capabilities(sensor_definition_file): from scos_actions.utils import load_from_json capabilities = {} @@ -511,41 +479,8 @@ def load_capabilities(sensor_definition_file): return capabilities -def load_actions(mock_sigan, running_tests, driver_dir, action_dir): - from scos_actions.actions import action_classes - from scos_actions.discover import test_actions - from scos_actions.discover import init - import pkgutil - logger.debug("********** Initializing actions **********") - - copy_driver_files(driver_dir) # copy driver files before loading plugins - discovered_plugins = { - name: importlib.import_module(name) - for finder, name, ispkg in pkgutil.iter_modules() - if name.startswith("scos_") and name != "scos_actions" - } - logger.debug(discovered_plugins) - actions = {} - if mock_sigan or running_tests: - for name, action in test_actions.items(): - logger.debug("test_action: " + name + "=" + str(action)) - else: - for name, module in discovered_plugins.items(): - logger.debug("Looking for actions in " + name + ": " + str(module)) - discover = importlib.import_module(name + ".discover") - if hasattr(discover, "actions"): - logger.debug(f"loading {len(discover.actions)} actions.") - actions.update(discover.actions) - if hasattr(discover, "action_classes") and discover.action_classes is not None: - action_classes.update(discover.action_classes) - - logger.debug(f"Loading actions in {action_dir}") - yaml_actions, yaml_test_actions = init(action_classes = action_classes, yaml_dir=action_dir) - actions.update(yaml_actions) - logger.debug("Finished loading and registering actions") - return actions + os.environ["RUNNING_TESTS"] = str(RUNNING_TESTS) -ACTIONS = load_actions(MOCK_SIGAN, RUNNING_TESTS, DRIVERS_DIR, ACTIONS_DIR) CAPABILITIES = load_capabilities(SENSOR_DEFINITION_FILE) diff --git a/src/tasks/models/task.py b/src/tasks/models/task.py index cded5af9..2b750fb4 100644 --- a/src/tasks/models/task.py +++ b/src/tasks/models/task.py @@ -6,9 +6,10 @@ import logging from collections import namedtuple -from django.conf import settings -actions = settings.ACTIONS +from actions import actions + + logger = logging.getLogger(__name__) logger.debug("*********** scos-sensor/models/task.py ****************") From f15ae64fac7cb1414b3a8fa0ebe5054cd13b3830 Mon Sep 17 00:00:00 2001 From: Justin Haze Date: Tue, 16 Jan 2024 10:04:55 -0700 Subject: [PATCH 127/255] add some additional fixes to create_supseruser.py, address feedback for auth.py log message --- scripts/create_superuser.py | 7 ++++--- src/authentication/auth.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/create_superuser.py b/scripts/create_superuser.py index 97cd9b4c..522825b0 100755 --- a/scripts/create_superuser.py +++ b/scripts/create_superuser.py @@ -22,6 +22,7 @@ def add_user(username, password, email=None): if email: admin_user.email = email admin_user.set_password(password) + admin_user.save() print("Reset admin account password and email from environment") except UserModel.DoesNotExist: UserModel._default_manager.create_superuser(username, email, password) @@ -35,7 +36,7 @@ def add_user(username, password, email=None): print("Retreived admin email from environment variable ADMIN_EMAIL") username = os.environ["ADMIN_NAME"] print("Retreived admin name from environment variable ADMIN_NAME") - add_user(username, password, email) + add_user(username.strip(), password.strip(), email.strip()) except KeyError: print("Not on a managed sensor, so not auto-generating admin account.") print("You can add an admin later with `./manage.py createsuperuser`") @@ -52,7 +53,7 @@ def add_user(username, password, email=None): "ADDITIONAL_USER_PASSWORD" in os.environ and os.environ["ADDITIONAL_USER_PASSWORD"] ): - additional_user_password = os.environ["ADDITIONAL_USER_PASSWORD"] + additional_user_password = os.environ["ADDITIONAL_USER_PASSWORD"].strip() else: # user will have unusable password # https://docs.djangoproject.com/en/3.2/ref/contrib/auth/#django.contrib.auth.models.UserManager.create_user @@ -69,4 +70,4 @@ def add_user(username, password, email=None): for additional_user_name in additional_user_names.split(","): add_user(additional_user_name.strip(), additional_user_password) else: - add_user(additional_user_names, additional_user_password) + add_user(additional_user_names.strip(), additional_user_password) diff --git a/src/authentication/auth.py b/src/authentication/auth.py index 920a5db0..07605e77 100644 --- a/src/authentication/auth.py +++ b/src/authentication/auth.py @@ -30,7 +30,7 @@ def authenticate(self, request): user.last_login = datetime.datetime.now() user.save() except user_model.DoesNotExist: - logger.error("No matching username found!") + logger.error(f"No username matching {cn} found in database!") raise exceptions.AuthenticationFailed("No matching username found!") except Exception as ex: logger.error("Error occurred during certificate authentication!") From 65ec944877e9ee09b972cde9347e91519e988270 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 11:00:34 -0700 Subject: [PATCH 128/255] load actions in actions app. load capabilities in capabilities. Import each from respective apps. Load actions and capabilities before other apps. --- src/actions/__init__.py | 8 +++++-- src/capabilities/__init__.py | 41 ++++++++++++++++++++++++++++++++++-- src/schedule/serializers.py | 11 +++++++++- src/sensor/settings.py | 37 +++----------------------------- src/sensor/wsgi.py | 6 +++--- 5 files changed, 61 insertions(+), 42 deletions(-) diff --git a/src/actions/__init__.py b/src/actions/__init__.py index f631b5b8..d34b2d01 100644 --- a/src/actions/__init__.py +++ b/src/actions/__init__.py @@ -9,6 +9,8 @@ from scos_actions.discover import test_actions from scos_actions.discover import init +from utils.signals import register_action + logger = logging.getLogger(__name__) def copy_driver_files(driver_dir): @@ -64,9 +66,11 @@ def load_actions(mock_sigan, running_tests, driver_dir, action_dir): discover = importlib.import_module(name + ".discover") if hasattr(discover, "actions"): logger.debug(f"loading {len(discover.actions)} actions.") - actions.update(discover.actions) + for name, action in discover.actions.items(): + register_action.send(name = name, action = action) if hasattr(discover, "action_classes") and discover.action_classes is not None: - action_classes.update(discover.action_classes) + for name, action in discover.action_classes.items(): + register_action_class(name= name, action=action) logger.debug(f"Loading actions in {action_dir}") yaml_actions, yaml_test_actions = init(action_classes = action_classes, yaml_dir=action_dir) diff --git a/src/capabilities/__init__.py b/src/capabilities/__init__.py index bbcbceff..7cd8686b 100644 --- a/src/capabilities/__init__.py +++ b/src/capabilities/__init__.py @@ -1,10 +1,47 @@ +import hashlib +import json import logging -from django.conf import settings from actions import actions +from django.conf import settings +from scos_actions.utils import load_from_json + logger = logging.getLogger(__name__) logger.debug("********** Initializing capabilities **********") actions_by_name = actions -sensor_capabilities = settings.CAPABILITIES + +def load_capabilities(sensor_definition_file): + + capabilities = {} + sensor_definition_hash = None + sensor_location = None + + logger.debug(f"Loading {sensor_definition_file}") + try: + capabilities["sensor"] = load_from_json(sensor_definition_file) + except Exception as e: + logger.warning( + f"Failed to load sensor definition file: {sensor_definition_file}" + + "\nAn empty sensor definition will be used" + ) + capabilities["sensor"] = {"sensor_spec": {"id": "unknown"}} + capabilities["sensor"]["sensor_sha512"] = "UNKNOWN SENSOR DEFINITION" + + # Generate sensor definition file hash (SHA 512) + try: + if "sensor_sha512" not in capabilities["sensor"]: + sensor_def = json.dumps(capabilities["sensor"], sort_keys=True) + sensor_definition_hash = hashlib.sha512(sensor_def.encode("UTF-8")).hexdigest() + capabilities["sensor"]["sensor_sha512"] = sensor_definition_hash + except: + capabilities["sensor"]["sensor_sha512"] = "ERROR GENERATING HASH" + # sensor_sha512 is None, do not raise Exception, but log it + logger.exception(f"Unable to generate sensor definition hash") + + return capabilities + +logger.debug("Capabilites connected to register_action") + +sensor_capabilities = load_capabilities(settings.SENSOR_DEFINITION_FILE) diff --git a/src/schedule/serializers.py b/src/schedule/serializers.py index 8d4b5e97..31507e91 100644 --- a/src/schedule/serializers.py +++ b/src/schedule/serializers.py @@ -6,13 +6,22 @@ convert_datetime_to_millisecond_iso_format, parse_datetime_iso_format_str, ) -from actions import actions + from sensor import V1 from sensor.utils import get_datetime_from_timestamp, get_timestamp_from_datetime from . import get_action_with_summary from .models import DEFAULT_PRIORITY, ScheduleEntry +actions = {} + +def action_registered(sender, **kwargs): + name = kwargs["name"] + action = kwargs["action"] + logger.debug(f"Adding {name}: {action} to capabilities") + actions_by_name[name] = actions + + action_help = "[Required] The name of the action to be scheduled" priority_help = f"Lower number is higher priority (default={DEFAULT_PRIORITY})" CHOICES = [] diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 5337d2af..e79493a5 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -197,8 +197,9 @@ "rest_framework.authtoken", "drf_yasg", # OpenAPI generator # project-local apps - "authentication.apps.AuthenticationConfig", + "actions.apps.ActionsConfig", "capabilities.apps.CapabilitiesConfig", + "authentication.apps.AuthenticationConfig", "handlers.apps.HandlersConfig", "tasks.apps.TasksConfig", "schedule.apps.ScheduleConfig", @@ -448,40 +449,8 @@ SIGAN_POWER_CYCLE_STATES = env("SIGAN_POWER_CYCLE_STATES", default=None) SIGAN_POWER_SWITCH = env("SIGAN_POWER_SWITCH", default=None) MAX_FAILURES = env("MAX_FAILURES", default=2) - -def load_capabilities(sensor_definition_file): - from scos_actions.utils import load_from_json - capabilities = {} - sensor_definition_hash = None - sensor_location = None - - logger.debug(f"Loading {sensor_definition_file}") - try: - capabilities["sensor"] = load_from_json(sensor_definition_file) - except Exception as e: - logger.warning( - f"Failed to load sensor definition file: {sensor_definition_file}" - + "\nAn empty sensor definition will be used" - ) - capabilities["sensor"] = {"sensor_spec": {"id": "unknown"}} - capabilities["sensor"]["sensor_sha512"] = "UNKNOWN SENSOR DEFINITION" - - # Generate sensor definition file hash (SHA 512) - try: - if "sensor_sha512" not in capabilities["sensor"]: - sensor_def = json.dumps(capabilities["sensor"], sort_keys=True) - sensor_definition_hash = hashlib.sha512(sensor_def.encode("UTF-8")).hexdigest() - capabilities["sensor"]["sensor_sha512"] = sensor_definition_hash - except: - capabilities["sensor"]["sensor_sha512"] = "ERROR GENERATING HASH" - # sensor_sha512 is None, do not raise Exception, but log it - logger.exception(f"Unable to generate sensor definition hash") - - return capabilities - - os.environ["RUNNING_TESTS"] = str(RUNNING_TESTS) -CAPABILITIES = load_capabilities(SENSOR_DEFINITION_FILE) + diff --git a/src/sensor/wsgi.py b/src/sensor/wsgi.py index b495fd99..635bd5ab 100644 --- a/src/sensor/wsgi.py +++ b/src/sensor/wsgi.py @@ -19,6 +19,7 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "sensor.settings") django.setup() # this is necessary because we need to handle our own thread +from capabilities import sensor_capabilities from scheduler import scheduler # noqa from sensor import settings # noqa @@ -61,12 +62,11 @@ register_signal_analyzer.send(sigan, signal_analyzer=sigan) switches = load_switches(settings.SWITCH_CONFIGS_DIR) - capabilities = settings.CAPABILITIES preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELECTOR_MODULE, settings.PRESELECTOR_CLASS, capabilities["sensor"]) location = None - if "location" in capabilities["sensor"]: + if "location" in sensor_capabilities["sensor"]: try: - sensor_loc = capabilities["sensor"].pop("location") + sensor_loc = sensor_capabilities["sensor"].pop("location") try: #if there is an active database location, use it over the value in the sensor def. db_location = Location.objects.get(active=True) From 0fb769d609b3c4bfbd69ebd86fabaa7324d5b5d5 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 11:06:52 -0700 Subject: [PATCH 129/255] don't use signals in action loading. --- src/actions/__init__.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/actions/__init__.py b/src/actions/__init__.py index d34b2d01..8242e9ba 100644 --- a/src/actions/__init__.py +++ b/src/actions/__init__.py @@ -9,8 +9,6 @@ from scos_actions.discover import test_actions from scos_actions.discover import init -from utils.signals import register_action - logger = logging.getLogger(__name__) def copy_driver_files(driver_dir): @@ -66,11 +64,10 @@ def load_actions(mock_sigan, running_tests, driver_dir, action_dir): discover = importlib.import_module(name + ".discover") if hasattr(discover, "actions"): logger.debug(f"loading {len(discover.actions)} actions.") - for name, action in discover.actions.items(): - register_action.send(name = name, action = action) + actions.update(discover.actions) if hasattr(discover, "action_classes") and discover.action_classes is not None: - for name, action in discover.action_classes.items(): - register_action_class(name= name, action=action) + action_classes.update(discover.action_classes) + logger.debug(f"Loading actions in {action_dir}") yaml_actions, yaml_test_actions = init(action_classes = action_classes, yaml_dir=action_dir) From 0bb485d9cf3f7f27d80c93a5e80ce1861f65f5d9 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 11:12:18 -0700 Subject: [PATCH 130/255] DRIVERS_DIR typo --- src/actions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/actions/__init__.py b/src/actions/__init__.py index 8242e9ba..3ec815c5 100644 --- a/src/actions/__init__.py +++ b/src/actions/__init__.py @@ -75,4 +75,4 @@ def load_actions(mock_sigan, running_tests, driver_dir, action_dir): logger.debug("Finished loading and registering actions") return actions -actions = load_actions(settings.MOCK_SIGAN, settings.RUNNING_TESTS, settings.DRIVER_DIR, settings.ACTION_DIR) \ No newline at end of file +actions = load_actions(settings.MOCK_SIGAN, settings.RUNNING_TESTS, settings.DRIVERS_DIR, settings.ACTION_DIR) \ No newline at end of file From d164b8312fcdaa9b83b1cfe4f22c9070f1c38da2 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 11:17:08 -0700 Subject: [PATCH 131/255] actions_dir typo --- src/actions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/actions/__init__.py b/src/actions/__init__.py index 3ec815c5..2871d4d6 100644 --- a/src/actions/__init__.py +++ b/src/actions/__init__.py @@ -75,4 +75,4 @@ def load_actions(mock_sigan, running_tests, driver_dir, action_dir): logger.debug("Finished loading and registering actions") return actions -actions = load_actions(settings.MOCK_SIGAN, settings.RUNNING_TESTS, settings.DRIVERS_DIR, settings.ACTION_DIR) \ No newline at end of file +actions = load_actions(settings.MOCK_SIGAN, settings.RUNNING_TESTS, settings.DRIVERS_DIR, settings.ACTIONS_DIR) \ No newline at end of file From 6e98d028dd3067f0dd8e359b9832caa1e46bf9e3 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 11:23:30 -0700 Subject: [PATCH 132/255] add missing actions/apps.py --- src/actions/apps.py | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 src/actions/apps.py diff --git a/src/actions/apps.py b/src/actions/apps.py new file mode 100644 index 00000000..8eadffac --- /dev/null +++ b/src/actions/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class ActionsConfig(AppConfig): + name = "actions" \ No newline at end of file From 78f1813d72129805baa97ea61a22700ff08b4f9f Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 11:33:38 -0700 Subject: [PATCH 133/255] import sensor_capabilities from capabilities. --- gunicorn/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index e210b943..c0650f66 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -31,6 +31,7 @@ def post_worker_init(worker): import django django.setup() + from capabilities import sensor_capabilities from scheduler import scheduler from initialization import ( load_preselector, @@ -59,7 +60,7 @@ def post_worker_init(worker): register_signal_analyzer.send(sigan, signal_analyzer=sigan) switches = load_switches(settings.SWITCH_CONFIGS_DIR) - capabilities = settings.CAPABILITIES + capabilities = sensor_capabilities preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELECTOR_MODULE, settings.PRESELECTOR_CLASS, capabilities["sensor"]) location = None if "location" in capabilities["sensor"]: From 12b9c59290d16df3ee7b257f2920ff70d1906c87 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 12:55:00 -0700 Subject: [PATCH 134/255] try ActionLoader singleton. --- scripts/print_action_docstring.py | 4 +- src/actions/__init__.py | 68 +------------------- src/actions/action_loader.py | 89 +++++++++++++++++++++++++++ src/capabilities/__init__.py | 4 +- src/schedule/__init__.py | 4 +- src/schedule/models/schedule_entry.py | 4 +- src/scheduler/tests/utils.py | 4 +- src/tasks/models/task.py | 4 +- 8 files changed, 104 insertions(+), 77 deletions(-) create mode 100644 src/actions/action_loader.py diff --git a/scripts/print_action_docstring.py b/scripts/print_action_docstring.py index 99023f32..5caef350 100755 --- a/scripts/print_action_docstring.py +++ b/scripts/print_action_docstring.py @@ -6,7 +6,7 @@ import django -from actions import actions +from actions import action_loader PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "src") @@ -16,7 +16,7 @@ django.setup() -action_names = sorted(actions.keys()) +action_names = sorted(action_loader.actions.keys()) if __name__ == "__main__": diff --git a/src/actions/__init__.py b/src/actions/__init__.py index 2871d4d6..d147fdf9 100644 --- a/src/actions/__init__.py +++ b/src/actions/__init__.py @@ -9,70 +9,8 @@ from scos_actions.discover import test_actions from scos_actions.discover import init -logger = logging.getLogger(__name__) - -def copy_driver_files(driver_dir): - import shutil - """Copy driver files where they need to go""" - for root, dirs, files in os.walk(driver_dir): - for filename in files: - name_without_ext, ext = os.path.splitext(filename) - if ext.lower() == ".json": - json_data = {} - file_path = os.path.join(root, filename) - with open(file_path) as json_file: - json_data = json.load(json_file) - if type(json_data) == dict and "scos_files" in json_data: - scos_files = json_data["scos_files"] - for scos_file in scos_files: - source_path = os.path.join( - driver_dir, scos_file["source_path"] - ) - if not os.path.isfile(source_path): - logger.error(f"Unable to find file at {source_path}") - continue - dest_path = scos_file["dest_path"] - dest_dir = os.path.dirname(dest_path) - try: - if not os.path.isdir(dest_dir): - os.makedirs(dest_dir) - logger.debug(f"copying {source_path} to {dest_path}") - shutil.copyfile(source_path, dest_path) - except Exception as e: - logger.error(f"Failed to copy {source_path} to {dest_path}") - logger.error(e) - -def load_actions(mock_sigan, running_tests, driver_dir, action_dir): - - import pkgutil - logger.debug("********** Initializing actions **********") +from action_loader import ActionLoader - copy_driver_files(driver_dir) # copy driver files before loading plugins - discovered_plugins = { - name: importlib.import_module(name) - for finder, name, ispkg in pkgutil.iter_modules() - if name.startswith("scos_") and name != "scos_actions" - } - logger.debug(discovered_plugins) - actions = {} - if mock_sigan or running_tests: - for name, action in test_actions.items(): - logger.debug("test_action: " + name + "=" + str(action)) - else: - for name, module in discovered_plugins.items(): - logger.debug("Looking for actions in " + name + ": " + str(module)) - discover = importlib.import_module(name + ".discover") - if hasattr(discover, "actions"): - logger.debug(f"loading {len(discover.actions)} actions.") - actions.update(discover.actions) - if hasattr(discover, "action_classes") and discover.action_classes is not None: - action_classes.update(discover.action_classes) - - - logger.debug(f"Loading actions in {action_dir}") - yaml_actions, yaml_test_actions = init(action_classes = action_classes, yaml_dir=action_dir) - actions.update(yaml_actions) - logger.debug("Finished loading and registering actions") - return actions +logger = logging.getLogger(__name__) -actions = load_actions(settings.MOCK_SIGAN, settings.RUNNING_TESTS, settings.DRIVERS_DIR, settings.ACTIONS_DIR) \ No newline at end of file +action_loader = ActionLoader() diff --git a/src/actions/action_loader.py b/src/actions/action_loader.py new file mode 100644 index 00000000..66dc8d5c --- /dev/null +++ b/src/actions/action_loader.py @@ -0,0 +1,89 @@ +import importlib +import logging +import os +import pkgutil +import shutil +from django.conf import settings +from scos_actions import action_classes +from scos_actions import test_actions + +logger = logging.getLogger(__name__) + +class ActionLoader(object): + _instance = None + _actions = None + def __new__(cls): + if cls._instance is None: + logger.debug('Creating the ActionLoader') + cls._instance = super(ActionLoader, cls).__new__(cls) + _actions = load_actions(settings.MOCK_SIGAN, settings.RUNNING_TESTS, settings.DRIVERS_DIR, settings.ACTIONS_DIR) + return cls._instance + + @property + def actions(self): + return self._actions + + @actions.setter + def actions(self, value): + self._actions = value + +def copy_driver_files(driver_dir): + """Copy driver files where they need to go""" + for root, dirs, files in os.walk(driver_dir): + for filename in files: + name_without_ext, ext = os.path.splitext(filename) + if ext.lower() == ".json": + json_data = {} + file_path = os.path.join(root, filename) + with open(file_path) as json_file: + json_data = json.load(json_file) + if type(json_data) == dict and "scos_files" in json_data: + scos_files = json_data["scos_files"] + for scos_file in scos_files: + source_path = os.path.join( + driver_dir, scos_file["source_path"] + ) + if not os.path.isfile(source_path): + logger.error(f"Unable to find file at {source_path}") + continue + dest_path = scos_file["dest_path"] + dest_dir = os.path.dirname(dest_path) + try: + if not os.path.isdir(dest_dir): + os.makedirs(dest_dir) + logger.debug(f"copying {source_path} to {dest_path}") + shutil.copyfile(source_path, dest_path) + except Exception as e: + logger.error(f"Failed to copy {source_path} to {dest_path}") + logger.error(e) + +def load_actions(mock_sigan, running_tests, driver_dir, action_dir): + + logger.debug("********** Initializing actions **********") + copy_driver_files(driver_dir) # copy driver files before loading plugins + discovered_plugins = { + name: importlib.import_module(name) + for finder, name, ispkg in pkgutil.iter_modules() + if name.startswith("scos_") and name != "scos_actions" + } + logger.debug(discovered_plugins) + actions = {} + + if mock_sigan or running_tests: + for name, action in test_actions.items(): + logger.debug("test_action: " + name + "=" + str(action)) + else: + for name, module in discovered_plugins.items(): + logger.debug("Looking for actions in " + name + ": " + str(module)) + discover = importlib.import_module(name + ".discover") + if hasattr(discover, "actions"): + logger.debug(f"loading {len(discover.actions)} actions.") + actions.update(discover.actions) + if hasattr(discover, "action_classes") and discover.action_classes is not None: + action_classes.update(discover.action_classes) + + logger.debug(f"Loading actions in {action_dir}") + yaml_actions, yaml_test_actions = init(action_classes=action_classes, yaml_dir=action_dir) + actions.update(yaml_actions) + logger.debug("Finished loading and registering actions") + return actions diff --git a/src/capabilities/__init__.py b/src/capabilities/__init__.py index 7cd8686b..d2a8f248 100644 --- a/src/capabilities/__init__.py +++ b/src/capabilities/__init__.py @@ -2,7 +2,7 @@ import json import logging -from actions import actions +from actions import actions_loader from django.conf import settings from scos_actions.utils import load_from_json @@ -10,7 +10,7 @@ logger = logging.getLogger(__name__) logger.debug("********** Initializing capabilities **********") -actions_by_name = actions +actions_by_name = actions_loader.actions def load_capabilities(sensor_definition_file): diff --git a/src/schedule/__init__.py b/src/schedule/__init__.py index 2969790b..e227affe 100644 --- a/src/schedule/__init__.py +++ b/src/schedule/__init__.py @@ -3,11 +3,11 @@ from utils import get_summary from django.conf import settings -from actions import actions +from actions import action_loader def get_action_with_summary(action): """Given an action, return the string 'action_name - summary'.""" - action_fn = actions[action] + action_fn = action_loader.actions[action] summary = get_summary(action_fn) action_with_summary = action if summary: diff --git a/src/schedule/models/schedule_entry.py b/src/schedule/models/schedule_entry.py index aafb45e5..e5d4d1c2 100644 --- a/src/schedule/models/schedule_entry.py +++ b/src/schedule/models/schedule_entry.py @@ -6,7 +6,7 @@ from django.db import models from constants import MAX_ACTION_LENGTH -from actions import actions +from actions import action_loader from scheduler import utils @@ -167,7 +167,7 @@ def __init__(self, *args, **kwargs): # used by .save to detect whether to reset .next_task_times self.__start = self.start self.__interval = self.interval - if self.action not in actions: + if self.action not in action_loader.actions: raise ValidationError(self.action + " does not exist") def update(self, *args, **kwargs): diff --git a/src/scheduler/tests/utils.py b/src/scheduler/tests/utils.py index 6101edb5..28768834 100644 --- a/src/scheduler/tests/utils.py +++ b/src/scheduler/tests/utils.py @@ -4,7 +4,7 @@ import time from itertools import chain, count, islice -from actions import actions +from actions import action_loader from authentication.models import User from schedule.models import Request, ScheduleEntry from scheduler.scheduler import Scheduler @@ -12,7 +12,7 @@ from scos_actions.hardware.mocks.mock_sigan import MockSignalAnalyzer logger = logging.getLogger(__name__) - +actions = action_loader.actions if actions is not None: logger.debug(f"Have {len(actions)} actions") else: diff --git a/src/tasks/models/task.py b/src/tasks/models/task.py index 2b750fb4..775ec37d 100644 --- a/src/tasks/models/task.py +++ b/src/tasks/models/task.py @@ -7,7 +7,7 @@ import logging from collections import namedtuple -from actions import actions +from actions import action_loader logger = logging.getLogger(__name__) @@ -21,7 +21,7 @@ class Task(TaskTuple): @property def action_caller(self): """Action function with curried keyword arguments""" - action_caller = actions[self.action] + action_caller = action_loader.actions[self.action] return action_caller def __eq__(s, o): From ef3204746c01a00c6bc851258ab0b1044b22b789 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Jan 2024 14:55:18 -0500 Subject: [PATCH 135/255] Bump jinja2 from 3.1.2 to 3.1.3 in /src (#265) * Bump jinja2 from 3.1.2 to 3.1.3 in /src Bumps [jinja2](https://github.com/pallets/jinja) from 3.1.2 to 3.1.3. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/3.1.2...3.1.3) --- updated-dependencies: - dependency-name: jinja2 dependency-type: indirect ... Signed-off-by: dependabot[bot] * pip-compile dependencies with jinja2>=3.1.3 * fix comment alignment --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Anthony Romaniello --- src/requirements-dev.txt | 2 +- src/requirements.in | 1 + src/requirements.txt | 6 ++++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index b4a97319..96beb804 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -149,7 +149,7 @@ itypes==1.2.0 # via # -r requirements.txt # coreapi -jinja2==3.1.2 +jinja2==3.1.3 # via # -r requirements.txt # coreschema diff --git a/src/requirements.in b/src/requirements.in index c34c131a..b99a9f25 100644 --- a/src/requirements.in +++ b/src/requirements.in @@ -18,5 +18,6 @@ scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@4.0.0 # The following are sub-dependencies for which SCOS Sensor enforces a # higher minimum patch version than the dependencies which require them. # This is done to ensure the inclusion of specific security patches. +jinja2>=3.1.3 # CVE-2024-22195 pyyaml>=5.4.0 # CVE-2020-14343 urllib3>=1.26.18 # CVE-2023-45803 diff --git a/src/requirements.txt b/src/requirements.txt index c73aa7f2..fe7f191b 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -69,8 +69,10 @@ its-preselector @ git+https://github.com/NTIA/Preselector@3.1.0 # via scos-actions itypes==1.2.0 # via coreapi -jinja2==3.1.2 - # via coreschema +jinja2==3.1.3 + # via + # -r requirements.in + # coreschema jsonfield==3.1.0 # via -r requirements.in jsonschema==3.2.0 From 13d1cdd17ca51a8049d033f618d9a43c25f56c52 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 13:01:42 -0700 Subject: [PATCH 136/255] typo --- src/actions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/actions/__init__.py b/src/actions/__init__.py index d147fdf9..45014bc1 100644 --- a/src/actions/__init__.py +++ b/src/actions/__init__.py @@ -9,7 +9,7 @@ from scos_actions.discover import test_actions from scos_actions.discover import init -from action_loader import ActionLoader +from .action_loader import ActionLoader logger = logging.getLogger(__name__) From 5e796f0b02c9fff054b2cbc399089d9afd983734 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 13:12:06 -0700 Subject: [PATCH 137/255] fix scos_actions imports. --- src/actions/action_loader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/actions/action_loader.py b/src/actions/action_loader.py index 66dc8d5c..ad07dfa0 100644 --- a/src/actions/action_loader.py +++ b/src/actions/action_loader.py @@ -4,8 +4,8 @@ import pkgutil import shutil from django.conf import settings -from scos_actions import action_classes -from scos_actions import test_actions +from scos_actions.actions import action_classes +from scos_actions.discover import test_actions logger = logging.getLogger(__name__) From 9a1190df2f31df40e08f47e15dfd8678afab994c Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 13:17:14 -0700 Subject: [PATCH 138/255] typo fix. --- src/capabilities/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/capabilities/__init__.py b/src/capabilities/__init__.py index d2a8f248..bf98f0d1 100644 --- a/src/capabilities/__init__.py +++ b/src/capabilities/__init__.py @@ -2,7 +2,7 @@ import json import logging -from actions import actions_loader +from actions import action_loader from django.conf import settings from scos_actions.utils import load_from_json @@ -10,7 +10,7 @@ logger = logging.getLogger(__name__) logger.debug("********** Initializing capabilities **********") -actions_by_name = actions_loader.actions +actions_by_name = action_loader.actions def load_capabilities(sensor_definition_file): From d99a3477c65651f2fffab3398b7542e21f192294 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 13:39:02 -0700 Subject: [PATCH 139/255] debugging. --- src/actions/action_loader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/actions/action_loader.py b/src/actions/action_loader.py index ad07dfa0..8208b023 100644 --- a/src/actions/action_loader.py +++ b/src/actions/action_loader.py @@ -16,6 +16,7 @@ def __new__(cls): if cls._instance is None: logger.debug('Creating the ActionLoader') cls._instance = super(ActionLoader, cls).__new__(cls) + logger.debug(f"Calling load_actions with {settings.MOCK_SIGAN}, {settings.RUNNING_TESTS}, {settings.DRIVERS_DIR}, {settings.ACTIONS_DIR}") _actions = load_actions(settings.MOCK_SIGAN, settings.RUNNING_TESTS, settings.DRIVERS_DIR, settings.ACTIONS_DIR) return cls._instance @@ -29,6 +30,7 @@ def actions(self, value): def copy_driver_files(driver_dir): """Copy driver files where they need to go""" + logger.debug(f"Copying driver files in {driver_dir}") for root, dirs, files in os.walk(driver_dir): for filename in files: name_without_ext, ext = os.path.splitext(filename) From 72d5a6be14b73772d69bd415d6c0ff7fb91cbcd0 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 13:45:24 -0700 Subject: [PATCH 140/255] add missing import. --- src/actions/action_loader.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/actions/action_loader.py b/src/actions/action_loader.py index 8208b023..e9c7faa0 100644 --- a/src/actions/action_loader.py +++ b/src/actions/action_loader.py @@ -1,4 +1,5 @@ import importlib +import json import logging import os import pkgutil From 7c89d38086ca46d578bbe86de64e354ca3dc7c1a Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 14:35:48 -0700 Subject: [PATCH 141/255] debugging --- src/actions/__init__.py | 1 + src/capabilities/__init__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/actions/__init__.py b/src/actions/__init__.py index 45014bc1..5815d5f4 100644 --- a/src/actions/__init__.py +++ b/src/actions/__init__.py @@ -14,3 +14,4 @@ logger = logging.getLogger(__name__) action_loader = ActionLoader() +logger.debug(f"Actions ActionLoader has {len(action_loader.actions)} actions") diff --git a/src/capabilities/__init__.py b/src/capabilities/__init__.py index bf98f0d1..68666f6a 100644 --- a/src/capabilities/__init__.py +++ b/src/capabilities/__init__.py @@ -11,7 +11,7 @@ logger = logging.getLogger(__name__) logger.debug("********** Initializing capabilities **********") actions_by_name = action_loader.actions - +logger.debug(f"ActionLoader has {len(action_loader.actions)} actions") def load_capabilities(sensor_definition_file): capabilities = {} From cfa4b49bf3d71f4081f3dbda1973ce6f61926192 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 14:42:33 -0700 Subject: [PATCH 142/255] Initialize actions as empty dictionary and set _instance actions. --- src/actions/action_loader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/actions/action_loader.py b/src/actions/action_loader.py index e9c7faa0..11c58a01 100644 --- a/src/actions/action_loader.py +++ b/src/actions/action_loader.py @@ -12,13 +12,13 @@ class ActionLoader(object): _instance = None - _actions = None + _actions = {} def __new__(cls): if cls._instance is None: logger.debug('Creating the ActionLoader') cls._instance = super(ActionLoader, cls).__new__(cls) logger.debug(f"Calling load_actions with {settings.MOCK_SIGAN}, {settings.RUNNING_TESTS}, {settings.DRIVERS_DIR}, {settings.ACTIONS_DIR}") - _actions = load_actions(settings.MOCK_SIGAN, settings.RUNNING_TESTS, settings.DRIVERS_DIR, settings.ACTIONS_DIR) + cls._instance._actions = load_actions(settings.MOCK_SIGAN, settings.RUNNING_TESTS, settings.DRIVERS_DIR, settings.ACTIONS_DIR) return cls._instance @property From 40bf0f25f007b2b9503a87948c4527f50d860bb3 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 15:02:05 -0700 Subject: [PATCH 143/255] remove setter for ActionLoader. --- src/actions/action_loader.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/actions/action_loader.py b/src/actions/action_loader.py index 11c58a01..52b11a97 100644 --- a/src/actions/action_loader.py +++ b/src/actions/action_loader.py @@ -13,21 +13,19 @@ class ActionLoader(object): _instance = None _actions = {} + def __new__(cls): if cls._instance is None: logger.debug('Creating the ActionLoader') cls._instance = super(ActionLoader, cls).__new__(cls) logger.debug(f"Calling load_actions with {settings.MOCK_SIGAN}, {settings.RUNNING_TESTS}, {settings.DRIVERS_DIR}, {settings.ACTIONS_DIR}") - cls._instance._actions = load_actions(settings.MOCK_SIGAN, settings.RUNNING_TESTS, settings.DRIVERS_DIR, settings.ACTIONS_DIR) + cls._actions = load_actions(settings.MOCK_SIGAN, settings.RUNNING_TESTS, settings.DRIVERS_DIR, settings.ACTIONS_DIR) return cls._instance @property def actions(self): return self._actions - @actions.setter - def actions(self, value): - self._actions = value def copy_driver_files(driver_dir): """Copy driver files where they need to go""" From 67b098c7d95af6a3d3dd960c0b2bb53aa5276f8a Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 15:08:07 -0700 Subject: [PATCH 144/255] set cls._instance.actions. --- src/actions/action_loader.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/actions/action_loader.py b/src/actions/action_loader.py index 52b11a97..c6f4c05e 100644 --- a/src/actions/action_loader.py +++ b/src/actions/action_loader.py @@ -19,13 +19,17 @@ def __new__(cls): logger.debug('Creating the ActionLoader') cls._instance = super(ActionLoader, cls).__new__(cls) logger.debug(f"Calling load_actions with {settings.MOCK_SIGAN}, {settings.RUNNING_TESTS}, {settings.DRIVERS_DIR}, {settings.ACTIONS_DIR}") - cls._actions = load_actions(settings.MOCK_SIGAN, settings.RUNNING_TESTS, settings.DRIVERS_DIR, settings.ACTIONS_DIR) + cls._instance.actions = load_actions(settings.MOCK_SIGAN, settings.RUNNING_TESTS, settings.DRIVERS_DIR, settings.ACTIONS_DIR) return cls._instance @property def actions(self): return self._actions + @actions.setter + def actions(self, value): + self._actions = value + def copy_driver_files(driver_dir): """Copy driver files where they need to go""" From a0c47b605c27766ef167392180031d3eee36e124 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 15:17:11 -0700 Subject: [PATCH 145/255] add init to ActionLoader. --- src/actions/action_loader.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/actions/action_loader.py b/src/actions/action_loader.py index c6f4c05e..3e317ea8 100644 --- a/src/actions/action_loader.py +++ b/src/actions/action_loader.py @@ -12,7 +12,9 @@ class ActionLoader(object): _instance = None - _actions = {} + + def __init__(self): + self._actions = {} def __new__(cls): if cls._instance is None: From 2c7db9078b69b3e30c793cc002acc6ec4db86f95 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 15:32:52 -0700 Subject: [PATCH 146/255] move ActionLoader actions initialization into __init__ --- src/actions/action_loader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/actions/action_loader.py b/src/actions/action_loader.py index 3e317ea8..f9b8d95c 100644 --- a/src/actions/action_loader.py +++ b/src/actions/action_loader.py @@ -14,14 +14,14 @@ class ActionLoader(object): _instance = None def __init__(self): - self._actions = {} + self._actions = load_actions(settings.MOCK_SIGAN, settings.RUNNING_TESTS, settings.DRIVERS_DIR, + settings.ACTIONS_DIR) def __new__(cls): if cls._instance is None: logger.debug('Creating the ActionLoader') cls._instance = super(ActionLoader, cls).__new__(cls) logger.debug(f"Calling load_actions with {settings.MOCK_SIGAN}, {settings.RUNNING_TESTS}, {settings.DRIVERS_DIR}, {settings.ACTIONS_DIR}") - cls._instance.actions = load_actions(settings.MOCK_SIGAN, settings.RUNNING_TESTS, settings.DRIVERS_DIR, settings.ACTIONS_DIR) return cls._instance @property From cda5e6b16edce893a94eda40c65ac9d4350de958 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 15:50:50 -0700 Subject: [PATCH 147/255] import init. --- src/actions/action_loader.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/actions/action_loader.py b/src/actions/action_loader.py index f9b8d95c..07e28a19 100644 --- a/src/actions/action_loader.py +++ b/src/actions/action_loader.py @@ -7,6 +7,7 @@ from django.conf import settings from scos_actions.actions import action_classes from scos_actions.discover import test_actions +from scos_actions.discover import init logger = logging.getLogger(__name__) From 89e482a13e60fe763397eb771df1096be84db97a Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 15:58:59 -0700 Subject: [PATCH 148/255] check if actions have been loaded. --- src/actions/action_loader.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/actions/action_loader.py b/src/actions/action_loader.py index 07e28a19..0817032e 100644 --- a/src/actions/action_loader.py +++ b/src/actions/action_loader.py @@ -15,8 +15,12 @@ class ActionLoader(object): _instance = None def __init__(self): - self._actions = load_actions(settings.MOCK_SIGAN, settings.RUNNING_TESTS, settings.DRIVERS_DIR, + if self._actions is None: + logger.debug("Actions have not been loaded. Loading actions...") + self._actions = load_actions(settings.MOCK_SIGAN, settings.RUNNING_TESTS, settings.DRIVERS_DIR, settings.ACTIONS_DIR) + else: + logger.debug("Already loaded actions. ") def __new__(cls): if cls._instance is None: From 037922a608775d744b318260dc5e21c3011c7a37 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 16:04:16 -0700 Subject: [PATCH 149/255] check if ActionLoader has attribute. --- src/actions/action_loader.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/actions/action_loader.py b/src/actions/action_loader.py index 0817032e..29ad9b0e 100644 --- a/src/actions/action_loader.py +++ b/src/actions/action_loader.py @@ -15,9 +15,9 @@ class ActionLoader(object): _instance = None def __init__(self): - if self._actions is None: + if not hasattr(self, "actions"): logger.debug("Actions have not been loaded. Loading actions...") - self._actions = load_actions(settings.MOCK_SIGAN, settings.RUNNING_TESTS, settings.DRIVERS_DIR, + self.actions = load_actions(settings.MOCK_SIGAN, settings.RUNNING_TESTS, settings.DRIVERS_DIR, settings.ACTIONS_DIR) else: logger.debug("Already loaded actions. ") @@ -29,15 +29,6 @@ def __new__(cls): logger.debug(f"Calling load_actions with {settings.MOCK_SIGAN}, {settings.RUNNING_TESTS}, {settings.DRIVERS_DIR}, {settings.ACTIONS_DIR}") return cls._instance - @property - def actions(self): - return self._actions - - @actions.setter - def actions(self, value): - self._actions = value - - def copy_driver_files(driver_dir): """Copy driver files where they need to go""" logger.debug(f"Copying driver files in {driver_dir}") From 566eb085d40b56813ae0e01cf24af5211c2a3ffc Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 16:56:11 -0700 Subject: [PATCH 150/255] use ActionLoader in schedule/serializers.py --- src/schedule/serializers.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/schedule/serializers.py b/src/schedule/serializers.py index 31507e91..4b03f8e2 100644 --- a/src/schedule/serializers.py +++ b/src/schedule/serializers.py @@ -6,26 +6,18 @@ convert_datetime_to_millisecond_iso_format, parse_datetime_iso_format_str, ) - +from actions import action_loader from sensor import V1 from sensor.utils import get_datetime_from_timestamp, get_timestamp_from_datetime from . import get_action_with_summary from .models import DEFAULT_PRIORITY, ScheduleEntry -actions = {} - -def action_registered(sender, **kwargs): - name = kwargs["name"] - action = kwargs["action"] - logger.debug(f"Adding {name}: {action} to capabilities") - actions_by_name[name] = actions - action_help = "[Required] The name of the action to be scheduled" priority_help = f"Lower number is higher priority (default={DEFAULT_PRIORITY})" CHOICES = [] -actions = sorted(actions.keys()) +actions = sorted(action_loader.keys()) for action in actions: CHOICES.append((action, get_action_with_summary(action))) From 936589bfa2177e7c01ced27ed9a7aca24103ab02 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 20:00:56 -0700 Subject: [PATCH 151/255] Make initiliazation an app and make it the first to load. Move ActionLoader and CapabilitiesLoader into initialization app. --- scripts/print_action_docstring.py | 2 +- src/actions/__init__.py | 17 ------ src/actions/apps.py | 5 -- src/capabilities/__init__.py | 38 ++---------- src/initialization/__init__.py | 19 ++++++ .../action_loader.py | 0 src/initialization/apps.py | 5 ++ src/initialization/capabilities_loader.py | 60 +++++++++++++++++++ src/schedule/__init__.py | 2 +- src/schedule/models/schedule_entry.py | 2 +- src/schedule/serializers.py | 4 +- src/scheduler/tests/utils.py | 2 +- src/sensor/action_loader.py | 11 ---- src/sensor/settings.py | 2 +- src/tasks/models/task.py | 2 +- 15 files changed, 98 insertions(+), 73 deletions(-) delete mode 100644 src/actions/__init__.py delete mode 100644 src/actions/apps.py rename src/{actions => initialization}/action_loader.py (100%) create mode 100644 src/initialization/apps.py create mode 100644 src/initialization/capabilities_loader.py delete mode 100644 src/sensor/action_loader.py diff --git a/scripts/print_action_docstring.py b/scripts/print_action_docstring.py index 5caef350..d6ef35ad 100755 --- a/scripts/print_action_docstring.py +++ b/scripts/print_action_docstring.py @@ -6,7 +6,7 @@ import django -from actions import action_loader +from initialization import action_loader PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "src") diff --git a/src/actions/__init__.py b/src/actions/__init__.py deleted file mode 100644 index 5815d5f4..00000000 --- a/src/actions/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -import importlib -import json -import logging -import os - -from django.conf import settings - -from scos_actions.actions import action_classes -from scos_actions.discover import test_actions -from scos_actions.discover import init - -from .action_loader import ActionLoader - -logger = logging.getLogger(__name__) - -action_loader = ActionLoader() -logger.debug(f"Actions ActionLoader has {len(action_loader.actions)} actions") diff --git a/src/actions/apps.py b/src/actions/apps.py deleted file mode 100644 index 8eadffac..00000000 --- a/src/actions/apps.py +++ /dev/null @@ -1,5 +0,0 @@ -from django.apps import AppConfig - - -class ActionsConfig(AppConfig): - name = "actions" \ No newline at end of file diff --git a/src/capabilities/__init__.py b/src/capabilities/__init__.py index 68666f6a..3f476db6 100644 --- a/src/capabilities/__init__.py +++ b/src/capabilities/__init__.py @@ -2,46 +2,20 @@ import json import logging -from actions import action_loader +from initialization import action_loader from django.conf import settings -from scos_actions.utils import load_from_json +from initialization import action_loader +from initialization import capabilities_loader + logger = logging.getLogger(__name__) logger.debug("********** Initializing capabilities **********") actions_by_name = action_loader.actions logger.debug(f"ActionLoader has {len(action_loader.actions)} actions") -def load_capabilities(sensor_definition_file): - - capabilities = {} - sensor_definition_hash = None - sensor_location = None - - logger.debug(f"Loading {sensor_definition_file}") - try: - capabilities["sensor"] = load_from_json(sensor_definition_file) - except Exception as e: - logger.warning( - f"Failed to load sensor definition file: {sensor_definition_file}" - + "\nAn empty sensor definition will be used" - ) - capabilities["sensor"] = {"sensor_spec": {"id": "unknown"}} - capabilities["sensor"]["sensor_sha512"] = "UNKNOWN SENSOR DEFINITION" - - # Generate sensor definition file hash (SHA 512) - try: - if "sensor_sha512" not in capabilities["sensor"]: - sensor_def = json.dumps(capabilities["sensor"], sort_keys=True) - sensor_definition_hash = hashlib.sha512(sensor_def.encode("UTF-8")).hexdigest() - capabilities["sensor"]["sensor_sha512"] = sensor_definition_hash - except: - capabilities["sensor"]["sensor_sha512"] = "ERROR GENERATING HASH" - # sensor_sha512 is None, do not raise Exception, but log it - logger.exception(f"Unable to generate sensor definition hash") - - return capabilities + logger.debug("Capabilites connected to register_action") -sensor_capabilities = load_capabilities(settings.SENSOR_DEFINITION_FILE) +sensor_capabilities = capabilities_loader.capabilities diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 1d6deb05..a58af4f2 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -1,6 +1,8 @@ import importlib +import json import logging +import os from os import path from pathlib import Path from its_preselector.configuration_exception import ConfigurationException @@ -11,8 +13,25 @@ from scos_actions.calibration.calibration import Calibration, load_from_json +from .action_loader import ActionLoader +from .capabilities_loader import CapabilitiesLoader + +logger = logging.getLogger(__name__) + +from django.conf import settings + +from scos_actions.actions import action_classes +from scos_actions.discover import test_actions +from scos_actions.discover import init + + + logger = logging.getLogger(__name__) +action_loader = ActionLoader() +logger.debug(f"Actions ActionLoader has {len(action_loader.actions)} actions") +capabilities_loader = CapabilitiesLoader() + def load_switches(switch_dir: Path) -> dict: switch_dict = {} diff --git a/src/actions/action_loader.py b/src/initialization/action_loader.py similarity index 100% rename from src/actions/action_loader.py rename to src/initialization/action_loader.py diff --git a/src/initialization/apps.py b/src/initialization/apps.py new file mode 100644 index 00000000..46e8f1e4 --- /dev/null +++ b/src/initialization/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class InitializationConfig(AppConfig): + name = "initialization" diff --git a/src/initialization/capabilities_loader.py b/src/initialization/capabilities_loader.py new file mode 100644 index 00000000..7701ec8a --- /dev/null +++ b/src/initialization/capabilities_loader.py @@ -0,0 +1,60 @@ +import hashlib +import json +import logging +import os +import pkgutil +import shutil +from django.conf import settings +from scos_actions.actions import action_classes +from scos_actions.discover import test_actions +from scos_actions.discover import init + +from scos_actions.utils import load_from_json + +logger = logging.getLogger(__name__) + +class CapabilitiesLoader(object): + _instance = None + + def __init__(self): + if not hasattr(self, "actions"): + logger.debug("Capabilities have not been loaded. Loading...") + self.capabilities = load_capabilities() + else: + logger.debug("Already loaded capabilities. ") + + def __new__(cls): + if cls._instance is None: + logger.debug('Creating the ActionLoader') + cls._instance = super(CapabilitiesLoader, cls).__new__(cls) + return cls._instance + +def load_capabilities(sensor_definition_file): + + capabilities = {} + sensor_definition_hash = None + sensor_location = None + + logger.debug(f"Loading {sensor_definition_file}") + try: + capabilities["sensor"] = load_from_json(sensor_definition_file) + except Exception as e: + logger.warning( + f"Failed to load sensor definition file: {sensor_definition_file}" + + "\nAn empty sensor definition will be used" + ) + capabilities["sensor"] = {"sensor_spec": {"id": "unknown"}} + capabilities["sensor"]["sensor_sha512"] = "UNKNOWN SENSOR DEFINITION" + + # Generate sensor definition file hash (SHA 512) + try: + if "sensor_sha512" not in capabilities["sensor"]: + sensor_def = json.dumps(capabilities["sensor"], sort_keys=True) + sensor_definition_hash = hashlib.sha512(sensor_def.encode("UTF-8")).hexdigest() + capabilities["sensor"]["sensor_sha512"] = sensor_definition_hash + except: + capabilities["sensor"]["sensor_sha512"] = "ERROR GENERATING HASH" + # sensor_sha512 is None, do not raise Exception, but log it + logger.exception(f"Unable to generate sensor definition hash") + + return capabilities diff --git a/src/schedule/__init__.py b/src/schedule/__init__.py index e227affe..cf61ad0b 100644 --- a/src/schedule/__init__.py +++ b/src/schedule/__init__.py @@ -3,7 +3,7 @@ from utils import get_summary from django.conf import settings -from actions import action_loader +from initialization import action_loader def get_action_with_summary(action): """Given an action, return the string 'action_name - summary'.""" diff --git a/src/schedule/models/schedule_entry.py b/src/schedule/models/schedule_entry.py index e5d4d1c2..dfba1c1a 100644 --- a/src/schedule/models/schedule_entry.py +++ b/src/schedule/models/schedule_entry.py @@ -6,7 +6,7 @@ from django.db import models from constants import MAX_ACTION_LENGTH -from actions import action_loader +from initialization import action_loader from scheduler import utils diff --git a/src/schedule/serializers.py b/src/schedule/serializers.py index 4b03f8e2..03b13a59 100644 --- a/src/schedule/serializers.py +++ b/src/schedule/serializers.py @@ -6,7 +6,7 @@ convert_datetime_to_millisecond_iso_format, parse_datetime_iso_format_str, ) -from actions import action_loader +from initialization import action_loader from sensor import V1 from sensor.utils import get_datetime_from_timestamp, get_timestamp_from_datetime @@ -17,7 +17,7 @@ action_help = "[Required] The name of the action to be scheduled" priority_help = f"Lower number is higher priority (default={DEFAULT_PRIORITY})" CHOICES = [] -actions = sorted(action_loader.keys()) +actions = sorted(action_loader.actions.keys()) for action in actions: CHOICES.append((action, get_action_with_summary(action))) diff --git a/src/scheduler/tests/utils.py b/src/scheduler/tests/utils.py index 28768834..8ef3d810 100644 --- a/src/scheduler/tests/utils.py +++ b/src/scheduler/tests/utils.py @@ -4,7 +4,7 @@ import time from itertools import chain, count, islice -from actions import action_loader +from initialization import action_loader from authentication.models import User from schedule.models import Request, ScheduleEntry from scheduler.scheduler import Scheduler diff --git a/src/sensor/action_loader.py b/src/sensor/action_loader.py deleted file mode 100644 index db26b9b2..00000000 --- a/src/sensor/action_loader.py +++ /dev/null @@ -1,11 +0,0 @@ -import importlib -import json -import logging -import pkgutil -import os -import shutil -from scos_actions.actions import action_classes -from scos_actions.discover import test_actions -from scos_actions.discover import init - -logger = logging.getLogger(__name__) diff --git a/src/sensor/settings.py b/src/sensor/settings.py index e79493a5..5e78bd82 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -197,7 +197,7 @@ "rest_framework.authtoken", "drf_yasg", # OpenAPI generator # project-local apps - "actions.apps.ActionsConfig", + "actions.apps.InitializationConfig", "capabilities.apps.CapabilitiesConfig", "authentication.apps.AuthenticationConfig", "handlers.apps.HandlersConfig", diff --git a/src/tasks/models/task.py b/src/tasks/models/task.py index 775ec37d..8e148073 100644 --- a/src/tasks/models/task.py +++ b/src/tasks/models/task.py @@ -7,7 +7,7 @@ import logging from collections import namedtuple -from actions import action_loader +from initialization import action_loader logger = logging.getLogger(__name__) From e9bcbfda7b5a8799f63c4a09818d1be994cd7276 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 20:05:17 -0700 Subject: [PATCH 152/255] make initialization app load first. --- src/sensor/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 5e78bd82..126f2921 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -197,7 +197,7 @@ "rest_framework.authtoken", "drf_yasg", # OpenAPI generator # project-local apps - "actions.apps.InitializationConfig", + "initialization.apps.InitializationConfig", "capabilities.apps.CapabilitiesConfig", "authentication.apps.AuthenticationConfig", "handlers.apps.HandlersConfig", From 7c8d14c649bd79a92b3b2575a39beaf46a62bc99 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 20:07:48 -0700 Subject: [PATCH 153/255] pass SENSOR_DEFINITION_FILE setting to load_capabilities --- src/initialization/capabilities_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/initialization/capabilities_loader.py b/src/initialization/capabilities_loader.py index 7701ec8a..3911f63f 100644 --- a/src/initialization/capabilities_loader.py +++ b/src/initialization/capabilities_loader.py @@ -19,7 +19,7 @@ class CapabilitiesLoader(object): def __init__(self): if not hasattr(self, "actions"): logger.debug("Capabilities have not been loaded. Loading...") - self.capabilities = load_capabilities() + self.capabilities = load_capabilities(settings.SENSOR_DEFINITION_FILE) else: logger.debug("Already loaded capabilities. ") From df83fc3f6351051148ab8ff0dff1fb9087ce0265 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 16 Jan 2024 20:46:13 -0700 Subject: [PATCH 154/255] Move sensor instantiation into scheduler. --- docs/openapi.json | 11 ++- gunicorn/config.py | 124 ++++++++++++++-------------- src/capabilities/__init__.py | 4 - src/conftest.py | 6 +- src/initialization/action_loader.py | 4 +- src/scheduler/scheduler.py | 70 ++++++++++++++-- src/scheduler/tests/utils.py | 1 - 7 files changed, 142 insertions(+), 78 deletions(-) diff --git a/docs/openapi.json b/docs/openapi.json index bfab7a52..f7dacc8c 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -1636,7 +1636,16 @@ "title": "Action", "description": "[Required] The name of the action to be scheduled", "type": "string", - "enum": [] + "enum": [ + "logger", + "test_monitor_sigan", + "test_multi_frequency_iq_action", + "test_nasctn_sea_data_product", + "test_single_frequency_iq_action", + "test_single_frequency_m4s_action", + "test_survey_iq_action", + "test_sync_gps" + ] }, "priority": { "title": "Priority", diff --git a/gunicorn/config.py b/gunicorn/config.py index c0650f66..38dbab56 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -31,68 +31,68 @@ def post_worker_init(worker): import django django.setup() - from capabilities import sensor_capabilities - from scheduler import scheduler - from initialization import ( - load_preselector, - load_switches, - ) - from initialization import ( - get_sensor_calibration, - get_sigan_calibration - ) - from django.conf import settings - from status.models import Location - from scos_actions.hardware.sensor import Sensor - from scos_actions.metadata.utils import construct_geojson_point - from scos_actions.signals import register_component_with_status - from scos_actions.signals import register_signal_analyzer - from scos_actions.signals import register_sensor - - sigan_module_setting = settings.SIGAN_MODULE - sigan_module = importlib.import_module(sigan_module_setting) - logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) - sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) - sensor_cal = get_sensor_calibration(settings.SENSOR_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) - sigan_cal = get_sigan_calibration(settings.SIGAN_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) - sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal) - register_component_with_status.send(sigan, component=sigan) - register_signal_analyzer.send(sigan, signal_analyzer=sigan) - - switches = load_switches(settings.SWITCH_CONFIGS_DIR) - capabilities = sensor_capabilities - preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELECTOR_MODULE, settings.PRESELECTOR_CLASS, capabilities["sensor"]) - location = None - if "location" in capabilities["sensor"]: - try: - sensor_loc = capabilities["sensor"].pop("location") - try: - #if there is an active database location, use it over the value in the sensor def. - db_location = Location.objects.get(active=True) - location = construct_geojson_point(db_location.longitude, db_location.latitude, db_location.height) - except Location.DoesNotExist: - # This should never occur because status/migrations/0003_auto_20211217_2229.py - # will load the No DB location. Use sensor def location and save to DB. - location = construct_geojson_point( - sensor_loc["x"], - sensor_loc["y"], - sensor_loc["z"] if "z" in sensor_loc else None, - ) - #Save the sensor location from the sensor def to the database - db_location = Location() - db_location.longitude = sensor_loc["x"] - db_location.latitude = sensor_loc["y"] - db_location.height = sensor_loc["z"] - db_location.gps = False - db_location.description = sensor_loc["description"] - db_location.save() - except: - logger.exception("Failed to get sensor location from sensor definition.") - - - sensor = Sensor(signal_analyzer=sigan, preselector = preselector, switches = switches, capabilities = capabilities, location = location) - scheduler.thread.sensor = sensor - register_sensor.send(sensor, sensor=sensor) + # from capabilities import sensor_capabilities + # from scheduler import scheduler + # from initialization import ( + # load_preselector, + # load_switches, + # ) + # from initialization import ( + # get_sensor_calibration, + # get_sigan_calibration + # ) + # from django.conf import settings + # from status.models import Location + # from scos_actions.hardware.sensor import Sensor + # from scos_actions.metadata.utils import construct_geojson_point + # from scos_actions.signals import register_component_with_status + # from scos_actions.signals import register_signal_analyzer + # from scos_actions.signals import register_sensor + # + # sigan_module_setting = settings.SIGAN_MODULE + # sigan_module = importlib.import_module(sigan_module_setting) + # logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) + # sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) + # sensor_cal = get_sensor_calibration(settings.SENSOR_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) + # sigan_cal = get_sigan_calibration(settings.SIGAN_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) + # sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal) + # register_component_with_status.send(sigan, component=sigan) + # register_signal_analyzer.send(sigan, signal_analyzer=sigan) + # + # switches = load_switches(settings.SWITCH_CONFIGS_DIR) + # capabilities = sensor_capabilities + # preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELECTOR_MODULE, settings.PRESELECTOR_CLASS, capabilities["sensor"]) + # location = None + # if "location" in capabilities["sensor"]: + # try: + # sensor_loc = capabilities["sensor"].pop("location") + # try: + # #if there is an active database location, use it over the value in the sensor def. + # db_location = Location.objects.get(active=True) + # location = construct_geojson_point(db_location.longitude, db_location.latitude, db_location.height) + # except Location.DoesNotExist: + # # This should never occur because status/migrations/0003_auto_20211217_2229.py + # # will load the No DB location. Use sensor def location and save to DB. + # location = construct_geojson_point( + # sensor_loc["x"], + # sensor_loc["y"], + # sensor_loc["z"] if "z" in sensor_loc else None, + # ) + # #Save the sensor location from the sensor def to the database + # db_location = Location() + # db_location.longitude = sensor_loc["x"] + # db_location.latitude = sensor_loc["y"] + # db_location.height = sensor_loc["z"] + # db_location.gps = False + # db_location.description = sensor_loc["description"] + # db_location.save() + # except: + # logger.exception("Failed to get sensor location from sensor definition.") + # + # + # sensor = Sensor(signal_analyzer=sigan, preselector = preselector, switches = switches, capabilities = capabilities, location = location) + # scheduler.thread.sensor = sensor + # register_sensor.send(sensor, sensor=sensor) scheduler.thread.start() diff --git a/src/capabilities/__init__.py b/src/capabilities/__init__.py index 3f476db6..c4f1e809 100644 --- a/src/capabilities/__init__.py +++ b/src/capabilities/__init__.py @@ -14,8 +14,4 @@ logger.debug("********** Initializing capabilities **********") actions_by_name = action_loader.actions logger.debug(f"ActionLoader has {len(action_loader.actions)} actions") - - -logger.debug("Capabilites connected to register_action") - sensor_capabilities = capabilities_loader.capabilities diff --git a/src/conftest.py b/src/conftest.py index dc9f6995..27970776 100644 --- a/src/conftest.py +++ b/src/conftest.py @@ -44,9 +44,9 @@ def testclock(): def test_scheduler(rf, testclock): """Instantiate test scheduler with fake request context and testclock.""" s = scheduler.scheduler.Scheduler() - mock_sigan = MockSignalAnalyzer() - sensor = Sensor(signal_analyzer=mock_sigan) - s.sensor = sensor + # mock_sigan = MockSignalAnalyzer() + # sensor = Sensor(signal_analyzer=mock_sigan) + # s.sensor = sensor s.request = rf.post("mock://cburl/schedule") return s diff --git a/src/initialization/action_loader.py b/src/initialization/action_loader.py index 29ad9b0e..74423beb 100644 --- a/src/initialization/action_loader.py +++ b/src/initialization/action_loader.py @@ -73,8 +73,8 @@ def load_actions(mock_sigan, running_tests, driver_dir, action_dir): actions = {} if mock_sigan or running_tests: - for name, action in test_actions.items(): - logger.debug("test_action: " + name + "=" + str(action)) + logger.debug(f"Loading {len(test_actions)} test actions.") + actions.update(test_actions) else: for name, module in discovered_plugins.items(): logger.debug("Looking for actions in " + name + ": " + str(module)) diff --git a/src/scheduler/scheduler.py b/src/scheduler/scheduler.py index 0f5a3d1c..6bed9b38 100644 --- a/src/scheduler/scheduler.py +++ b/src/scheduler/scheduler.py @@ -1,22 +1,38 @@ """Queue and run tasks.""" - +import importlib import json import logging import threading +import requests + from contextlib import contextmanager from pathlib import Path -import requests + from django.utils import timezone -from scos_actions.hardware.sensor import Sensor from scos_actions.signals import trigger_api_restart from authentication import oauth from schedule.models import ScheduleEntry -from sensor import settings from tasks.consts import MAX_DETAIL_LEN from tasks.models import TaskResult from tasks.serializers import TaskResultSerializer from tasks.task_queue import TaskQueue +from capabilities import sensor_capabilities +from initialization import ( + load_preselector, + load_switches, +) +from initialization import ( + get_sensor_calibration, + get_sigan_calibration +) +from django.conf import settings +from status.models import Location +from scos_actions.hardware.sensor import Sensor +from scos_actions.metadata.utils import construct_geojson_point +from scos_actions.signals import register_component_with_status +from scos_actions.signals import register_signal_analyzer +from scos_actions.signals import register_sensor from . import utils @@ -32,7 +48,6 @@ def __init__(self): self.task_status_lock = threading.Lock() self.timefn = utils.timefn self.delayfn = utils.delayfn - self.task_queue = TaskQueue() # scheduler looks ahead `interval_multiplier` times the shortest @@ -49,6 +64,51 @@ def __init__(self): self.consecutive_failures = 0 self._sensor = None + + sigan_module_setting = settings.SIGAN_MODULE + sigan_module = importlib.import_module(sigan_module_setting) + logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) + sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) + sensor_cal = get_sensor_calibration(settings.SENSOR_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) + sigan_cal = get_sigan_calibration(settings.SIGAN_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) + sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal) + register_component_with_status.send(sigan, component=sigan) + register_signal_analyzer.send(sigan, signal_analyzer=sigan) + + switches = load_switches(settings.SWITCH_CONFIGS_DIR) + capabilities = sensor_capabilities + preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELECTOR_MODULE, + settings.PRESELECTOR_CLASS, capabilities["sensor"]) + location = None + self.sensor = Sensor(signal_analyzer=sigan, preselector=preselector, switches=switches, capabilities=capabilities, + location=location) + register_sensor.send(sender=__name__, sensor = self.sensor) + if "location" in capabilities["sensor"]: + try: + sensor_loc = capabilities["sensor"].pop("location") + try: + # if there is an active database location, use it over the value in the sensor def. + db_location = Location.objects.get(active=True) + location = construct_geojson_point(db_location.longitude, db_location.latitude, db_location.height) + except Location.DoesNotExist: + # This should never occur because status/migrations/0003_auto_20211217_2229.py + # will load the No DB location. Use sensor def location and save to DB. + location = construct_geojson_point( + sensor_loc["x"], + sensor_loc["y"], + sensor_loc["z"] if "z" in sensor_loc else None, + ) + # Save the sensor location from the sensor def to the database + db_location = Location() + db_location.longitude = sensor_loc["x"] + db_location.latitude = sensor_loc["y"] + db_location.height = sensor_loc["z"] + db_location.gps = False + db_location.description = sensor_loc["description"] + db_location.save() + except: + logger.exception("Failed to get sensor location from sensor definition.") + @property def sensor(self): return self._sensor diff --git a/src/scheduler/tests/utils.py b/src/scheduler/tests/utils.py index 8ef3d810..912dbdea 100644 --- a/src/scheduler/tests/utils.py +++ b/src/scheduler/tests/utils.py @@ -66,7 +66,6 @@ def advance_testclock(iterator, n): def simulate_scheduler_run(n=1): s = Scheduler() - s.sensor = Sensor(signal_analyzer=MockSignalAnalyzer()) for _ in range(n): advance_testclock(s.timefn, 1) s.run(blocking=False) From 63e8f43f82ff8707db1d3cf2bc158f2693f68fb4 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Wed, 17 Jan 2024 07:30:29 -0700 Subject: [PATCH 155/255] Fix tests. --- src/initialization/tests/test_initialization.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/initialization/tests/test_initialization.py b/src/initialization/tests/test_initialization.py index d74b2f41..3ae263c2 100644 --- a/src/initialization/tests/test_initialization.py +++ b/src/initialization/tests/test_initialization.py @@ -1,9 +1,15 @@ from src.initialization import load_preselector +import logging +import os +logger = logging.getLogger(__name__) def test_load_preselector(): - preselector = load_preselector(preselector_config= - {"name": "test", "base_url": "http://127.0.0.1"}, + preselector_config = os.getcwd() + index = preselector_config.index("src") + preselector_config = os.path.join( preselector_config[:index], "configs/preselector_config.json") + logger.debug("Loading preselector config: " + preselector_config) + preselector = load_preselector(preselector_config=preselector_config, module="its_preselector.web_relay_preselector", preselector_class_name = "WebRelayPreselector", sensor_definition={} ) From 78df6a44fc79ae3bfbe5ca202afe39cb8d0e085f Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Wed, 17 Jan 2024 08:26:35 -0700 Subject: [PATCH 156/255] Consolidate initialization into initialization app. --- src/initialization/__init__.py | 157 ++------------- src/initialization/capabilities_loader.py | 1 + src/initialization/sensor_loader.py | 185 ++++++++++++++++++ src/initialization/status_monitor.py | 25 +++ .../tests/test_initialization.py | 2 +- src/scheduler/scheduler.py | 65 +----- src/status/__init__.py | 25 +-- src/status/views.py | 9 +- 8 files changed, 234 insertions(+), 235 deletions(-) create mode 100644 src/initialization/sensor_loader.py create mode 100644 src/initialization/status_monitor.py diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index a58af4f2..b535fa97 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -1,158 +1,27 @@ - -import importlib -import json import logging -import os -from os import path -from pathlib import Path -from its_preselector.configuration_exception import ConfigurationException -from its_preselector.controlbyweb_web_relay import ControlByWebWebRelay - -from scos_actions import utils -from scos_actions.signals import register_component_with_status - -from scos_actions.calibration.calibration import Calibration, load_from_json from .action_loader import ActionLoader from .capabilities_loader import CapabilitiesLoader +from .sensor_loader import SensorLoader +from .status_monitor import StatusMonitor -logger = logging.getLogger(__name__) - -from django.conf import settings +from scos_actions.signals import register_component_with_status -from scos_actions.actions import action_classes -from scos_actions.discover import test_actions -from scos_actions.discover import init +logger = logging.getLogger(__name__) +status_monitor = StatusMonitor() +def status_registration_handler(sender, **kwargs): + try: + logger.debug(f"Registering {sender} as status provider") + status_monitor.add_component(kwargs["component"]) + except: + logger.exception("Error registering status component") -logger = logging.getLogger(__name__) +register_component_with_status.connect(status_registration_handler) action_loader = ActionLoader() logger.debug(f"Actions ActionLoader has {len(action_loader.actions)} actions") capabilities_loader = CapabilitiesLoader() - - -def load_switches(switch_dir: Path) -> dict: - switch_dict = {} - if switch_dir is not None and switch_dir.is_dir(): - for f in switch_dir.iterdir(): - file_path = f.resolve() - logger.debug(f"loading switch config {file_path}") - conf = utils.load_from_json(file_path) - try: - switch = ControlByWebWebRelay(conf) - logger.debug(f"Adding {switch.id}") - - switch_dict[switch.id] = switch - logger.debug(f"Registering switch status for {switch.name}") - register_component_with_status.send(__name__, component=switch) - except ConfigurationException: - logger.error(f"Unable to configure switch defined in: {file_path}") - - return switch_dict - - -def load_preselector_from_file(preselector_module, preselector_class, preselector_config_file: Path): - if preselector_config_file is None: - return None - else: - try: - preselector_config = utils.load_from_json(preselector_config_file) - return load_preselector( - preselector_config, preselector_module, preselector_class - ) - except ConfigurationException: - logger.exception( - f"Unable to create preselector defined in: {preselector_config_file}" - ) - return None - - -def load_preselector(preselector_config: str, module: str, preselector_class_name: str, sensor_definition: dict): - logger.debug(f"loading {preselector_class_name} from {module} with config: {preselector_config}") - if module is not None and preselector_class_name is not None: - preselector_module = importlib.import_module(module) - preselector_constructor = getattr(preselector_module, preselector_class_name) - preselector_config = utils.load_from_json(preselector_config) - ps = preselector_constructor(sensor_definition, preselector_config) - register_component_with_status.send(ps, component=ps) - else: - ps = None - return ps - - - - - -def get_sigan_calibration(sigan_cal_file_path: str, default_cal_file_path: str) -> Calibration: - """ - Load signal analyzer calibration data from file. - - :param sigan_cal_file_path: Path to JSON file containing signal - analyzer calibration data. - :param default_cal_file_path: Path to the default cal file. - :return: The signal analyzer ``Calibration`` object. - """ - try: - sigan_cal = None - if sigan_cal_file_path is None or sigan_cal_file_path == "": - logger.warning("No sigan calibration file specified. Not loading calibration file.") - elif not path.exists(sigan_cal_file_path): - logger.warning( - sigan_cal_file_path + " does not exist. Not loading sigan calibration file." - ) - else: - logger.debug(f"Loading sigan cal file: {sigan_cal_file_path}") - default = check_for_default_calibration(sigan_cal_file_path,default_cal_file_path, "Sigan") - sigan_cal = load_from_json(sigan_cal_file_path, default) - sigan_cal.is_default = default - except Exception: - sigan_cal = None - logger.exception("Unable to load sigan calibration data, reverting to none") - return sigan_cal - - -def get_sensor_calibration(sensor_cal_file_path: str, default_cal_file_path: str) -> Calibration: - """ - Load sensor calibration data from file. - - :param sensor_cal_file_path: Path to JSON file containing sensor - calibration data. - :param default_cal_file_path: Name of the default calibration file. - :return: The sensor ``Calibration`` object. - """ - try: - sensor_cal = None - if sensor_cal_file_path is None or sensor_cal_file_path == "": - logger.warning( - "No sensor calibration file specified. Not loading calibration file." - ) - elif not path.exists(sensor_cal_file_path): - logger.warning( - sensor_cal_file_path - + " does not exist. Not loading sensor calibration file." - ) - else: - logger.debug(f"Loading sensor cal file: {sensor_cal_file_path}") - default = check_for_default_calibration( - sensor_cal_file_path, default_cal_file_path, "Sensor" - ) - sensor_cal = load_from_json(sensor_cal_file_path, default) - sensor_cal.is_default = default - except Exception: - sensor_cal = None - logger.exception("Unable to load sensor calibration data, reverting to none") - return sensor_cal - - -def check_for_default_calibration(cal_file_path: str,default_cal_path: str, cal_type: str) -> bool: - default_cal = False - if cal_file_path == default_cal_path: - default_cal = True - logger.warning( - f"***************LOADING DEFAULT {cal_type} CALIBRATION***************" - ) - return default_cal - +sensor_loader = SensorLoader(capabilities_loader.capabilities) diff --git a/src/initialization/capabilities_loader.py b/src/initialization/capabilities_loader.py index 3911f63f..10a16e72 100644 --- a/src/initialization/capabilities_loader.py +++ b/src/initialization/capabilities_loader.py @@ -57,4 +57,5 @@ def load_capabilities(sensor_definition_file): # sensor_sha512 is None, do not raise Exception, but log it logger.exception(f"Unable to generate sensor definition hash") + return capabilities diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py new file mode 100644 index 00000000..986921bb --- /dev/null +++ b/src/initialization/sensor_loader.py @@ -0,0 +1,185 @@ +import importlib +import logging +from django.conf import settings +from scos_actions.hardware.sensor import Sensor +from scos_actions.metadata.utils import construct_geojson_point +from scos_actions.signals import register_component_with_status +from scos_actions.signals import register_signal_analyzer +from os import path +from pathlib import Path +from its_preselector.configuration_exception import ConfigurationException +from its_preselector.controlbyweb_web_relay import ControlByWebWebRelay + +from scos_actions import utils +from scos_actions.signals import register_component_with_status + +from scos_actions.calibration.calibration import Calibration, load_from_json + + +logger = logging.getLogger(__name__) + +class SensorLoader(object): + _instance = None + + def __init__(self, sensor_capabilities): + if not hasattr(self, "actions"): + logger.debug("Sensor has not been loaded. Loading...") + self.sensor = load_sensor(sensor_capabilities) + else: + logger.debug("Already loaded sensor. ") + + def __new__(cls, sensor_capabilities): + if cls._instance is None: + logger.debug('Creating the SensorLoader') + cls._instance = super(SensorLoader, cls).__new__(cls) + return cls._instance + +def load_sensor(sensor_capabilities): + location = None + #Remove location from sensor definition and convert to geojson. + #Db may have an updated location, but status module will update it + #if needed. + if "location" in sensor_capabilities["sensor"]: + sensor_loc = sensor_capabilities["sensor"].pop("location") + location = construct_geojson_point( + sensor_loc["x"], + sensor_loc["y"], + sensor_loc["z"] if "z" in sensor_loc else None, + ) + sigan_module_setting = settings.SIGAN_MODULE + sigan_module = importlib.import_module(sigan_module_setting) + logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) + sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) + sensor_cal = get_sensor_calibration(settings.SENSOR_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) + sigan_cal = get_sigan_calibration(settings.SIGAN_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) + sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal) + register_component_with_status.send(sigan, component=sigan) + register_signal_analyzer.send(sigan, signal_analyzer=sigan) + switches = load_switches(settings.SWITCH_CONFIGS_DIR) + preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELECTOR_MODULE, + settings.PRESELECTOR_CLASS, sensor_capabilities["sensor"]) + sensor = Sensor(signal_analyzer=sigan, preselector=preselector, switches=switches, capabilities=sensor_capabilities, + location=location) + return sensor + +def load_switches(switch_dir: Path) -> dict: + switch_dict = {} + if switch_dir is not None and switch_dir.is_dir(): + for f in switch_dir.iterdir(): + file_path = f.resolve() + logger.debug(f"loading switch config {file_path}") + conf = utils.load_from_json(file_path) + try: + switch = ControlByWebWebRelay(conf) + logger.debug(f"Adding {switch.id}") + + switch_dict[switch.id] = switch + logger.debug(f"Registering switch status for {switch.name}") + register_component_with_status.send(__name__, component=switch) + except ConfigurationException: + logger.error(f"Unable to configure switch defined in: {file_path}") + + return switch_dict + + +def load_preselector_from_file(preselector_module, preselector_class, preselector_config_file: Path): + if preselector_config_file is None: + return None + else: + try: + preselector_config = utils.load_from_json(preselector_config_file) + return load_preselector( + preselector_config, preselector_module, preselector_class + ) + except ConfigurationException: + logger.exception( + f"Unable to create preselector defined in: {preselector_config_file}" + ) + return None + + +def load_preselector(preselector_config: str, module: str, preselector_class_name: str, sensor_definition: dict): + logger.debug(f"loading {preselector_class_name} from {module} with config: {preselector_config}") + if module is not None and preselector_class_name is not None: + preselector_module = importlib.import_module(module) + preselector_constructor = getattr(preselector_module, preselector_class_name) + preselector_config = utils.load_from_json(preselector_config) + ps = preselector_constructor(sensor_definition, preselector_config) + register_component_with_status.send(ps, component=ps) + else: + ps = None + return ps + + + + + +def get_sigan_calibration(sigan_cal_file_path: str, default_cal_file_path: str) -> Calibration: + """ + Load signal analyzer calibration data from file. + + :param sigan_cal_file_path: Path to JSON file containing signal + analyzer calibration data. + :param default_cal_file_path: Path to the default cal file. + :return: The signal analyzer ``Calibration`` object. + """ + try: + sigan_cal = None + if sigan_cal_file_path is None or sigan_cal_file_path == "": + logger.warning("No sigan calibration file specified. Not loading calibration file.") + elif not path.exists(sigan_cal_file_path): + logger.warning( + sigan_cal_file_path + " does not exist. Not loading sigan calibration file." + ) + else: + logger.debug(f"Loading sigan cal file: {sigan_cal_file_path}") + default = check_for_default_calibration(sigan_cal_file_path,default_cal_file_path, "Sigan") + sigan_cal = load_from_json(sigan_cal_file_path, default) + sigan_cal.is_default = default + except Exception: + sigan_cal = None + logger.exception("Unable to load sigan calibration data, reverting to none") + return sigan_cal + + +def get_sensor_calibration(sensor_cal_file_path: str, default_cal_file_path: str) -> Calibration: + """ + Load sensor calibration data from file. + + :param sensor_cal_file_path: Path to JSON file containing sensor + calibration data. + :param default_cal_file_path: Name of the default calibration file. + :return: The sensor ``Calibration`` object. + """ + try: + sensor_cal = None + if sensor_cal_file_path is None or sensor_cal_file_path == "": + logger.warning( + "No sensor calibration file specified. Not loading calibration file." + ) + elif not path.exists(sensor_cal_file_path): + logger.warning( + sensor_cal_file_path + + " does not exist. Not loading sensor calibration file." + ) + else: + logger.debug(f"Loading sensor cal file: {sensor_cal_file_path}") + default = check_for_default_calibration( + sensor_cal_file_path, default_cal_file_path, "Sensor" + ) + sensor_cal = load_from_json(sensor_cal_file_path, default) + sensor_cal.is_default = default + except Exception: + sensor_cal = None + logger.exception("Unable to load sensor calibration data, reverting to none") + return sensor_cal + + +def check_for_default_calibration(cal_file_path: str,default_cal_path: str, cal_type: str) -> bool: + default_cal = False + if cal_file_path == default_cal_path: + default_cal = True + logger.warning( + f"***************LOADING DEFAULT {cal_type} CALIBRATION***************" + ) + return default_cal \ No newline at end of file diff --git a/src/initialization/status_monitor.py b/src/initialization/status_monitor.py new file mode 100644 index 00000000..611d62d9 --- /dev/null +++ b/src/initialization/status_monitor.py @@ -0,0 +1,25 @@ +import logging + +logger = logging.getLogger(__name__) + +class StatusMonitor(object): + _instance = None + + def __new__(cls): + if cls._instance is None: + logger.debug('Creating the ActionLoader') + cls._instance = super(StatusMonitor, cls).__new__(cls) + cls._instance.status_components = [] + return cls._instance + + def add_component(self, component): + """ + Allows objects to be registered to provide status. Any object registered will + be included in scos-sensors status endpoint. All objects registered must + implement a get_status() method that returns a dictionary. + + :param component: the object to add to the list of status providing objects. + """ + if hasattr(component, "get_status"): + self.status_components.append(component) + diff --git a/src/initialization/tests/test_initialization.py b/src/initialization/tests/test_initialization.py index 3ae263c2..2c3bb383 100644 --- a/src/initialization/tests/test_initialization.py +++ b/src/initialization/tests/test_initialization.py @@ -1,4 +1,4 @@ -from src.initialization import load_preselector +from initialization.sensor_loader import load_preselector import logging import os diff --git a/src/scheduler/scheduler.py b/src/scheduler/scheduler.py index 6bed9b38..e42c02f6 100644 --- a/src/scheduler/scheduler.py +++ b/src/scheduler/scheduler.py @@ -12,27 +12,15 @@ from django.utils import timezone from scos_actions.signals import trigger_api_restart from authentication import oauth +from initialization import sensor_loader from schedule.models import ScheduleEntry from tasks.consts import MAX_DETAIL_LEN from tasks.models import TaskResult from tasks.serializers import TaskResultSerializer from tasks.task_queue import TaskQueue -from capabilities import sensor_capabilities -from initialization import ( - load_preselector, - load_switches, -) -from initialization import ( - get_sensor_calibration, - get_sigan_calibration -) from django.conf import settings -from status.models import Location from scos_actions.hardware.sensor import Sensor -from scos_actions.metadata.utils import construct_geojson_point -from scos_actions.signals import register_component_with_status -from scos_actions.signals import register_signal_analyzer -from scos_actions.signals import register_sensor + from . import utils @@ -49,65 +37,18 @@ def __init__(self): self.timefn = utils.timefn self.delayfn = utils.delayfn self.task_queue = TaskQueue() - # scheduler looks ahead `interval_multiplier` times the shortest # interval in the schedule in order to keep memory-usage low self.interval_multiplier = 10 self.name = "Scheduler" self.running = False self.interrupt_flag = threading.Event() - # Cache the currently running task state self.entry = None # ScheduleEntry that created the current task self.task = None # Task object describing current task self.last_status = "" self.consecutive_failures = 0 - self._sensor = None - - - sigan_module_setting = settings.SIGAN_MODULE - sigan_module = importlib.import_module(sigan_module_setting) - logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) - sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) - sensor_cal = get_sensor_calibration(settings.SENSOR_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) - sigan_cal = get_sigan_calibration(settings.SIGAN_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) - sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal) - register_component_with_status.send(sigan, component=sigan) - register_signal_analyzer.send(sigan, signal_analyzer=sigan) - - switches = load_switches(settings.SWITCH_CONFIGS_DIR) - capabilities = sensor_capabilities - preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELECTOR_MODULE, - settings.PRESELECTOR_CLASS, capabilities["sensor"]) - location = None - self.sensor = Sensor(signal_analyzer=sigan, preselector=preselector, switches=switches, capabilities=capabilities, - location=location) - register_sensor.send(sender=__name__, sensor = self.sensor) - if "location" in capabilities["sensor"]: - try: - sensor_loc = capabilities["sensor"].pop("location") - try: - # if there is an active database location, use it over the value in the sensor def. - db_location = Location.objects.get(active=True) - location = construct_geojson_point(db_location.longitude, db_location.latitude, db_location.height) - except Location.DoesNotExist: - # This should never occur because status/migrations/0003_auto_20211217_2229.py - # will load the No DB location. Use sensor def location and save to DB. - location = construct_geojson_point( - sensor_loc["x"], - sensor_loc["y"], - sensor_loc["z"] if "z" in sensor_loc else None, - ) - # Save the sensor location from the sensor def to the database - db_location = Location() - db_location.longitude = sensor_loc["x"] - db_location.latitude = sensor_loc["y"] - db_location.height = sensor_loc["z"] - db_location.gps = False - db_location.description = sensor_loc["description"] - db_location.save() - except: - logger.exception("Failed to get sensor location from sensor definition.") + self._sensor = sensor_loader.sensor @property def sensor(self): diff --git a/src/status/__init__.py b/src/status/__init__.py index f55078f5..62a41c79 100644 --- a/src/status/__init__.py +++ b/src/status/__init__.py @@ -1,35 +1,12 @@ import datetime import logging -from scos_actions.signals import register_component_with_status -from scos_actions.signals import register_signal_analyzer -from scos_actions.status.status_monitor import StatusMonitor - - -signal_analyzers = [] logger = logging.getLogger(__name__) logger.debug("********** Initializing status **********") start_time = datetime.datetime.utcnow() -status_monitor = StatusMonitor() -def signal_analyzer_registration_handler(sender, **kwargs): - try: - logger.debug(f"Registering {sender} as signa analyzer") - if len(signal_analyzers) > 0: - signal_analyzers[0] = kwargs["signal_analyzer"] - else: - signal_analyzers.append( kwargs["signal_analyzer"]) - except: - logger.exception("Error registering status component") -def status_registration_handler(sender, **kwargs): - try: - logger.debug(f"Registering {sender} as status provider") - status_monitor.add_component(kwargs["component"]) - except: - logger.exception("Error registering status component") -register_component_with_status.connect(status_registration_handler) -register_signal_analyzer.connect(signal_analyzer_registration_handler) + diff --git a/src/status/views.py b/src/status/views.py index 7b000ebb..72222c44 100644 --- a/src/status/views.py +++ b/src/status/views.py @@ -1,9 +1,6 @@ import datetime import logging import shutil - -from . import status_monitor -from . import signal_analyzers from its_preselector.preselector import Preselector from its_preselector.web_relay import WebRelay from rest_framework.decorators import api_view @@ -14,6 +11,10 @@ get_datetime_str_now, ) +from initialization import ( + status_monitor, + sensor_loader +) from scheduler import scheduler from . import start_time @@ -58,7 +59,7 @@ def status(request, version, format=None): "location": serialize_location(), "system_time": get_datetime_str_now(), "start_time": convert_datetime_to_millisecond_iso_format(start_time), - "last_calibration_datetime": signal_analyzers[0].sensor_calibration.last_calibration_datetime, + "last_calibration_datetime": sensor_loader.sensor.signal_analyzer.sensor_calibration.last_calibration_datetime, "disk_usage": disk_usage(), "days_up": get_days_up(), } From 5375e0f6c797e2f7da343b70a5333b04e0e03a00 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Wed, 17 Jan 2024 08:34:14 -0700 Subject: [PATCH 157/255] Fix gunicorn/config.py post_worker_init --- gunicorn/config.py | 65 +--------------------------------------------- 1 file changed, 1 insertion(+), 64 deletions(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index 38dbab56..2d82f58a 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -27,72 +27,9 @@ def post_worker_init(worker): """Start scheduler in worker.""" _modify_path() os.environ.setdefault("DJANGO_SETTINGS_MODULE", "sensor.settings") - import django - django.setup() - # from capabilities import sensor_capabilities - # from scheduler import scheduler - # from initialization import ( - # load_preselector, - # load_switches, - # ) - # from initialization import ( - # get_sensor_calibration, - # get_sigan_calibration - # ) - # from django.conf import settings - # from status.models import Location - # from scos_actions.hardware.sensor import Sensor - # from scos_actions.metadata.utils import construct_geojson_point - # from scos_actions.signals import register_component_with_status - # from scos_actions.signals import register_signal_analyzer - # from scos_actions.signals import register_sensor - # - # sigan_module_setting = settings.SIGAN_MODULE - # sigan_module = importlib.import_module(sigan_module_setting) - # logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) - # sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) - # sensor_cal = get_sensor_calibration(settings.SENSOR_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) - # sigan_cal = get_sigan_calibration(settings.SIGAN_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) - # sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal) - # register_component_with_status.send(sigan, component=sigan) - # register_signal_analyzer.send(sigan, signal_analyzer=sigan) - # - # switches = load_switches(settings.SWITCH_CONFIGS_DIR) - # capabilities = sensor_capabilities - # preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELECTOR_MODULE, settings.PRESELECTOR_CLASS, capabilities["sensor"]) - # location = None - # if "location" in capabilities["sensor"]: - # try: - # sensor_loc = capabilities["sensor"].pop("location") - # try: - # #if there is an active database location, use it over the value in the sensor def. - # db_location = Location.objects.get(active=True) - # location = construct_geojson_point(db_location.longitude, db_location.latitude, db_location.height) - # except Location.DoesNotExist: - # # This should never occur because status/migrations/0003_auto_20211217_2229.py - # # will load the No DB location. Use sensor def location and save to DB. - # location = construct_geojson_point( - # sensor_loc["x"], - # sensor_loc["y"], - # sensor_loc["z"] if "z" in sensor_loc else None, - # ) - # #Save the sensor location from the sensor def to the database - # db_location = Location() - # db_location.longitude = sensor_loc["x"] - # db_location.latitude = sensor_loc["y"] - # db_location.height = sensor_loc["z"] - # db_location.gps = False - # db_location.description = sensor_loc["description"] - # db_location.save() - # except: - # logger.exception("Failed to get sensor location from sensor definition.") - # - # - # sensor = Sensor(signal_analyzer=sigan, preselector = preselector, switches = switches, capabilities = capabilities, location = location) - # scheduler.thread.sensor = sensor - # register_sensor.send(sensor, sensor=sensor) + from scheduler import scheduler scheduler.thread.start() From bc1a14ea5b01bc99b7ecb6fdb85b811f4f16494d Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Wed, 17 Jan 2024 10:20:13 -0700 Subject: [PATCH 158/255] add register_component_with_status signal. Remove register_sensor signal usage. --- src/handlers/__init__.py | 1 - src/handlers/apps.py | 6 +-- src/handlers/location_handler.py | 15 +++---- src/handlers/tests/test_handlers.py | 21 ++++----- src/initialization/__init__.py | 2 +- src/initialization/sensor_loader.py | 8 +--- src/sensor/wsgi.py | 68 +---------------------------- src/status/__init__.py | 3 +- src/utils/signals.py | 4 ++ 9 files changed, 25 insertions(+), 103 deletions(-) create mode 100644 src/utils/signals.py diff --git a/src/handlers/__init__.py b/src/handlers/__init__.py index fb6877a6..2d4dd503 100644 --- a/src/handlers/__init__.py +++ b/src/handlers/__init__.py @@ -2,4 +2,3 @@ logger = logging.getLogger(__name__) logger.debug("********** Initializing handlers **********") -sensors = [] \ No newline at end of file diff --git a/src/handlers/apps.py b/src/handlers/apps.py index 8c3a3f6c..8afad0e2 100644 --- a/src/handlers/apps.py +++ b/src/handlers/apps.py @@ -5,8 +5,7 @@ from scos_actions.signals import ( location_action_completed, measurement_action_completed, - trigger_api_restart, - register_sensor + trigger_api_restart ) logger = logging.getLogger(__name__) @@ -24,7 +23,6 @@ def ready(self): ) from handlers.measurement_handler import measurement_action_completed_callback - from handlers.register_sensor_handler import sensor_registered measurement_action_completed.connect(measurement_action_completed_callback) logger.debug( @@ -43,6 +41,4 @@ def ready(self): trigger_api_restart.connect(trigger_api_restart_callback) logger.debug("trigger_api_restart_callback registered to trigger_api_restart") - register_sensor.connect(sensor_registered) - logger.debug("sensor_registered handler connected to register_sensor signal") diff --git a/src/handlers/location_handler.py b/src/handlers/location_handler.py index 3cfb6d86..0afffc87 100644 --- a/src/handlers/location_handler.py +++ b/src/handlers/location_handler.py @@ -1,7 +1,6 @@ import logging -from . import sensors +from initialization import sensor_loader from status.models import GPS_LOCATION_DESCRIPTION, Location -from django.conf import settings from scos_actions.metadata.utils import construct_geojson_point logger = logging.getLogger(__name__) @@ -37,9 +36,9 @@ def db_location_updated(sender, **kwargs): logger.debug(f"DB location updated by {sender}") if isinstance(instance, Location) and instance.active: geojson = construct_geojson_point(longitude = instance.longitude, latitude=instance.latitude, altitude= instance.height) - if len(sensors) > 0: - sensors[0].location = geojson - logger.debug(f"Updated {sensors[0]} location to {geojson}") + if sensor_loader.sensor: + sensor_loader.sensor.location = geojson + logger.debug(f"Updated {sensor_loader.sensor} location to {geojson}") else: logger.warning("No sensor is registered. Unable to update sensor location.") @@ -48,8 +47,8 @@ def db_location_deleted(sender, **kwargs): instance = kwargs["instance"] if isinstance(instance, Location): if instance.active: - if len(sensors) >0: - sensors[0].location = None - logger.debug(f"Set {sensors[0]} location to None.") + if sensor_loader.sensor: + sensor_loader.sensor.location = None + logger.debug(f"Set {sensor_loader.sensor} location to None.") else: logger.warning("No sensor registered. Unable to remove sensor location.") diff --git a/src/handlers/tests/test_handlers.py b/src/handlers/tests/test_handlers.py index d4084b92..e08ad49c 100644 --- a/src/handlers/tests/test_handlers.py +++ b/src/handlers/tests/test_handlers.py @@ -1,22 +1,18 @@ import logging import pytest -from handlers import sensors from django.conf import settings +from initialization import sensor_loader from status.models import Location from scos_actions.hardware.sensor import Sensor from scos_actions.metadata.utils import construct_geojson_point -from scos_actions.signals import register_sensor logger = logging.getLogger(__name__) @pytest.mark.django_db def test_db_location_update_handler(): location = construct_geojson_point(-105.7, 40.5, 0) - sensor = Sensor(location = location) - register_sensor.send(sensor, sensor = sensor) - logger.debug(f"len(sensors) sensors registered") - logger.debug(f"sigan: {sensors[0].signal_analyzer}") - logger.debug(f"Registered sigan = {sensors}") + sensor = sensor_loader.sensor + logger.debug(f"Sensor: {sensor}") location = Location() location.gps = False location.height = 10 @@ -34,8 +30,7 @@ def test_db_location_update_handler(): @pytest.mark.django_db def test_db_location_update_handler_current_location_none(): - sensor = Sensor() - register_sensor.send(sensor, sensor = sensor) + sensor = sensor_loader.sensor logger.debug(f"len(sensors) sensors registered") logger.debug(f"sigan: {sensors[0].signal_analyzer}") logger.debug(f"Registered sigan = {sensors}") @@ -55,9 +50,9 @@ def test_db_location_update_handler_current_location_none(): @pytest.mark.django_db def test_db_location_update_handler_not_active(): + sensor = sensor_loader.sensor location = construct_geojson_point(-105.7, 40.5, 0) - sensor = Sensor(location=location) - register_sensor.send(sensor, sensor=sensor) + sensor.location = location logger.debug(f"len(sensors) sensors registered") logger.debug(f"sigan: {sensors[0].signal_analyzer}") logger.debug(f"Registered sigan = {sensors}") @@ -99,8 +94,8 @@ def test_db_location_deleted_handler(): @pytest.mark.django_db def test_db_location_deleted_inactive_handler(): location = construct_geojson_point(-105.7, 40.5, 0) - sensor = Sensor(location=location) - register_sensor.send(sensor, sensor=sensor) + sensor = sensor_loader.sensor + sensor.location = location location = Location() location.gps = False location.height = 10 diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index b535fa97..3f03cf72 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -5,7 +5,7 @@ from .sensor_loader import SensorLoader from .status_monitor import StatusMonitor -from scos_actions.signals import register_component_with_status +from utils.signals import register_component_with_status logger = logging.getLogger(__name__) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index 986921bb..fcb2465f 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -3,18 +3,13 @@ from django.conf import settings from scos_actions.hardware.sensor import Sensor from scos_actions.metadata.utils import construct_geojson_point -from scos_actions.signals import register_component_with_status -from scos_actions.signals import register_signal_analyzer from os import path from pathlib import Path from its_preselector.configuration_exception import ConfigurationException from its_preselector.controlbyweb_web_relay import ControlByWebWebRelay - from scos_actions import utils -from scos_actions.signals import register_component_with_status - from scos_actions.calibration.calibration import Calibration, load_from_json - +from utils.signals import register_component_with_status logger = logging.getLogger(__name__) @@ -54,7 +49,6 @@ def load_sensor(sensor_capabilities): sigan_cal = get_sigan_calibration(settings.SIGAN_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal) register_component_with_status.send(sigan, component=sigan) - register_signal_analyzer.send(sigan, signal_analyzer=sigan) switches = load_switches(settings.SWITCH_CONFIGS_DIR) preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELECTOR_MODULE, settings.PRESELECTOR_CLASS, sensor_capabilities["sensor"]) diff --git a/src/sensor/wsgi.py b/src/sensor/wsgi.py index 635bd5ab..610456e7 100644 --- a/src/sensor/wsgi.py +++ b/src/sensor/wsgi.py @@ -12,14 +12,11 @@ import os import django -import importlib -import logging from django.core.wsgi import get_wsgi_application os.environ.setdefault("DJANGO_SETTINGS_MODULE", "sensor.settings") django.setup() # this is necessary because we need to handle our own thread -from capabilities import sensor_capabilities from scheduler import scheduler # noqa from sensor import settings # noqa @@ -30,70 +27,7 @@ faulthandler.enable() application = get_wsgi_application() -logger = logging.getLogger(__name__) if not settings.IN_DOCKER: # Normally scheduler is started by gunicorn worker process - from scheduler import scheduler - from initialization import ( - load_preselector, - load_switches, - ) - from initialization import ( - get_sensor_calibration, - get_sigan_calibration - ) - from django.conf import settings - from status.models import Location - from scos_actions.hardware.sensor import Sensor - from scos_actions.metadata.utils import construct_geojson_point - from scos_actions.signals import register_component_with_status - from scos_actions.signals import register_signal_analyzer - from scos_actions.signals import register_sensor - - sigan_module_setting = settings.SIGAN_MODULE - sigan_module = importlib.import_module(sigan_module_setting) - logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) - sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) - sensor_cal = get_sensor_calibration(settings.SENSOR_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) - sigan_cal = get_sigan_calibration(settings.SIGAN_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) - sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal) - register_component_with_status.send(sigan, component=sigan) - register_signal_analyzer.send(sigan, signal_analyzer=sigan) - - switches = load_switches(settings.SWITCH_CONFIGS_DIR) - preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELECTOR_MODULE, settings.PRESELECTOR_CLASS, capabilities["sensor"]) - location = None - if "location" in sensor_capabilities["sensor"]: - try: - sensor_loc = sensor_capabilities["sensor"].pop("location") - try: - #if there is an active database location, use it over the value in the sensor def. - db_location = Location.objects.get(active=True) - location = construct_geojson_point(db_location.longitude, db_location.latitude, db_location.height) - except Location.DoesNotExist: - # This should never occur because status/migrations/0003_auto_20211217_2229.py - # will load the No DB location. Use sensor def location and save to DB. - location = construct_geojson_point( - sensor_loc["x"], - sensor_loc["y"], - sensor_loc["z"] if "z" in sensor_loc else None, - ) - #Save the sensor location from the sensor def to the database - db_location = Location() - db_location.longitude = sensor_loc["x"] - db_location.latitude = sensor_loc["y"] - db_location.height = sensor_loc["z"] - db_location.gps = False - db_location.description = sensor_loc["description"] - db_location.save() - except: - logger.exception("Failed to get sensor location from sensor definition.") - - - sensor = Sensor(signal_analyzer=sigan, preselector = preselector, switches = switches, capabilities = capabilities, location = location) - scheduler.thread.sensor = sensor - register_sensor.send(sensor, sensor=sensor) - scheduler.thread.start() - - + scheduler.thread.start() \ No newline at end of file diff --git a/src/status/__init__.py b/src/status/__init__.py index 62a41c79..8e0b0384 100644 --- a/src/status/__init__.py +++ b/src/status/__init__.py @@ -1,9 +1,10 @@ import datetime import logging +from initialization import sensor_loader logger = logging.getLogger(__name__) logger.debug("********** Initializing status **********") -start_time = datetime.datetime.utcnow() +start_time = sensor_loader.sensor.start_time diff --git a/src/utils/signals.py b/src/utils/signals.py new file mode 100644 index 00000000..225298b5 --- /dev/null +++ b/src/utils/signals.py @@ -0,0 +1,4 @@ +from django.dispatch import Signal + +#provides component +register_component_with_status = Signal() \ No newline at end of file From 9f5bb32b71751f7c8fe2b017cb1147259b44acee Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Wed, 17 Jan 2024 10:21:42 -0700 Subject: [PATCH 159/255] Remove sensors refs from test_handlers.py --- src/handlers/tests/test_handlers.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/handlers/tests/test_handlers.py b/src/handlers/tests/test_handlers.py index e08ad49c..d5f150cf 100644 --- a/src/handlers/tests/test_handlers.py +++ b/src/handlers/tests/test_handlers.py @@ -31,9 +31,7 @@ def test_db_location_update_handler(): @pytest.mark.django_db def test_db_location_update_handler_current_location_none(): sensor = sensor_loader.sensor - logger.debug(f"len(sensors) sensors registered") - logger.debug(f"sigan: {sensors[0].signal_analyzer}") - logger.debug(f"Registered sigan = {sensors}") + logger.debug(f"Sensor: {sensor}") location = Location() location.gps = False location.height = 10 @@ -53,9 +51,7 @@ def test_db_location_update_handler_not_active(): sensor = sensor_loader.sensor location = construct_geojson_point(-105.7, 40.5, 0) sensor.location = location - logger.debug(f"len(sensors) sensors registered") - logger.debug(f"sigan: {sensors[0].signal_analyzer}") - logger.debug(f"Registered sigan = {sensors}") + logger.debug(f"Sensor = {sensor}") location = Location() location.gps = False location.height = 10 From 80ecacf6867c08251410eaf01474fcc37a5bcd6d Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Wed, 17 Jan 2024 10:23:13 -0700 Subject: [PATCH 160/255] remove register_sensor from test_handlers.py --- src/handlers/tests/test_handlers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/handlers/tests/test_handlers.py b/src/handlers/tests/test_handlers.py index d5f150cf..4af79eff 100644 --- a/src/handlers/tests/test_handlers.py +++ b/src/handlers/tests/test_handlers.py @@ -1,6 +1,5 @@ import logging import pytest -from django.conf import settings from initialization import sensor_loader from status.models import Location from scos_actions.hardware.sensor import Sensor @@ -70,7 +69,6 @@ def test_db_location_update_handler_not_active(): def test_db_location_deleted_handler(): location = construct_geojson_point(-105.7, 40.5, 0) sensor = Sensor(location=location) - register_sensor.send(sensor, sensor=sensor) location = Location() location.gps = False location.height = 10 From 284c001141b743dcb42309cee5bd27fcf9f8699d Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Wed, 17 Jan 2024 13:47:56 -0700 Subject: [PATCH 161/255] correct checking for sensor and capabilities attributes. --- src/initialization/capabilities_loader.py | 9 +-------- src/initialization/sensor_loader.py | 2 +- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/src/initialization/capabilities_loader.py b/src/initialization/capabilities_loader.py index 10a16e72..cc7bc099 100644 --- a/src/initialization/capabilities_loader.py +++ b/src/initialization/capabilities_loader.py @@ -1,14 +1,7 @@ import hashlib import json import logging -import os -import pkgutil -import shutil from django.conf import settings -from scos_actions.actions import action_classes -from scos_actions.discover import test_actions -from scos_actions.discover import init - from scos_actions.utils import load_from_json logger = logging.getLogger(__name__) @@ -17,7 +10,7 @@ class CapabilitiesLoader(object): _instance = None def __init__(self): - if not hasattr(self, "actions"): + if not hasattr(self, "capabilities"): logger.debug("Capabilities have not been loaded. Loading...") self.capabilities = load_capabilities(settings.SENSOR_DEFINITION_FILE) else: diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index fcb2465f..faa58e90 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -17,7 +17,7 @@ class SensorLoader(object): _instance = None def __init__(self, sensor_capabilities): - if not hasattr(self, "actions"): + if not hasattr(self, "sensor"): logger.debug("Sensor has not been loaded. Loading...") self.sensor = load_sensor(sensor_capabilities) else: From 6d2382fe05261fe061fe1eed377a4d00762c7fc4 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 18 Jan 2024 07:32:21 -0700 Subject: [PATCH 162/255] Update sensor location with database location on startup. --- src/status/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/status/__init__.py b/src/status/__init__.py index 8e0b0384..4c90a318 100644 --- a/src/status/__init__.py +++ b/src/status/__init__.py @@ -1,10 +1,23 @@ import datetime import logging +from .models import Location from initialization import sensor_loader +from scos_actions.metadata.utils import construct_geojson_point logger = logging.getLogger(__name__) logger.debug("********** Initializing status **********") start_time = sensor_loader.sensor.start_time +try: + location = Location.objects.get(active=True) + db_location_geojson = construct_geojson_point( + location.longitude, + location.latitude, + location.height, + ) + logger.debug(f"Location found in DB. Updating sensor location to {location}.") + sensor_loader.sensor.location = db_location_geojson +except Location.DoesNotExist: + #No location, no problem From 8284fafc51679e6ea3be9e2308b3d59a1d87da4b Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 18 Jan 2024 07:42:02 -0700 Subject: [PATCH 163/255] Make sure sensor is not None when updating location. --- src/status/__init__.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/status/__init__.py b/src/status/__init__.py index 4c90a318..53f390d2 100644 --- a/src/status/__init__.py +++ b/src/status/__init__.py @@ -1,4 +1,3 @@ -import datetime import logging from .models import Location from initialization import sensor_loader @@ -12,15 +11,10 @@ db_location_geojson = construct_geojson_point( location.longitude, location.latitude, - location.height, + location.height ) logger.debug(f"Location found in DB. Updating sensor location to {location}.") - sensor_loader.sensor.location = db_location_geojson + if sensor_loader.sensor is not None: + sensor_loader.sensor.location = db_location_geojson except Location.DoesNotExist: - #No location, no problem - - - - - - + pass \ No newline at end of file From 9dc682f9278916a4383f3537b3c7dd53caef48b9 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 18 Jan 2024 07:50:37 -0700 Subject: [PATCH 164/255] Move location updating in status to app ready. --- src/status/__init__.py | 17 ++--------------- src/status/apps.py | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/status/__init__.py b/src/status/__init__.py index 53f390d2..0e2d5c5e 100644 --- a/src/status/__init__.py +++ b/src/status/__init__.py @@ -1,20 +1,7 @@ import logging -from .models import Location from initialization import sensor_loader -from scos_actions.metadata.utils import construct_geojson_point logger = logging.getLogger(__name__) logger.debug("********** Initializing status **********") -start_time = sensor_loader.sensor.start_time -try: - location = Location.objects.get(active=True) - db_location_geojson = construct_geojson_point( - location.longitude, - location.latitude, - location.height - ) - logger.debug(f"Location found in DB. Updating sensor location to {location}.") - if sensor_loader.sensor is not None: - sensor_loader.sensor.location = db_location_geojson -except Location.DoesNotExist: - pass \ No newline at end of file +if sensor_loader.sensor is not None: + start_time = sensor_loader.sensor.start_time diff --git a/src/status/apps.py b/src/status/apps.py index d207502e..b0dd3a66 100644 --- a/src/status/apps.py +++ b/src/status/apps.py @@ -1,5 +1,25 @@ +import logging from django.apps import AppConfig +from initialization import sensor_loader +from .models import Location +from scos_actions.metadata.utils import construct_geojson_point +logger = logging.getLogger(__name__) + class StatusConfig(AppConfig): name = "status" + + def ready(self): + try: + location = Location.objects.get(active=True) + db_location_geojson = construct_geojson_point( + location.longitude, + location.latitude, + location.height + ) + logger.debug(f"Location found in DB. Updating sensor location to {location}.") + if sensor_loader.sensor is not None: + sensor_loader.sensor.location = db_location_geojson + except Location.DoesNotExist: + pass From 2f8e431eef03c027fa81760e401b02e3d021a486 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 18 Jan 2024 08:15:14 -0700 Subject: [PATCH 165/255] import Location within ready. --- src/status/apps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/status/apps.py b/src/status/apps.py index b0dd3a66..ea3eb25a 100644 --- a/src/status/apps.py +++ b/src/status/apps.py @@ -1,7 +1,6 @@ import logging from django.apps import AppConfig from initialization import sensor_loader -from .models import Location from scos_actions.metadata.utils import construct_geojson_point @@ -11,6 +10,7 @@ class StatusConfig(AppConfig): name = "status" def ready(self): + from .models import Location try: location = Location.objects.get(active=True) db_location_geojson = construct_geojson_point( From 5638fbf8a25aa8dd663889e33a2cd40f81e7c442 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 18 Jan 2024 08:26:44 -0700 Subject: [PATCH 166/255] catch any exception in status ready. --- src/status/apps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/status/apps.py b/src/status/apps.py index ea3eb25a..0f213ea0 100644 --- a/src/status/apps.py +++ b/src/status/apps.py @@ -21,5 +21,5 @@ def ready(self): logger.debug(f"Location found in DB. Updating sensor location to {location}.") if sensor_loader.sensor is not None: sensor_loader.sensor.location = db_location_geojson - except Location.DoesNotExist: + except: pass From 50dad78429fb67f5561213a1f3be2e8aa7c0de50 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 18 Jan 2024 11:00:54 -0700 Subject: [PATCH 167/255] pass switches to sigan. --- src/initialization/sensor_loader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index faa58e90..eea75a88 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -41,15 +41,15 @@ def load_sensor(sensor_capabilities): sensor_loc["y"], sensor_loc["z"] if "z" in sensor_loc else None, ) + switches = load_switches(settings.SWITCH_CONFIGS_DIR) sigan_module_setting = settings.SIGAN_MODULE sigan_module = importlib.import_module(sigan_module_setting) logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) sensor_cal = get_sensor_calibration(settings.SENSOR_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) sigan_cal = get_sigan_calibration(settings.SIGAN_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) - sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal) + sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal, switches) register_component_with_status.send(sigan, component=sigan) - switches = load_switches(settings.SWITCH_CONFIGS_DIR) preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELECTOR_MODULE, settings.PRESELECTOR_CLASS, sensor_capabilities["sensor"]) sensor = Sensor(signal_analyzer=sigan, preselector=preselector, switches=switches, capabilities=sensor_capabilities, From 578de5be6c94ed57977161e4eb6edefaa37d0655 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 18 Jan 2024 11:05:08 -0700 Subject: [PATCH 168/255] add named argument for switches. --- src/initialization/sensor_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index eea75a88..61e3dbe1 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -48,7 +48,7 @@ def load_sensor(sensor_capabilities): sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) sensor_cal = get_sensor_calibration(settings.SENSOR_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) sigan_cal = get_sigan_calibration(settings.SIGAN_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) - sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal, switches) + sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal, switches = switches) register_component_with_status.send(sigan, component=sigan) preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELECTOR_MODULE, settings.PRESELECTOR_CLASS, sensor_capabilities["sensor"]) From 30b40e26df1d14cf386058da2b435da68f690931 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 18 Jan 2024 12:25:44 -0700 Subject: [PATCH 169/255] catch exceptions creating signal analyzer. --- src/initialization/sensor_loader.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index 61e3dbe1..05244b5a 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -42,14 +42,19 @@ def load_sensor(sensor_capabilities): sensor_loc["z"] if "z" in sensor_loc else None, ) switches = load_switches(settings.SWITCH_CONFIGS_DIR) - sigan_module_setting = settings.SIGAN_MODULE - sigan_module = importlib.import_module(sigan_module_setting) - logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) - sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) sensor_cal = get_sensor_calibration(settings.SENSOR_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) sigan_cal = get_sigan_calibration(settings.SIGAN_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) - sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal, switches = switches) - register_component_with_status.send(sigan, component=sigan) + sigan = None + try: + sigan_module_setting = settings.SIGAN_MODULE + sigan_module = importlib.import_module(sigan_module_setting) + logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) + sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) + sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal, switches = switches) + register_component_with_status.send(sigan, component=sigan) + except Exception as ex: + logger.warning(f"unable to create signal analyzer: {ex}") + preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELECTOR_MODULE, settings.PRESELECTOR_CLASS, sensor_capabilities["sensor"]) sensor = Sensor(signal_analyzer=sigan, preselector=preselector, switches=switches, capabilities=sensor_capabilities, From 80a002cd0984becce87db48a2ef8e9baa3d61bd0 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 18 Jan 2024 12:53:53 -0700 Subject: [PATCH 170/255] make SWITCH_CONFIGS_DIR a Path. --- src/initialization/sensor_loader.py | 1 + src/sensor/settings.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index 05244b5a..9824e38a 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -63,6 +63,7 @@ def load_sensor(sensor_capabilities): def load_switches(switch_dir: Path) -> dict: switch_dict = {} + if switch_dir is not None and switch_dir.is_dir(): for f in switch_dir.iterdir(): file_path = f.resolve() diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 126f2921..6d426f32 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -441,9 +441,9 @@ "PRESELECTOR_MODULE", default="its_preselector.web_relay_preselector" ) PRESELECTOR_CLASS = env.str("PRESELECTOR_CLASS", default="WebRelayPreselector") -SWITCH_CONFIGS_DIR =env.str( +SWITCH_CONFIGS_DIR = Path(env.str( "SWITCH_CONFIGS_DIR", default=str(path.join(CONFIG_DIR, "switches")) -) +)) os.environ["SWITCH_CONFIGS_DIR"] = str(SWITCH_CONFIGS_DIR) SWITCH_CONFIGS_DIR = Path(SWITCH_CONFIGS_DIR) SIGAN_POWER_CYCLE_STATES = env("SIGAN_POWER_CYCLE_STATES", default=None) From 65e577b8d4f5954c314af755e35bb54b23fab30f Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 18 Jan 2024 13:10:00 -0700 Subject: [PATCH 171/255] Use environment variable to flag when running migrations to avoid connecting to signal analyzer. --- entrypoints/api_entrypoint.sh | 3 +- src/initialization/sensor_loader.py | 47 +++++++++++++++-------------- src/sensor/settings.py | 1 + 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/entrypoints/api_entrypoint.sh b/entrypoints/api_entrypoint.sh index 0fa7f5e0..82171619 100644 --- a/entrypoints/api_entrypoint.sh +++ b/entrypoints/api_entrypoint.sh @@ -10,9 +10,10 @@ function cleanup_demodb { trap cleanup_demodb SIGTERM trap cleanup_demodb SIGINT +export RUNNING_MIGRATIONS = True echo "Starting Migrations" python3.8 manage.py migrate - +RUNNING_MIGRATIONS = False echo "Creating superuser (if managed)" python3.8 /scripts/create_superuser.py diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index 9824e38a..7101ceed 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -46,12 +46,15 @@ def load_sensor(sensor_capabilities): sigan_cal = get_sigan_calibration(settings.SIGAN_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) sigan = None try: - sigan_module_setting = settings.SIGAN_MODULE - sigan_module = importlib.import_module(sigan_module_setting) - logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) - sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) - sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal, switches = switches) - register_component_with_status.send(sigan, component=sigan) + if not settings.RUNNING_MIGRATIONS: + sigan_module_setting = settings.SIGAN_MODULE + sigan_module = importlib.import_module(sigan_module_setting) + logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) + sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) + sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal, switches = switches) + register_component_with_status.send(sigan, component=sigan) + else: + logger.info("Running migrations. Not loading signal analyzer.") except Exception as ex: logger.warning(f"unable to create signal analyzer: {ex}") @@ -63,22 +66,22 @@ def load_sensor(sensor_capabilities): def load_switches(switch_dir: Path) -> dict: switch_dict = {} - - if switch_dir is not None and switch_dir.is_dir(): - for f in switch_dir.iterdir(): - file_path = f.resolve() - logger.debug(f"loading switch config {file_path}") - conf = utils.load_from_json(file_path) - try: - switch = ControlByWebWebRelay(conf) - logger.debug(f"Adding {switch.id}") - - switch_dict[switch.id] = switch - logger.debug(f"Registering switch status for {switch.name}") - register_component_with_status.send(__name__, component=switch) - except ConfigurationException: - logger.error(f"Unable to configure switch defined in: {file_path}") - + try: + if switch_dir is not None and switch_dir.is_dir(): + for f in switch_dir.iterdir(): + file_path = f.resolve() + logger.debug(f"loading switch config {file_path}") + conf = utils.load_from_json(file_path) + try: + switch = ControlByWebWebRelay(conf) + logger.debug(f"Adding {switch.id}") + switch_dict[switch.id] = switch + logger.debug(f"Registering switch status for {switch.name}") + register_component_with_status.send(__name__, component=switch) + except ConfigurationException: + logger.error(f"Unable to configure switch defined in: {file_path}") + except Exception as ex: + logger.error(f"Unable to load switches {ex}") return switch_dict diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 6d426f32..48585ad7 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -185,6 +185,7 @@ ``` """ +RUNNING_MIGRATIONS=env.bool("RUNNING_MIGRATIONS", True) INSTALLED_APPS = [ "django.contrib.admin", From 5b608209871693d907644e27672217b620e23851 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 18 Jan 2024 13:37:07 -0700 Subject: [PATCH 172/255] default running migrations to false to ensure sensor is created. --- entrypoints/api_entrypoint.sh | 6 +++--- src/sensor/settings.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/entrypoints/api_entrypoint.sh b/entrypoints/api_entrypoint.sh index 82171619..f87d7d53 100644 --- a/entrypoints/api_entrypoint.sh +++ b/entrypoints/api_entrypoint.sh @@ -10,13 +10,13 @@ function cleanup_demodb { trap cleanup_demodb SIGTERM trap cleanup_demodb SIGINT -export RUNNING_MIGRATIONS = True +RUNNING_MIGRATIONS="True" +export RUNNING_MIGRATIONS echo "Starting Migrations" python3.8 manage.py migrate -RUNNING_MIGRATIONS = False +RUNNING_MIGRATIONS="False" echo "Creating superuser (if managed)" python3.8 /scripts/create_superuser.py - echo "Starting Gunicorn" exec gunicorn sensor.wsgi -c ../gunicorn/config.py & wait diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 48585ad7..efef52fb 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -185,7 +185,7 @@ ``` """ -RUNNING_MIGRATIONS=env.bool("RUNNING_MIGRATIONS", True) +RUNNING_MIGRATIONS=env.bool("RUNNING_MIGRATIONS", False) INSTALLED_APPS = [ "django.contrib.admin", From 8b3aa0c10fab89cb07a3eb76aa1cac7565b41ed0 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 18 Jan 2024 14:17:23 -0700 Subject: [PATCH 173/255] defer instantiation of signal_analyzer to sigan repo. --- src/initialization/__init__.py | 2 +- src/initialization/action_loader.py | 10 ++++++---- src/initialization/sensor_loader.py | 30 ++++++++++++++--------------- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 3f03cf72..bba42316 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -23,5 +23,5 @@ def status_registration_handler(sender, **kwargs): action_loader = ActionLoader() logger.debug(f"Actions ActionLoader has {len(action_loader.actions)} actions") capabilities_loader = CapabilitiesLoader() -sensor_loader = SensorLoader(capabilities_loader.capabilities) +sensor_loader = SensorLoader(action_loader.signal_analyzer, capabilities_loader.capabilities) diff --git a/src/initialization/action_loader.py b/src/initialization/action_loader.py index 74423beb..2f8fa1d3 100644 --- a/src/initialization/action_loader.py +++ b/src/initialization/action_loader.py @@ -17,7 +17,7 @@ class ActionLoader(object): def __init__(self): if not hasattr(self, "actions"): logger.debug("Actions have not been loaded. Loading actions...") - self.actions = load_actions(settings.MOCK_SIGAN, settings.RUNNING_TESTS, settings.DRIVERS_DIR, + self.actions, self.signal_analyzer = load_actions_and_sigan(settings.MOCK_SIGAN, settings.RUNNING_TESTS, settings.DRIVERS_DIR, settings.ACTIONS_DIR) else: logger.debug("Already loaded actions. ") @@ -60,7 +60,7 @@ def copy_driver_files(driver_dir): logger.error(f"Failed to copy {source_path} to {dest_path}") logger.error(e) -def load_actions(mock_sigan, running_tests, driver_dir, action_dir): +def load_actions_and_sigan(mock_sigan, running_tests, driver_dir, action_dir): logger.debug("********** Initializing actions **********") copy_driver_files(driver_dir) # copy driver files before loading plugins @@ -71,7 +71,7 @@ def load_actions(mock_sigan, running_tests, driver_dir, action_dir): } logger.debug(discovered_plugins) actions = {} - + signal_analyzer = None if mock_sigan or running_tests: logger.debug(f"Loading {len(test_actions)} test actions.") actions.update(test_actions) @@ -84,9 +84,11 @@ def load_actions(mock_sigan, running_tests, driver_dir, action_dir): actions.update(discover.actions) if hasattr(discover, "action_classes") and discover.action_classes is not None: action_classes.update(discover.action_classes) + if hasattr(discover, "signal_analyzer") and discover.signal_analyzer is not None: + signal_analyzer = discover.signal_analyzer logger.debug(f"Loading actions in {action_dir}") yaml_actions, yaml_test_actions = init(action_classes=action_classes, yaml_dir=action_dir) actions.update(yaml_actions) logger.debug("Finished loading and registering actions") - return actions + return actions, signal_analyzer diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index 7101ceed..36269cf9 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -29,7 +29,7 @@ def __new__(cls, sensor_capabilities): cls._instance = super(SensorLoader, cls).__new__(cls) return cls._instance -def load_sensor(sensor_capabilities): +def load_sensor(signal_analyzer, sensor_capabilities): location = None #Remove location from sensor definition and convert to geojson. #Db may have an updated location, but status module will update it @@ -44,23 +44,23 @@ def load_sensor(sensor_capabilities): switches = load_switches(settings.SWITCH_CONFIGS_DIR) sensor_cal = get_sensor_calibration(settings.SENSOR_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) sigan_cal = get_sigan_calibration(settings.SIGAN_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) - sigan = None - try: - if not settings.RUNNING_MIGRATIONS: - sigan_module_setting = settings.SIGAN_MODULE - sigan_module = importlib.import_module(sigan_module_setting) - logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) - sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) - sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal, switches = switches) - register_component_with_status.send(sigan, component=sigan) - else: - logger.info("Running migrations. Not loading signal analyzer.") - except Exception as ex: - logger.warning(f"unable to create signal analyzer: {ex}") +# sigan = None +# try: +# if not settings.RUNNING_MIGRATIONS: +# sigan_module_setting = settings.SIGAN_MODULE +# sigan_module = importlib.import_module(sigan_module_setting) +# logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) +# sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) +# sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal, switches = switches) +# register_component_with_status.send(sigan, component=sigan) +# else: +# logger.info("Running migrations. Not loading signal analyzer.") +# except Exception as ex: +# logger.warning(f"unable to create signal analyzer: {ex}") preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELECTOR_MODULE, settings.PRESELECTOR_CLASS, sensor_capabilities["sensor"]) - sensor = Sensor(signal_analyzer=sigan, preselector=preselector, switches=switches, capabilities=sensor_capabilities, + sensor = Sensor(signal_analyzer=signal_analyzer, preselector=preselector, switches=switches, capabilities=sensor_capabilities, location=location) return sensor From 0b3c7d6401c1b2589d3c7814a9e23b387d259cdb Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 18 Jan 2024 14:22:39 -0700 Subject: [PATCH 174/255] add signal_analyzer to sensor loader. --- src/initialization/sensor_loader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index 36269cf9..c6e6d904 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -16,10 +16,10 @@ class SensorLoader(object): _instance = None - def __init__(self, sensor_capabilities): + def __init__(self, signal_analyzer, sensor_capabilities): if not hasattr(self, "sensor"): logger.debug("Sensor has not been loaded. Loading...") - self.sensor = load_sensor(sensor_capabilities) + self.sensor = load_sensor(signal_analyzer, sensor_capabilities) else: logger.debug("Already loaded sensor. ") From 812f11d853a8eca29448b554bc6d9f32f52800ab Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 18 Jan 2024 14:38:09 -0700 Subject: [PATCH 175/255] Add signal_analyzer to SensorLoader new. --- src/initialization/sensor_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index c6e6d904..84837426 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -23,7 +23,7 @@ def __init__(self, signal_analyzer, sensor_capabilities): else: logger.debug("Already loaded sensor. ") - def __new__(cls, sensor_capabilities): + def __new__(cls, signal_analyzer, sensor_capabilities): if cls._instance is None: logger.debug('Creating the SensorLoader') cls._instance = super(SensorLoader, cls).__new__(cls) From 666ab227c5840e6372eb4000296947ba35f3826b Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 18 Jan 2024 14:51:34 -0700 Subject: [PATCH 176/255] Only add last_calibration_datetime if it exists. --- src/status/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/status/views.py b/src/status/views.py index 72222c44..19c1d92b 100644 --- a/src/status/views.py +++ b/src/status/views.py @@ -59,10 +59,11 @@ def status(request, version, format=None): "location": serialize_location(), "system_time": get_datetime_str_now(), "start_time": convert_datetime_to_millisecond_iso_format(start_time), - "last_calibration_datetime": sensor_loader.sensor.signal_analyzer.sensor_calibration.last_calibration_datetime, "disk_usage": disk_usage(), "days_up": get_days_up(), } + if sensor_loader.sensor is not None and sensor_loader.sensor.signal_analyzer is not None and sensor_loader.sensor.signal_analyzer.sensor_calibration is not None : + status_json["last_calibration_datetime"] = sensor_loader.sensor.signal_analyzer.sensor_calibration.last_calibration_datetime, for component in status_monitor.status_components: component_status = component.get_status() if isinstance(component, WebRelay): From 7819d201367aae242336a9b46a78e0a750cdd660 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 18 Jan 2024 14:53:05 -0700 Subject: [PATCH 177/255] log signal analyzer found. --- src/initialization/action_loader.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/initialization/action_loader.py b/src/initialization/action_loader.py index 2f8fa1d3..1e4f8d4c 100644 --- a/src/initialization/action_loader.py +++ b/src/initialization/action_loader.py @@ -85,6 +85,7 @@ def load_actions_and_sigan(mock_sigan, running_tests, driver_dir, action_dir): if hasattr(discover, "action_classes") and discover.action_classes is not None: action_classes.update(discover.action_classes) if hasattr(discover, "signal_analyzer") and discover.signal_analyzer is not None: + logger.debug(f"Found signal_analyzer: {discover.signal_analyzer}") signal_analyzer = discover.signal_analyzer logger.debug(f"Loading actions in {action_dir}") From 272f8ca4bd4fc57832b81e494baea1ef527674f5 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 18 Jan 2024 15:19:40 -0700 Subject: [PATCH 178/255] debugging. --- src/initialization/__init__.py | 1 + src/initialization/action_loader.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index bba42316..ba238f31 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -22,6 +22,7 @@ def status_registration_handler(sender, **kwargs): action_loader = ActionLoader() logger.debug(f"Actions ActionLoader has {len(action_loader.actions)} actions") +logger.debug(f"Action loader sigan: {action_loader.signal_analyzer}") capabilities_loader = CapabilitiesLoader() sensor_loader = SensorLoader(action_loader.signal_analyzer, capabilities_loader.capabilities) diff --git a/src/initialization/action_loader.py b/src/initialization/action_loader.py index 1e4f8d4c..3e229e36 100644 --- a/src/initialization/action_loader.py +++ b/src/initialization/action_loader.py @@ -87,6 +87,8 @@ def load_actions_and_sigan(mock_sigan, running_tests, driver_dir, action_dir): if hasattr(discover, "signal_analyzer") and discover.signal_analyzer is not None: logger.debug(f"Found signal_analyzer: {discover.signal_analyzer}") signal_analyzer = discover.signal_analyzer + else: + logger.debug(f"{discover} has no signal_analyzer attribute") logger.debug(f"Loading actions in {action_dir}") yaml_actions, yaml_test_actions = init(action_classes=action_classes, yaml_dir=action_dir) From dc37ebb09e62e8fc27dc2dbee69d5d75624c5ad8 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 18 Jan 2024 15:31:01 -0700 Subject: [PATCH 179/255] debugging --- src/initialization/__init__.py | 20 +++++++++++--------- src/initialization/sensor_loader.py | 2 ++ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index ba238f31..1af78662 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -17,12 +17,14 @@ def status_registration_handler(sender, **kwargs): status_monitor.add_component(kwargs["component"]) except: logger.exception("Error registering status component") - -register_component_with_status.connect(status_registration_handler) - -action_loader = ActionLoader() -logger.debug(f"Actions ActionLoader has {len(action_loader.actions)} actions") -logger.debug(f"Action loader sigan: {action_loader.signal_analyzer}") -capabilities_loader = CapabilitiesLoader() -sensor_loader = SensorLoader(action_loader.signal_analyzer, capabilities_loader.capabilities) - +try: + register_component_with_status.connect(status_registration_handler) + + action_loader = ActionLoader() + logger.debug(f"Actions ActionLoader has {len(action_loader.actions)} actions") + logger.debug(f"Action loader sigan: {action_loader.signal_analyzer}") + capabilities_loader = CapabilitiesLoader() + logger.debug("Calling sensor loader.") + sensor_loader = SensorLoader(action_loader.signal_analyzer, capabilities_loader.capabilities) +except Exception as ex: + logger.error(f"Error during initialization: {ex}") diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index 84837426..3d8a1878 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -60,11 +60,13 @@ def load_sensor(signal_analyzer, sensor_capabilities): preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELECTOR_MODULE, settings.PRESELECTOR_CLASS, sensor_capabilities["sensor"]) + logger.debug("Creating sensor") sensor = Sensor(signal_analyzer=signal_analyzer, preselector=preselector, switches=switches, capabilities=sensor_capabilities, location=location) return sensor def load_switches(switch_dir: Path) -> dict: + logger.debug(f"Loading switches in {switch_dir}") switch_dict = {} try: if switch_dir is not None and switch_dir.is_dir(): From 610778e4ce807f03a5204434375309d3c6afe04c Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 18 Jan 2024 15:59:47 -0700 Subject: [PATCH 180/255] debugging --- src/initialization/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 1af78662..84557156 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -21,6 +21,7 @@ def status_registration_handler(sender, **kwargs): register_component_with_status.connect(status_registration_handler) action_loader = ActionLoader() + logger.debug("test") logger.debug(f"Actions ActionLoader has {len(action_loader.actions)} actions") logger.debug(f"Action loader sigan: {action_loader.signal_analyzer}") capabilities_loader = CapabilitiesLoader() From 2b235fee827b2ae7b9abb1a95157ed8e3fc6e777 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 18 Jan 2024 16:37:47 -0700 Subject: [PATCH 181/255] load switches in sigan. --- src/initialization/sensor_loader.py | 51 ++++++++++++++++------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index 3d8a1878..d9768835 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -29,7 +29,7 @@ def __new__(cls, signal_analyzer, sensor_capabilities): cls._instance = super(SensorLoader, cls).__new__(cls) return cls._instance -def load_sensor(signal_analyzer, sensor_capabilities): +def load_sensor(signal_analyzer: SignalAnalyzer, sensor_capabilities): location = None #Remove location from sensor definition and convert to geojson. #Db may have an updated location, but status module will update it @@ -41,7 +41,7 @@ def load_sensor(signal_analyzer, sensor_capabilities): sensor_loc["y"], sensor_loc["z"] if "z" in sensor_loc else None, ) - switches = load_switches(settings.SWITCH_CONFIGS_DIR) + #switches = load_switches(settings.SWITCH_CONFIGS_DIR) sensor_cal = get_sensor_calibration(settings.SENSOR_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) sigan_cal = get_sigan_calibration(settings.SIGAN_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) # sigan = None @@ -61,30 +61,35 @@ def load_sensor(signal_analyzer, sensor_capabilities): preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELECTOR_MODULE, settings.PRESELECTOR_CLASS, sensor_capabilities["sensor"]) logger.debug("Creating sensor") - sensor = Sensor(signal_analyzer=signal_analyzer, preselector=preselector, switches=switches, capabilities=sensor_capabilities, + signal_analyzer.sensor_calibration = sensor_cal + signal_analyzer.sigan_calibration = sigan_cal + for name, switch in signal_analyzer.switches.items(): + register_component_with_status(name, switch) + + sensor = Sensor(signal_analyzer=signal_analyzer, preselector=preselector, switches=signal_analyzer.switches, capabilities=sensor_capabilities, location=location) return sensor -def load_switches(switch_dir: Path) -> dict: - logger.debug(f"Loading switches in {switch_dir}") - switch_dict = {} - try: - if switch_dir is not None and switch_dir.is_dir(): - for f in switch_dir.iterdir(): - file_path = f.resolve() - logger.debug(f"loading switch config {file_path}") - conf = utils.load_from_json(file_path) - try: - switch = ControlByWebWebRelay(conf) - logger.debug(f"Adding {switch.id}") - switch_dict[switch.id] = switch - logger.debug(f"Registering switch status for {switch.name}") - register_component_with_status.send(__name__, component=switch) - except ConfigurationException: - logger.error(f"Unable to configure switch defined in: {file_path}") - except Exception as ex: - logger.error(f"Unable to load switches {ex}") - return switch_dict +# def load_switches(switch_dir: Path) -> dict: +# logger.debug(f"Loading switches in {switch_dir}") +# switch_dict = {} +# try: +# if switch_dir is not None and switch_dir.is_dir(): +# for f in switch_dir.iterdir(): +# file_path = f.resolve() +# logger.debug(f"loading switch config {file_path}") +# conf = utils.load_from_json(file_path) +# try: +# switch = ControlByWebWebRelay(conf) +# logger.debug(f"Adding {switch.id}") +# switch_dict[switch.id] = switch +# logger.debug(f"Registering switch status for {switch.name}") +# register_component_with_status.send(__name__, component=switch) +# except ConfigurationException: +# logger.error(f"Unable to configure switch defined in: {file_path}") +# except Exception as ex: +# logger.error(f"Unable to load switches {ex}") +# return switch_dict def load_preselector_from_file(preselector_module, preselector_class, preselector_config_file: Path): From fd46c57aa1bb767ce2718bf0eea6b970368bde5d Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 19 Jan 2024 07:10:28 -0700 Subject: [PATCH 182/255] Return to loading sigan in sensor loader for testing. --- src/initialization/__init__.py | 2 +- src/initialization/action_loader.py | 14 +++--- src/initialization/sensor_loader.py | 75 +++++++++++++++-------------- 3 files changed, 46 insertions(+), 45 deletions(-) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 84557156..64d0f0aa 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -26,6 +26,6 @@ def status_registration_handler(sender, **kwargs): logger.debug(f"Action loader sigan: {action_loader.signal_analyzer}") capabilities_loader = CapabilitiesLoader() logger.debug("Calling sensor loader.") - sensor_loader = SensorLoader(action_loader.signal_analyzer, capabilities_loader.capabilities) + sensor_loader = SensorLoader(capabilities_loader.capabilities) except Exception as ex: logger.error(f"Error during initialization: {ex}") diff --git a/src/initialization/action_loader.py b/src/initialization/action_loader.py index 3e229e36..fa3289d8 100644 --- a/src/initialization/action_loader.py +++ b/src/initialization/action_loader.py @@ -17,7 +17,7 @@ class ActionLoader(object): def __init__(self): if not hasattr(self, "actions"): logger.debug("Actions have not been loaded. Loading actions...") - self.actions, self.signal_analyzer = load_actions_and_sigan(settings.MOCK_SIGAN, settings.RUNNING_TESTS, settings.DRIVERS_DIR, + self.actions = load_actions_and_sigan(settings.MOCK_SIGAN, settings.RUNNING_TESTS, settings.DRIVERS_DIR, settings.ACTIONS_DIR) else: logger.debug("Already loaded actions. ") @@ -84,14 +84,14 @@ def load_actions_and_sigan(mock_sigan, running_tests, driver_dir, action_dir): actions.update(discover.actions) if hasattr(discover, "action_classes") and discover.action_classes is not None: action_classes.update(discover.action_classes) - if hasattr(discover, "signal_analyzer") and discover.signal_analyzer is not None: - logger.debug(f"Found signal_analyzer: {discover.signal_analyzer}") - signal_analyzer = discover.signal_analyzer - else: - logger.debug(f"{discover} has no signal_analyzer attribute") + # if hasattr(discover, "signal_analyzer") and discover.signal_analyzer is not None: + # logger.debug(f"Found signal_analyzer: {discover.signal_analyzer}") + # signal_analyzer = discover.signal_analyzer + # else: + # logger.debug(f"{discover} has no signal_analyzer attribute") logger.debug(f"Loading actions in {action_dir}") yaml_actions, yaml_test_actions = init(action_classes=action_classes, yaml_dir=action_dir) actions.update(yaml_actions) logger.debug("Finished loading and registering actions") - return actions, signal_analyzer + return actions diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index d9768835..5b223e58 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -2,6 +2,7 @@ import logging from django.conf import settings from scos_actions.hardware.sensor import Sensor +from scos_actions.hardware.sigan_iface import SignalAnalyzerInterface from scos_actions.metadata.utils import construct_geojson_point from os import path from pathlib import Path @@ -16,20 +17,20 @@ class SensorLoader(object): _instance = None - def __init__(self, signal_analyzer, sensor_capabilities): + def __init__(self, sensor_capabilities): if not hasattr(self, "sensor"): logger.debug("Sensor has not been loaded. Loading...") self.sensor = load_sensor(signal_analyzer, sensor_capabilities) else: logger.debug("Already loaded sensor. ") - def __new__(cls, signal_analyzer, sensor_capabilities): + def __new__(cls, sensor_capabilities): if cls._instance is None: logger.debug('Creating the SensorLoader') cls._instance = super(SensorLoader, cls).__new__(cls) return cls._instance -def load_sensor(signal_analyzer: SignalAnalyzer, sensor_capabilities): +def load_sensor(signal_analyzer: SignalAnalyzerInterface, sensor_capabilities): location = None #Remove location from sensor definition and convert to geojson. #Db may have an updated location, but status module will update it @@ -41,22 +42,22 @@ def load_sensor(signal_analyzer: SignalAnalyzer, sensor_capabilities): sensor_loc["y"], sensor_loc["z"] if "z" in sensor_loc else None, ) - #switches = load_switches(settings.SWITCH_CONFIGS_DIR) + switches = load_switches(settings.SWITCH_CONFIGS_DIR) sensor_cal = get_sensor_calibration(settings.SENSOR_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) sigan_cal = get_sigan_calibration(settings.SIGAN_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) -# sigan = None -# try: -# if not settings.RUNNING_MIGRATIONS: -# sigan_module_setting = settings.SIGAN_MODULE -# sigan_module = importlib.import_module(sigan_module_setting) -# logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) -# sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) -# sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal, switches = switches) -# register_component_with_status.send(sigan, component=sigan) -# else: -# logger.info("Running migrations. Not loading signal analyzer.") -# except Exception as ex: -# logger.warning(f"unable to create signal analyzer: {ex}") + sigan = None + try: + if not settings.RUNNING_MIGRATIONS: + sigan_module_setting = settings.SIGAN_MODULE + sigan_module = importlib.import_module(sigan_module_setting) + logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) + sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) + sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal, switches = switches) + register_component_with_status.send(sigan, component=sigan) + else: + logger.info("Running migrations. Not loading signal analyzer.") + except Exception as ex: + logger.warning(f"unable to create signal analyzer: {ex}") preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELECTOR_MODULE, settings.PRESELECTOR_CLASS, sensor_capabilities["sensor"]) @@ -70,26 +71,26 @@ def load_sensor(signal_analyzer: SignalAnalyzer, sensor_capabilities): location=location) return sensor -# def load_switches(switch_dir: Path) -> dict: -# logger.debug(f"Loading switches in {switch_dir}") -# switch_dict = {} -# try: -# if switch_dir is not None and switch_dir.is_dir(): -# for f in switch_dir.iterdir(): -# file_path = f.resolve() -# logger.debug(f"loading switch config {file_path}") -# conf = utils.load_from_json(file_path) -# try: -# switch = ControlByWebWebRelay(conf) -# logger.debug(f"Adding {switch.id}") -# switch_dict[switch.id] = switch -# logger.debug(f"Registering switch status for {switch.name}") -# register_component_with_status.send(__name__, component=switch) -# except ConfigurationException: -# logger.error(f"Unable to configure switch defined in: {file_path}") -# except Exception as ex: -# logger.error(f"Unable to load switches {ex}") -# return switch_dict +def load_switches(switch_dir: Path) -> dict: + logger.debug(f"Loading switches in {switch_dir}") + switch_dict = {} + try: + if switch_dir is not None and switch_dir.is_dir(): + for f in switch_dir.iterdir(): + file_path = f.resolve() + logger.debug(f"loading switch config {file_path}") + conf = utils.load_from_json(file_path) + try: + switch = ControlByWebWebRelay(conf) + logger.debug(f"Adding {switch.id}") + switch_dict[switch.id] = switch + logger.debug(f"Registering switch status for {switch.name}") + register_component_with_status.send(__name__, component=switch) + except ConfigurationException: + logger.error(f"Unable to configure switch defined in: {file_path}") + except Exception as ex: + logger.error(f"Unable to load switches {ex}") + return switch_dict def load_preselector_from_file(preselector_module, preselector_class, preselector_config_file: Path): From 08a942181b95dea519d4d765728f6c4bd9b73c4a Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 19 Jan 2024 07:13:50 -0700 Subject: [PATCH 183/255] remove sigan arg. --- src/initialization/sensor_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index 5b223e58..bd024074 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -20,7 +20,7 @@ class SensorLoader(object): def __init__(self, sensor_capabilities): if not hasattr(self, "sensor"): logger.debug("Sensor has not been loaded. Loading...") - self.sensor = load_sensor(signal_analyzer, sensor_capabilities) + self.sensor = load_sensor(sensor_capabilities) else: logger.debug("Already loaded sensor. ") From 58ca738068b36124ee9dcd5f8895c2f690bb6e4e Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 19 Jan 2024 07:19:44 -0700 Subject: [PATCH 184/255] remove action_loader.signal_analyzer. --- src/initialization/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 64d0f0aa..ece51a20 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -23,7 +23,6 @@ def status_registration_handler(sender, **kwargs): action_loader = ActionLoader() logger.debug("test") logger.debug(f"Actions ActionLoader has {len(action_loader.actions)} actions") - logger.debug(f"Action loader sigan: {action_loader.signal_analyzer}") capabilities_loader = CapabilitiesLoader() logger.debug("Calling sensor loader.") sensor_loader = SensorLoader(capabilities_loader.capabilities) From aa31c9cc83ff1f040a87fcf667d402fa1105ae8d Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 19 Jan 2024 07:30:44 -0700 Subject: [PATCH 185/255] remove sigan arg from load_sensor. --- src/initialization/sensor_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index bd024074..21db2a98 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -30,7 +30,7 @@ def __new__(cls, sensor_capabilities): cls._instance = super(SensorLoader, cls).__new__(cls) return cls._instance -def load_sensor(signal_analyzer: SignalAnalyzerInterface, sensor_capabilities): +def load_sensor(sensor_capabilities): location = None #Remove location from sensor definition and convert to geojson. #Db may have an updated location, but status module will update it From 92b7dfc73ac0a258fbd63d4015906422fb7f3c55 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 19 Jan 2024 07:36:40 -0700 Subject: [PATCH 186/255] use dynamically loaded sigan. --- src/initialization/sensor_loader.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index 21db2a98..e1f5c627 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -61,13 +61,8 @@ def load_sensor(sensor_capabilities): preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELECTOR_MODULE, settings.PRESELECTOR_CLASS, sensor_capabilities["sensor"]) - logger.debug("Creating sensor") - signal_analyzer.sensor_calibration = sensor_cal - signal_analyzer.sigan_calibration = sigan_cal - for name, switch in signal_analyzer.switches.items(): - register_component_with_status(name, switch) - sensor = Sensor(signal_analyzer=signal_analyzer, preselector=preselector, switches=signal_analyzer.switches, capabilities=sensor_capabilities, + sensor = Sensor(signal_analyzer=sigan, preselector=preselector, switches=switches, capabilities=sensor_capabilities, location=location) return sensor From 8993bdd804fd7590468a006522702ed2f94a15c8 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 19 Jan 2024 12:00:15 -0700 Subject: [PATCH 187/255] handle SIGABRT. --- src/handlers/register_sensor_handler.py | 12 ------------ src/initialization/sensor_loader.py | 7 +++++++ src/sensor/settings.py | 3 ++- 3 files changed, 9 insertions(+), 13 deletions(-) delete mode 100644 src/handlers/register_sensor_handler.py diff --git a/src/handlers/register_sensor_handler.py b/src/handlers/register_sensor_handler.py deleted file mode 100644 index 09f139f2..00000000 --- a/src/handlers/register_sensor_handler.py +++ /dev/null @@ -1,12 +0,0 @@ -import logging -from . import sensors - -logger = logging.getLogger(__name__) - -def sensor_registered(sender, **kwargs): - sensor = kwargs["sensor"] - if len(sensors) > 0: - sensors[0] = sensor - else: - sensors.append(sensor) - logger.debug(f"Registered sensor {sensor}") \ No newline at end of file diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index e1f5c627..9bf9d01c 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -1,5 +1,6 @@ import importlib import logging +import signal from django.conf import settings from scos_actions.hardware.sensor import Sensor from scos_actions.hardware.sigan_iface import SignalAnalyzerInterface @@ -30,6 +31,9 @@ def __new__(cls, sensor_capabilities): cls._instance = super(SensorLoader, cls).__new__(cls) return cls._instance +def sigabrt_handler(signum, frame): + logger.error("Recieved SIGABRT") + def load_sensor(sensor_capabilities): location = None #Remove location from sensor definition and convert to geojson. @@ -48,6 +52,7 @@ def load_sensor(sensor_capabilities): sigan = None try: if not settings.RUNNING_MIGRATIONS: + signal.signal(signal.SIGABRT, sigabrt_handler) sigan_module_setting = settings.SIGAN_MODULE sigan_module = importlib.import_module(sigan_module_setting) logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) @@ -66,6 +71,8 @@ def load_sensor(sensor_capabilities): location=location) return sensor + + def load_switches(switch_dir: Path) -> dict: logger.debug(f"Loading switches in {switch_dir}") switch_dict = {} diff --git a/src/sensor/settings.py b/src/sensor/settings.py index efef52fb..67ff386f 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -26,7 +26,6 @@ logger = logging.getLogger(__name__) logger.debug("Initializing scos-sensor settings.") env = Env() - # See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ # Build paths inside the project like this: path.join(BASE_DIR, ...) @@ -89,6 +88,8 @@ if path.exists(path.join(CONFIG_DIR, "sensor_definition.json")): SENSOR_DEFINITION_FILE = path.join(CONFIG_DIR, "sensor_definition.json") +os.environ["SENSOR_CALIBRATION_FILE"] = SENSOR_CALIBRATION_FILE +os.environ["SIGAN_CALIBRATION_FILE"] = SIGAN_CALIBRATION_FILE MEDIA_ROOT = path.join(REPO_ROOT, "files") PRESELECTOR_CONFIG = path.join(CONFIG_DIR, "preselector_config.json") From d77688cd580166fb9f64b5f23b54c92f557581d5 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 19 Jan 2024 12:23:17 -0700 Subject: [PATCH 188/255] use importlib to load sigan. --- src/handlers/health_check_handler.py | 2 +- src/initialization/sensor_loader.py | 17 +-- src/utils/sigan_loader.py | 150 +++++++++++++++++++++++++++ 3 files changed, 161 insertions(+), 8 deletions(-) create mode 100644 src/utils/sigan_loader.py diff --git a/src/handlers/health_check_handler.py b/src/handlers/health_check_handler.py index d04967b5..c4423038 100644 --- a/src/handlers/health_check_handler.py +++ b/src/handlers/health_check_handler.py @@ -1,7 +1,7 @@ import logging from pathlib import Path -from sensor import settings +from django.conf import settings logger = logging.getLogger(__name__) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index 9bf9d01c..214b2579 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -52,13 +52,16 @@ def load_sensor(sensor_capabilities): sigan = None try: if not settings.RUNNING_MIGRATIONS: - signal.signal(signal.SIGABRT, sigabrt_handler) - sigan_module_setting = settings.SIGAN_MODULE - sigan_module = importlib.import_module(sigan_module_setting) - logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) - sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) - sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal, switches = switches) - register_component_with_status.send(sigan, component=sigan) + sigan_loader = importlib.import_module("utils.sigan_loader") + sigan = sigan_loader.signal_analyzer + register_component_with_status(sigan, coponent=sigan) + # signal.signal(signal.SIGABRT, sigabrt_handler) + # sigan_module_setting = settings.SIGAN_MODULE + # sigan_module = importlib.import_module(sigan_module_setting) + # logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) + # sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) + # sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal, switches = switches) + # register_component_with_status.send(sigan, component=sigan) else: logger.info("Running migrations. Not loading signal analyzer.") except Exception as ex: diff --git a/src/utils/sigan_loader.py b/src/utils/sigan_loader.py new file mode 100644 index 00000000..8d184ab2 --- /dev/null +++ b/src/utils/sigan_loader.py @@ -0,0 +1,150 @@ +import importlib +import logging +from django.conf import settings +from os import path +from pathlib import Path +from its_preselector.configuration_exception import ConfigurationException +from its_preselector.controlbyweb_web_relay import ControlByWebWebRelay +from scos_actions import utils +from scos_actions.calibration.calibration import Calibration, load_from_json +from environs import Env + +logger = logging.getLogger(__name__) + +def load_switches(switch_dir: Path) -> dict: + logger.debug(f"Loading switches in {switch_dir}") + switch_dict = {} + try: + if switch_dir is not None and switch_dir.is_dir(): + for f in switch_dir.iterdir(): + file_path = f.resolve() + logger.debug(f"loading switch config {file_path}") + conf = utils.load_from_json(file_path) + try: + switch = ControlByWebWebRelay(conf) + logger.debug(f"Adding {switch.id}") + switch_dict[switch.id] = switch + logger.debug(f"Registering switch status for {switch.name}") + except ConfigurationException: + logger.error(f"Unable to configure switch defined in: {file_path}") + except Exception as ex: + logger.error(f"Unable to load switches {ex}") + return switch_dict + + +def load_preselector_from_file(preselector_module, preselector_class, preselector_config_file: Path): + if preselector_config_file is None: + return None + else: + try: + preselector_config = utils.load_from_json(preselector_config_file) + return load_preselector( + preselector_config, preselector_module, preselector_class + ) + except ConfigurationException: + logger.exception( + f"Unable to create preselector defined in: {preselector_config_file}" + ) + return None + + +def load_preselector(preselector_config: str, module: str, preselector_class_name: str, sensor_definition: dict): + logger.debug(f"loading {preselector_class_name} from {module} with config: {preselector_config}") + if module is not None and preselector_class_name is not None: + preselector_module = importlib.import_module(module) + preselector_constructor = getattr(preselector_module, preselector_class_name) + preselector_config = utils.load_from_json(preselector_config) + ps = preselector_constructor(sensor_definition, preselector_config) + register_component_with_status.send(ps, component=ps) + else: + ps = None + return ps + + +def get_sigan_calibration(sigan_cal_file_path: str, default_cal_file_path: str) -> Calibration: + """ + Load signal analyzer calibration data from file. + + :param sigan_cal_file_path: Path to JSON file containing signal + analyzer calibration data. + :param default_cal_file_path: Path to the default cal file. + :return: The signal analyzer ``Calibration`` object. + """ + try: + sigan_cal = None + if sigan_cal_file_path is None or sigan_cal_file_path == "": + logger.warning("No sigan calibration file specified. Not loading calibration file.") + elif not path.exists(sigan_cal_file_path): + logger.warning( + sigan_cal_file_path + " does not exist. Not loading sigan calibration file." + ) + else: + logger.debug(f"Loading sigan cal file: {sigan_cal_file_path}") + default = check_for_default_calibration(sigan_cal_file_path, default_cal_file_path, "Sigan") + sigan_cal = load_from_json(sigan_cal_file_path, default) + sigan_cal.is_default = default + except Exception: + sigan_cal = None + logger.exception("Unable to load sigan calibration data, reverting to none") + return sigan_cal + + +def get_sensor_calibration(sensor_cal_file_path: str, default_cal_file_path: str) -> Calibration: + """ + Load sensor calibration data from file. + + :param sensor_cal_file_path: Path to JSON file containing sensor + calibration data. + :param default_cal_file_path: Name of the default calibration file. + :return: The sensor ``Calibration`` object. + """ + try: + sensor_cal = None + if sensor_cal_file_path is None or sensor_cal_file_path == "": + logger.warning( + "No sensor calibration file specified. Not loading calibration file." + ) + elif not path.exists(sensor_cal_file_path): + logger.warning( + sensor_cal_file_path + + " does not exist. Not loading sensor calibration file." + ) + else: + logger.debug(f"Loading sensor cal file: {sensor_cal_file_path}") + default = check_for_default_calibration( + sensor_cal_file_path, default_cal_file_path, "Sensor" + ) + sensor_cal = load_from_json(sensor_cal_file_path, default) + sensor_cal.is_default = default + except Exception: + sensor_cal = None + logger.exception("Unable to load sensor calibration data, reverting to none") + return sensor_cal + + +def check_for_default_calibration(cal_file_path: str, default_cal_path: str, cal_type: str) -> bool: + default_cal = False + if cal_file_path == default_cal_path: + default_cal = True + logger.warning( + f"***************LOADING DEFAULT {cal_type} CALIBRATION***************" + ) + return default_cal + +env = Env() +switches = load_switches(env("SWITCH_CONFIGS_DIR")) +sensor_cal = get_sensor_calibration(env("SENSOR_CALIBRATION_FILE"), env("DEFAULT_CALIBRATION_FILE")) +sigan_cal = get_sigan_calibration(env("SIGAN_CALIBRATION_FILE"), env("DEFAULT_CALIBRATION_FILE")) +signal_analyzer = None +try: + if not env("RUNNING_MIGRATIONS"): + sigan_module_setting = settings.SIGAN_MODULE + sigan_module = importlib.import_module(sigan_module_setting) + logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) + sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) + signal_analyzer = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal, switches = switches) + else: + logger.info("Running migrations. Not loading signal analyzer.") +except Exception as ex: + logger.warning(f"unable to create signal analyzer: {ex}") + From 0cbd37d8924e04a76575c73c3f4731df3867affc Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 19 Jan 2024 12:31:16 -0700 Subject: [PATCH 189/255] fix sigan_loader --- src/utils/sigan_loader.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/utils/sigan_loader.py b/src/utils/sigan_loader.py index 8d184ab2..420a551b 100644 --- a/src/utils/sigan_loader.py +++ b/src/utils/sigan_loader.py @@ -1,6 +1,5 @@ import importlib import logging -from django.conf import settings from os import path from pathlib import Path from its_preselector.configuration_exception import ConfigurationException @@ -132,16 +131,16 @@ def check_for_default_calibration(cal_file_path: str, default_cal_path: str, cal return default_cal env = Env() -switches = load_switches(env("SWITCH_CONFIGS_DIR")) +switches = load_switches(Path(env("SWITCH_CONFIGS_DIR"))) sensor_cal = get_sensor_calibration(env("SENSOR_CALIBRATION_FILE"), env("DEFAULT_CALIBRATION_FILE")) sigan_cal = get_sigan_calibration(env("SIGAN_CALIBRATION_FILE"), env("DEFAULT_CALIBRATION_FILE")) signal_analyzer = None try: if not env("RUNNING_MIGRATIONS"): - sigan_module_setting = settings.SIGAN_MODULE + sigan_module_setting = env("SIGAN_MODULE") sigan_module = importlib.import_module(sigan_module_setting) - logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) - sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) + logger.info("Creating " + env("SIGAN_CLASS") + " from " + env("SIGAN_MODULE")) + sigan_constructor = getattr(sigan_module, env("SIGAN_CLASS")) signal_analyzer = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal, switches = switches) else: logger.info("Running migrations. Not loading signal analyzer.") From 3f38efaadcaa8e2edf9895ffe24e2c6e93f30f98 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 19 Jan 2024 12:39:23 -0700 Subject: [PATCH 190/255] send signal. --- src/initialization/sensor_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index 214b2579..54a8b8c1 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -54,7 +54,7 @@ def load_sensor(sensor_capabilities): if not settings.RUNNING_MIGRATIONS: sigan_loader = importlib.import_module("utils.sigan_loader") sigan = sigan_loader.signal_analyzer - register_component_with_status(sigan, coponent=sigan) + register_component_with_status.send(sigan, coponent=sigan) # signal.signal(signal.SIGABRT, sigabrt_handler) # sigan_module_setting = settings.SIGAN_MODULE # sigan_module = importlib.import_module(sigan_module_setting) From 6dbf45efd41b9dd45fc03a33a3e1b1021773cde7 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 19 Jan 2024 12:47:12 -0700 Subject: [PATCH 191/255] correct component in signal. --- src/initialization/sensor_loader.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index 54a8b8c1..1da7b510 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -54,7 +54,11 @@ def load_sensor(sensor_capabilities): if not settings.RUNNING_MIGRATIONS: sigan_loader = importlib.import_module("utils.sigan_loader") sigan = sigan_loader.signal_analyzer - register_component_with_status.send(sigan, coponent=sigan) + if sigan: + logger.debug(f"loaded {sigan}") + register_component_with_status.send(sigan, component=sigan) + else: + logger.warning("Sigan loader did not create signal analyzer.") # signal.signal(signal.SIGABRT, sigabrt_handler) # sigan_module_setting = settings.SIGAN_MODULE # sigan_module = importlib.import_module(sigan_module_setting) From 28fdddd3598c156c19590503064f53db6ef67c5b Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 19 Jan 2024 12:55:18 -0700 Subject: [PATCH 192/255] remove preselector loaders from sigan_loader. --- src/utils/sigan_loader.py | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/src/utils/sigan_loader.py b/src/utils/sigan_loader.py index 420a551b..41036d41 100644 --- a/src/utils/sigan_loader.py +++ b/src/utils/sigan_loader.py @@ -31,35 +31,6 @@ def load_switches(switch_dir: Path) -> dict: return switch_dict -def load_preselector_from_file(preselector_module, preselector_class, preselector_config_file: Path): - if preselector_config_file is None: - return None - else: - try: - preselector_config = utils.load_from_json(preselector_config_file) - return load_preselector( - preselector_config, preselector_module, preselector_class - ) - except ConfigurationException: - logger.exception( - f"Unable to create preselector defined in: {preselector_config_file}" - ) - return None - - -def load_preselector(preselector_config: str, module: str, preselector_class_name: str, sensor_definition: dict): - logger.debug(f"loading {preselector_class_name} from {module} with config: {preselector_config}") - if module is not None and preselector_class_name is not None: - preselector_module = importlib.import_module(module) - preselector_constructor = getattr(preselector_module, preselector_class_name) - preselector_config = utils.load_from_json(preselector_config) - ps = preselector_constructor(sensor_definition, preselector_config) - register_component_with_status.send(ps, component=ps) - else: - ps = None - return ps - - def get_sigan_calibration(sigan_cal_file_path: str, default_cal_file_path: str) -> Calibration: """ Load signal analyzer calibration data from file. From 3477638753124b98643da90812fe3c9bdd1121e2 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 19 Jan 2024 13:05:59 -0700 Subject: [PATCH 193/255] move sigan_loader into initialization. --- src/initialization/sensor_loader.py | 3 ++- src/{utils => initialization}/sigan_loader.py | 0 2 files changed, 2 insertions(+), 1 deletion(-) rename src/{utils => initialization}/sigan_loader.py (100%) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index 1da7b510..0b4c2424 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -52,8 +52,9 @@ def load_sensor(sensor_capabilities): sigan = None try: if not settings.RUNNING_MIGRATIONS: - sigan_loader = importlib.import_module("utils.sigan_loader") + sigan_loader = importlib.import_module("sigan_loader") sigan = sigan_loader.signal_analyzer + logger.debug(f"{sigan_loader.sensor_cal}") if sigan: logger.debug(f"loaded {sigan}") register_component_with_status.send(sigan, component=sigan) diff --git a/src/utils/sigan_loader.py b/src/initialization/sigan_loader.py similarity index 100% rename from src/utils/sigan_loader.py rename to src/initialization/sigan_loader.py From 306d05560eef311585fa33088b2ab0b9d21905f0 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 19 Jan 2024 13:10:50 -0700 Subject: [PATCH 194/255] import initialization.sigan_loader. --- src/initialization/sensor_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index 0b4c2424..52c46a8b 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -52,7 +52,7 @@ def load_sensor(sensor_capabilities): sigan = None try: if not settings.RUNNING_MIGRATIONS: - sigan_loader = importlib.import_module("sigan_loader") + sigan_loader = importlib.import_module("initialization.sigan_loader") sigan = sigan_loader.signal_analyzer logger.debug(f"{sigan_loader.sensor_cal}") if sigan: From e0363a9ea41978001e9027dc8418cef2e45a1ba7 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 19 Jan 2024 13:20:24 -0700 Subject: [PATCH 195/255] Set logging to debug in sigan_loader. --- src/initialization/sigan_loader.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/initialization/sigan_loader.py b/src/initialization/sigan_loader.py index 41036d41..854ee8dc 100644 --- a/src/initialization/sigan_loader.py +++ b/src/initialization/sigan_loader.py @@ -8,6 +8,7 @@ from scos_actions.calibration.calibration import Calibration, load_from_json from environs import Env +logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) def load_switches(switch_dir: Path) -> dict: From b2aed98d7bf51ebb8753332408b2ba0b8802bf3a Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 19 Jan 2024 13:33:59 -0700 Subject: [PATCH 196/255] Don't check if running migrations in sigan_loader. --- docker-compose.yml | 1 + src/initialization/sigan_loader.py | 13 +++++-------- src/sensor/settings.py | 1 + 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7d6cda35..29a1daec 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -64,6 +64,7 @@ services: - SIGAN_CLASS - SIGAN_POWER_SWITCH - SIGAN_POWER_CYCLE_STATES + - RUNNING_MIGRATIONS expose: - '8000' volumes: diff --git a/src/initialization/sigan_loader.py b/src/initialization/sigan_loader.py index 854ee8dc..ff7a7285 100644 --- a/src/initialization/sigan_loader.py +++ b/src/initialization/sigan_loader.py @@ -108,14 +108,11 @@ def check_for_default_calibration(cal_file_path: str, default_cal_path: str, cal sigan_cal = get_sigan_calibration(env("SIGAN_CALIBRATION_FILE"), env("DEFAULT_CALIBRATION_FILE")) signal_analyzer = None try: - if not env("RUNNING_MIGRATIONS"): - sigan_module_setting = env("SIGAN_MODULE") - sigan_module = importlib.import_module(sigan_module_setting) - logger.info("Creating " + env("SIGAN_CLASS") + " from " + env("SIGAN_MODULE")) - sigan_constructor = getattr(sigan_module, env("SIGAN_CLASS")) - signal_analyzer = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal, switches = switches) - else: - logger.info("Running migrations. Not loading signal analyzer.") + sigan_module_setting = env("SIGAN_MODULE") + sigan_module = importlib.import_module(sigan_module_setting) + logger.info("Creating " + env("SIGAN_CLASS") + " from " + env("SIGAN_MODULE")) + sigan_constructor = getattr(sigan_module, env("SIGAN_CLASS")) + signal_analyzer = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal, switches = switches) except Exception as ex: logger.warning(f"unable to create signal analyzer: {ex}") diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 67ff386f..e7a6ee00 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -187,6 +187,7 @@ """ RUNNING_MIGRATIONS=env.bool("RUNNING_MIGRATIONS", False) +os.environ["RUNNING_MIGRATIONS"] = RUNNING_MIGRATIONS INSTALLED_APPS = [ "django.contrib.admin", From ae7a369b20fbadbbeb19ab4465fc7bb9cd95eeb1 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 19 Jan 2024 13:38:25 -0700 Subject: [PATCH 197/255] cast to string before setting env var. --- src/sensor/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sensor/settings.py b/src/sensor/settings.py index e7a6ee00..b476d104 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -187,7 +187,7 @@ """ RUNNING_MIGRATIONS=env.bool("RUNNING_MIGRATIONS", False) -os.environ["RUNNING_MIGRATIONS"] = RUNNING_MIGRATIONS +os.environ["RUNNING_MIGRATIONS"] = str(RUNNING_MIGRATIONS) INSTALLED_APPS = [ "django.contrib.admin", From 4ce7ce69be2039fdf3e8bf755e5f610b26d85df1 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 19 Jan 2024 13:57:11 -0700 Subject: [PATCH 198/255] try sigan_initializer --- src/initialization/sigan_initializer.py | 117 ++++++++++++++++++++++++ src/initialization/sigan_loader.py | 112 +---------------------- 2 files changed, 119 insertions(+), 110 deletions(-) create mode 100644 src/initialization/sigan_initializer.py diff --git a/src/initialization/sigan_initializer.py b/src/initialization/sigan_initializer.py new file mode 100644 index 00000000..f08b9756 --- /dev/null +++ b/src/initialization/sigan_initializer.py @@ -0,0 +1,117 @@ +import importlib +import logging +from os import path +from pathlib import Path +from its_preselector.configuration_exception import ConfigurationException +from its_preselector.controlbyweb_web_relay import ControlByWebWebRelay +from scos_actions import utils +from scos_actions.calibration.calibration import Calibration, load_from_json +from environs import Env + +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) + +def load_switches(switch_dir: Path) -> dict: + logger.debug(f"Loading switches in {switch_dir}") + switch_dict = {} + try: + if switch_dir is not None and switch_dir.is_dir(): + for f in switch_dir.iterdir(): + file_path = f.resolve() + logger.debug(f"loading switch config {file_path}") + conf = utils.load_from_json(file_path) + try: + switch = ControlByWebWebRelay(conf) + logger.debug(f"Adding {switch.id}") + switch_dict[switch.id] = switch + logger.debug(f"Registering switch status for {switch.name}") + except ConfigurationException: + logger.error(f"Unable to configure switch defined in: {file_path}") + except Exception as ex: + logger.error(f"Unable to load switches {ex}") + return switch_dict + + +def get_sigan_calibration(sigan_cal_file_path: str, default_cal_file_path: str) -> Calibration: + """ + Load signal analyzer calibration data from file. + + :param sigan_cal_file_path: Path to JSON file containing signal + analyzer calibration data. + :param default_cal_file_path: Path to the default cal file. + :return: The signal analyzer ``Calibration`` object. + """ + try: + sigan_cal = None + if sigan_cal_file_path is None or sigan_cal_file_path == "": + logger.warning("No sigan calibration file specified. Not loading calibration file.") + elif not path.exists(sigan_cal_file_path): + logger.warning( + sigan_cal_file_path + " does not exist. Not loading sigan calibration file." + ) + else: + logger.debug(f"Loading sigan cal file: {sigan_cal_file_path}") + default = check_for_default_calibration(sigan_cal_file_path, default_cal_file_path, "Sigan") + sigan_cal = load_from_json(sigan_cal_file_path, default) + sigan_cal.is_default = default + except Exception: + sigan_cal = None + logger.exception("Unable to load sigan calibration data, reverting to none") + return sigan_cal + + +def get_sensor_calibration(sensor_cal_file_path: str, default_cal_file_path: str) -> Calibration: + """ + Load sensor calibration data from file. + + :param sensor_cal_file_path: Path to JSON file containing sensor + calibration data. + :param default_cal_file_path: Name of the default calibration file. + :return: The sensor ``Calibration`` object. + """ + try: + sensor_cal = None + if sensor_cal_file_path is None or sensor_cal_file_path == "": + logger.warning( + "No sensor calibration file specified. Not loading calibration file." + ) + elif not path.exists(sensor_cal_file_path): + logger.warning( + sensor_cal_file_path + + " does not exist. Not loading sensor calibration file." + ) + else: + logger.debug(f"Loading sensor cal file: {sensor_cal_file_path}") + default = check_for_default_calibration( + sensor_cal_file_path, default_cal_file_path, "Sensor" + ) + sensor_cal = load_from_json(sensor_cal_file_path, default) + sensor_cal.is_default = default + except Exception: + sensor_cal = None + logger.exception("Unable to load sensor calibration data, reverting to none") + return sensor_cal + + +def check_for_default_calibration(cal_file_path: str, default_cal_path: str, cal_type: str) -> bool: + default_cal = False + if cal_file_path == default_cal_path: + default_cal = True + logger.warning( + f"***************LOADING DEFAULT {cal_type} CALIBRATION***************" + ) + return default_cal + +env = Env() +switches = load_switches(Path(env("SWITCH_CONFIGS_DIR"))) +sensor_cal = get_sensor_calibration(env("SENSOR_CALIBRATION_FILE"), env("DEFAULT_CALIBRATION_FILE")) +sigan_cal = get_sigan_calibration(env("SIGAN_CALIBRATION_FILE"), env("DEFAULT_CALIBRATION_FILE")) +sigan = None +try: + sigan_module_setting = env("SIGAN_MODULE") + sigan_module = importlib.import_module(sigan_module_setting) + logger.info("Creating " + env("SIGAN_CLASS") + " from " + env("SIGAN_MODULE")) + sigan_constructor = getattr(sigan_module, env("SIGAN_CLASS")) + sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal, switches = switches) +except Exception as ex: + logger.warning(f"unable to create signal analyzer: {ex}") \ No newline at end of file diff --git a/src/initialization/sigan_loader.py b/src/initialization/sigan_loader.py index ff7a7285..3bc0611d 100644 --- a/src/initialization/sigan_loader.py +++ b/src/initialization/sigan_loader.py @@ -1,118 +1,10 @@ -import importlib import logging -from os import path -from pathlib import Path -from its_preselector.configuration_exception import ConfigurationException -from its_preselector.controlbyweb_web_relay import ControlByWebWebRelay -from scos_actions import utils -from scos_actions.calibration.calibration import Calibration, load_from_json -from environs import Env +from sigan_initializer import sigan logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) -def load_switches(switch_dir: Path) -> dict: - logger.debug(f"Loading switches in {switch_dir}") - switch_dict = {} - try: - if switch_dir is not None and switch_dir.is_dir(): - for f in switch_dir.iterdir(): - file_path = f.resolve() - logger.debug(f"loading switch config {file_path}") - conf = utils.load_from_json(file_path) - try: - switch = ControlByWebWebRelay(conf) - logger.debug(f"Adding {switch.id}") - switch_dict[switch.id] = switch - logger.debug(f"Registering switch status for {switch.name}") - except ConfigurationException: - logger.error(f"Unable to configure switch defined in: {file_path}") - except Exception as ex: - logger.error(f"Unable to load switches {ex}") - return switch_dict +signal_analyzer = sigan -def get_sigan_calibration(sigan_cal_file_path: str, default_cal_file_path: str) -> Calibration: - """ - Load signal analyzer calibration data from file. - - :param sigan_cal_file_path: Path to JSON file containing signal - analyzer calibration data. - :param default_cal_file_path: Path to the default cal file. - :return: The signal analyzer ``Calibration`` object. - """ - try: - sigan_cal = None - if sigan_cal_file_path is None or sigan_cal_file_path == "": - logger.warning("No sigan calibration file specified. Not loading calibration file.") - elif not path.exists(sigan_cal_file_path): - logger.warning( - sigan_cal_file_path + " does not exist. Not loading sigan calibration file." - ) - else: - logger.debug(f"Loading sigan cal file: {sigan_cal_file_path}") - default = check_for_default_calibration(sigan_cal_file_path, default_cal_file_path, "Sigan") - sigan_cal = load_from_json(sigan_cal_file_path, default) - sigan_cal.is_default = default - except Exception: - sigan_cal = None - logger.exception("Unable to load sigan calibration data, reverting to none") - return sigan_cal - - -def get_sensor_calibration(sensor_cal_file_path: str, default_cal_file_path: str) -> Calibration: - """ - Load sensor calibration data from file. - - :param sensor_cal_file_path: Path to JSON file containing sensor - calibration data. - :param default_cal_file_path: Name of the default calibration file. - :return: The sensor ``Calibration`` object. - """ - try: - sensor_cal = None - if sensor_cal_file_path is None or sensor_cal_file_path == "": - logger.warning( - "No sensor calibration file specified. Not loading calibration file." - ) - elif not path.exists(sensor_cal_file_path): - logger.warning( - sensor_cal_file_path - + " does not exist. Not loading sensor calibration file." - ) - else: - logger.debug(f"Loading sensor cal file: {sensor_cal_file_path}") - default = check_for_default_calibration( - sensor_cal_file_path, default_cal_file_path, "Sensor" - ) - sensor_cal = load_from_json(sensor_cal_file_path, default) - sensor_cal.is_default = default - except Exception: - sensor_cal = None - logger.exception("Unable to load sensor calibration data, reverting to none") - return sensor_cal - - -def check_for_default_calibration(cal_file_path: str, default_cal_path: str, cal_type: str) -> bool: - default_cal = False - if cal_file_path == default_cal_path: - default_cal = True - logger.warning( - f"***************LOADING DEFAULT {cal_type} CALIBRATION***************" - ) - return default_cal - -env = Env() -switches = load_switches(Path(env("SWITCH_CONFIGS_DIR"))) -sensor_cal = get_sensor_calibration(env("SENSOR_CALIBRATION_FILE"), env("DEFAULT_CALIBRATION_FILE")) -sigan_cal = get_sigan_calibration(env("SIGAN_CALIBRATION_FILE"), env("DEFAULT_CALIBRATION_FILE")) -signal_analyzer = None -try: - sigan_module_setting = env("SIGAN_MODULE") - sigan_module = importlib.import_module(sigan_module_setting) - logger.info("Creating " + env("SIGAN_CLASS") + " from " + env("SIGAN_MODULE")) - sigan_constructor = getattr(sigan_module, env("SIGAN_CLASS")) - signal_analyzer = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal, switches = switches) -except Exception as ex: - logger.warning(f"unable to create signal analyzer: {ex}") From b8a542a059bb8e4f675a9ad113652908ee549bce Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 19 Jan 2024 14:37:00 -0700 Subject: [PATCH 199/255] fix import --- src/initialization/sigan_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/initialization/sigan_loader.py b/src/initialization/sigan_loader.py index 3bc0611d..eca476ae 100644 --- a/src/initialization/sigan_loader.py +++ b/src/initialization/sigan_loader.py @@ -1,5 +1,5 @@ import logging -from sigan_initializer import sigan +from .sigan_initializer import sigan logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) From 211154f52ac93a7d2cb5c76d1ce0ad1d4db2f071 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 19 Jan 2024 20:15:28 -0700 Subject: [PATCH 200/255] Remove sigan_loader and sigan_initializer and create sigan directly in sensor_loader. --- src/initialization/sensor_loader.py | 22 ++--- src/initialization/sigan_initializer.py | 117 ------------------------ src/initialization/sigan_loader.py | 10 -- 3 files changed, 7 insertions(+), 142 deletions(-) delete mode 100644 src/initialization/sigan_initializer.py delete mode 100644 src/initialization/sigan_loader.py diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index 52c46a8b..9bf9d01c 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -52,21 +52,13 @@ def load_sensor(sensor_capabilities): sigan = None try: if not settings.RUNNING_MIGRATIONS: - sigan_loader = importlib.import_module("initialization.sigan_loader") - sigan = sigan_loader.signal_analyzer - logger.debug(f"{sigan_loader.sensor_cal}") - if sigan: - logger.debug(f"loaded {sigan}") - register_component_with_status.send(sigan, component=sigan) - else: - logger.warning("Sigan loader did not create signal analyzer.") - # signal.signal(signal.SIGABRT, sigabrt_handler) - # sigan_module_setting = settings.SIGAN_MODULE - # sigan_module = importlib.import_module(sigan_module_setting) - # logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) - # sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) - # sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal, switches = switches) - # register_component_with_status.send(sigan, component=sigan) + signal.signal(signal.SIGABRT, sigabrt_handler) + sigan_module_setting = settings.SIGAN_MODULE + sigan_module = importlib.import_module(sigan_module_setting) + logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) + sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) + sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal, switches = switches) + register_component_with_status.send(sigan, component=sigan) else: logger.info("Running migrations. Not loading signal analyzer.") except Exception as ex: diff --git a/src/initialization/sigan_initializer.py b/src/initialization/sigan_initializer.py deleted file mode 100644 index f08b9756..00000000 --- a/src/initialization/sigan_initializer.py +++ /dev/null @@ -1,117 +0,0 @@ -import importlib -import logging -from os import path -from pathlib import Path -from its_preselector.configuration_exception import ConfigurationException -from its_preselector.controlbyweb_web_relay import ControlByWebWebRelay -from scos_actions import utils -from scos_actions.calibration.calibration import Calibration, load_from_json -from environs import Env - -logging.basicConfig(level=logging.DEBUG) -logger = logging.getLogger(__name__) - -def load_switches(switch_dir: Path) -> dict: - logger.debug(f"Loading switches in {switch_dir}") - switch_dict = {} - try: - if switch_dir is not None and switch_dir.is_dir(): - for f in switch_dir.iterdir(): - file_path = f.resolve() - logger.debug(f"loading switch config {file_path}") - conf = utils.load_from_json(file_path) - try: - switch = ControlByWebWebRelay(conf) - logger.debug(f"Adding {switch.id}") - switch_dict[switch.id] = switch - logger.debug(f"Registering switch status for {switch.name}") - except ConfigurationException: - logger.error(f"Unable to configure switch defined in: {file_path}") - except Exception as ex: - logger.error(f"Unable to load switches {ex}") - return switch_dict - - -def get_sigan_calibration(sigan_cal_file_path: str, default_cal_file_path: str) -> Calibration: - """ - Load signal analyzer calibration data from file. - - :param sigan_cal_file_path: Path to JSON file containing signal - analyzer calibration data. - :param default_cal_file_path: Path to the default cal file. - :return: The signal analyzer ``Calibration`` object. - """ - try: - sigan_cal = None - if sigan_cal_file_path is None or sigan_cal_file_path == "": - logger.warning("No sigan calibration file specified. Not loading calibration file.") - elif not path.exists(sigan_cal_file_path): - logger.warning( - sigan_cal_file_path + " does not exist. Not loading sigan calibration file." - ) - else: - logger.debug(f"Loading sigan cal file: {sigan_cal_file_path}") - default = check_for_default_calibration(sigan_cal_file_path, default_cal_file_path, "Sigan") - sigan_cal = load_from_json(sigan_cal_file_path, default) - sigan_cal.is_default = default - except Exception: - sigan_cal = None - logger.exception("Unable to load sigan calibration data, reverting to none") - return sigan_cal - - -def get_sensor_calibration(sensor_cal_file_path: str, default_cal_file_path: str) -> Calibration: - """ - Load sensor calibration data from file. - - :param sensor_cal_file_path: Path to JSON file containing sensor - calibration data. - :param default_cal_file_path: Name of the default calibration file. - :return: The sensor ``Calibration`` object. - """ - try: - sensor_cal = None - if sensor_cal_file_path is None or sensor_cal_file_path == "": - logger.warning( - "No sensor calibration file specified. Not loading calibration file." - ) - elif not path.exists(sensor_cal_file_path): - logger.warning( - sensor_cal_file_path - + " does not exist. Not loading sensor calibration file." - ) - else: - logger.debug(f"Loading sensor cal file: {sensor_cal_file_path}") - default = check_for_default_calibration( - sensor_cal_file_path, default_cal_file_path, "Sensor" - ) - sensor_cal = load_from_json(sensor_cal_file_path, default) - sensor_cal.is_default = default - except Exception: - sensor_cal = None - logger.exception("Unable to load sensor calibration data, reverting to none") - return sensor_cal - - -def check_for_default_calibration(cal_file_path: str, default_cal_path: str, cal_type: str) -> bool: - default_cal = False - if cal_file_path == default_cal_path: - default_cal = True - logger.warning( - f"***************LOADING DEFAULT {cal_type} CALIBRATION***************" - ) - return default_cal - -env = Env() -switches = load_switches(Path(env("SWITCH_CONFIGS_DIR"))) -sensor_cal = get_sensor_calibration(env("SENSOR_CALIBRATION_FILE"), env("DEFAULT_CALIBRATION_FILE")) -sigan_cal = get_sigan_calibration(env("SIGAN_CALIBRATION_FILE"), env("DEFAULT_CALIBRATION_FILE")) -sigan = None -try: - sigan_module_setting = env("SIGAN_MODULE") - sigan_module = importlib.import_module(sigan_module_setting) - logger.info("Creating " + env("SIGAN_CLASS") + " from " + env("SIGAN_MODULE")) - sigan_constructor = getattr(sigan_module, env("SIGAN_CLASS")) - sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal, switches = switches) -except Exception as ex: - logger.warning(f"unable to create signal analyzer: {ex}") \ No newline at end of file diff --git a/src/initialization/sigan_loader.py b/src/initialization/sigan_loader.py deleted file mode 100644 index eca476ae..00000000 --- a/src/initialization/sigan_loader.py +++ /dev/null @@ -1,10 +0,0 @@ -import logging -from .sigan_initializer import sigan - -logging.basicConfig(level=logging.DEBUG) -logger = logging.getLogger(__name__) - - -signal_analyzer = sigan - - From 2ed9a1fd672826b6cec125c6cbbd2e46a6719704 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 19 Jan 2024 20:24:40 -0700 Subject: [PATCH 201/255] remove registration for abort signal. --- src/initialization/sensor_loader.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index 9bf9d01c..538ec101 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -31,9 +31,6 @@ def __new__(cls, sensor_capabilities): cls._instance = super(SensorLoader, cls).__new__(cls) return cls._instance -def sigabrt_handler(signum, frame): - logger.error("Recieved SIGABRT") - def load_sensor(sensor_capabilities): location = None #Remove location from sensor definition and convert to geojson. @@ -52,7 +49,6 @@ def load_sensor(sensor_capabilities): sigan = None try: if not settings.RUNNING_MIGRATIONS: - signal.signal(signal.SIGABRT, sigabrt_handler) sigan_module_setting = settings.SIGAN_MODULE sigan_module = importlib.import_module(sigan_module_setting) logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) From 0c0510d0d324ad89005d84401b0b12ba36dc9f3c Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 22 Jan 2024 07:24:52 -0700 Subject: [PATCH 202/255] test triggering container restart. --- src/initialization/__init__.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index ece51a20..28f5d93a 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -1,5 +1,6 @@ import logging - +from pathlib import Path +from django.conf import settings from .action_loader import ActionLoader from .capabilities_loader import CapabilitiesLoader from .sensor_loader import SensorLoader @@ -19,7 +20,12 @@ def status_registration_handler(sender, **kwargs): logger.exception("Error registering status component") try: register_component_with_status.connect(status_registration_handler) - + logger.debug("Checking for /dev/bus/usb/002/003") + usb = Path("/dev/bus/usb/002/003") + if not usb.exists(): + logger.debug("Usb is not ready. Marking container as unhealthy") + if settings.IN_DOCKER: + Path(settings.SDR_HEALTHCHECK_FILE).touch() action_loader = ActionLoader() logger.debug("test") logger.debug(f"Actions ActionLoader has {len(action_loader.actions)} actions") From aec3311206a8582f184a97be2b0e212cee04b200 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 22 Jan 2024 08:07:07 -0700 Subject: [PATCH 203/255] Configure logging for initialization app. --- src/sensor/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sensor/settings.py b/src/sensor/settings.py index b476d104..c32dd343 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -398,6 +398,7 @@ "authentication": {"handlers": ["console"], "level": LOGLEVEL}, "capabilities": {"handlers": ["console"], "level": LOGLEVEL}, "handlers": {"handlers": ["console"], "level": LOGLEVEL}, + "initialization": {"handlers": ["console"], "level": LOGLEVEL} , "schedule": {"handlers": ["console"], "level": LOGLEVEL}, "scheduler": {"handlers": ["console"], "level": LOGLEVEL}, "sensor": {"handlers": ["console"], "level": LOGLEVEL}, From 734c96dcb40c7b1943a927aeafb85bdcb8c84dba Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 22 Jan 2024 08:07:58 -0700 Subject: [PATCH 204/255] Remove actions logging config. --- src/sensor/settings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sensor/settings.py b/src/sensor/settings.py index c32dd343..0f0eb8f8 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -394,7 +394,6 @@ "filters": {"require_debug_true": {"()": "django.utils.log.RequireDebugTrue"}}, "handlers": {"console": {"class": "logging.StreamHandler", "formatter": "simple"}}, "loggers": { - "actions": {"handlers": ["console"], "level": LOGLEVEL}, "authentication": {"handlers": ["console"], "level": LOGLEVEL}, "capabilities": {"handlers": ["console"], "level": LOGLEVEL}, "handlers": {"handlers": ["console"], "level": LOGLEVEL}, From ddbe4d0ad30b161248fca9bd6d3e49f1a2f6a4ee Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 22 Jan 2024 08:18:11 -0700 Subject: [PATCH 205/255] Set start_time in status when sigan is not initialized. --- src/status/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/status/__init__.py b/src/status/__init__.py index 0e2d5c5e..5409aa3e 100644 --- a/src/status/__init__.py +++ b/src/status/__init__.py @@ -1,7 +1,10 @@ +import datetime import logging from initialization import sensor_loader logger = logging.getLogger(__name__) logger.debug("********** Initializing status **********") -if sensor_loader.sensor is not None: +if sensor_loader.sigan is not None: start_time = sensor_loader.sensor.start_time +else: + start_time = datetime.datetime.utcnow() \ No newline at end of file From 657fdae95f8fc3145668f3517688bce76ae3a53d Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 22 Jan 2024 08:49:07 -0700 Subject: [PATCH 206/255] correct sigan -> signal_analyzer. --- src/status/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/status/__init__.py b/src/status/__init__.py index 5409aa3e..72feff7e 100644 --- a/src/status/__init__.py +++ b/src/status/__init__.py @@ -4,7 +4,7 @@ logger = logging.getLogger(__name__) logger.debug("********** Initializing status **********") -if sensor_loader.sigan is not None: +if sensor_loader.signal_analyzer is not None: start_time = sensor_loader.sensor.start_time else: start_time = datetime.datetime.utcnow() \ No newline at end of file From 3cf56329daff8750fecd4c5a408cf4cd6a9d0cbd Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 22 Jan 2024 08:50:33 -0700 Subject: [PATCH 207/255] correct sensor_loader.sensor.signal_analyzer --- src/status/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/status/__init__.py b/src/status/__init__.py index 72feff7e..65d0294d 100644 --- a/src/status/__init__.py +++ b/src/status/__init__.py @@ -4,7 +4,7 @@ logger = logging.getLogger(__name__) logger.debug("********** Initializing status **********") -if sensor_loader.signal_analyzer is not None: +if sensor_loader.sensor.signal_analyzer is not None: start_time = sensor_loader.sensor.start_time else: start_time = datetime.datetime.utcnow() \ No newline at end of file From 28ec3fd2ec8de939bddb080fcfd9793b4188d4e9 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 22 Jan 2024 09:00:21 -0700 Subject: [PATCH 208/255] Add USB_PATH setting and don't set RUNNING_MIGRATIONS to false until after create super user. --- docker-compose.yml | 1 + entrypoints/api_entrypoint.sh | 2 +- src/initialization/__init__.py | 17 ++++++++++++----- src/sensor/settings.py | 2 +- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 29a1daec..7af23fad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -65,6 +65,7 @@ services: - SIGAN_POWER_SWITCH - SIGAN_POWER_CYCLE_STATES - RUNNING_MIGRATIONS + - USB_PATH expose: - '8000' volumes: diff --git a/entrypoints/api_entrypoint.sh b/entrypoints/api_entrypoint.sh index f87d7d53..983a567d 100644 --- a/entrypoints/api_entrypoint.sh +++ b/entrypoints/api_entrypoint.sh @@ -14,9 +14,9 @@ RUNNING_MIGRATIONS="True" export RUNNING_MIGRATIONS echo "Starting Migrations" python3.8 manage.py migrate -RUNNING_MIGRATIONS="False" echo "Creating superuser (if managed)" python3.8 /scripts/create_superuser.py +RUNNING_MIGRATIONS="False" echo "Starting Gunicorn" exec gunicorn sensor.wsgi -c ../gunicorn/config.py & wait diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 28f5d93a..a1905419 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -12,6 +12,17 @@ status_monitor = StatusMonitor() +def check_for_usb(): + logger.debug("Checking for USB...") + if settings.USB_PATH is not None: + usb = settings.USB_PATH + if not usb.exists(): + logger.debug("Usb is not ready. Marking container as unhealthy") + if settings.IN_DOCKER: + Path(settings.SDR_HEALTHCHECK_FILE).touch() + else: + logger.debug("Found USB") + def status_registration_handler(sender, **kwargs): try: logger.debug(f"Registering {sender} as status provider") @@ -21,11 +32,7 @@ def status_registration_handler(sender, **kwargs): try: register_component_with_status.connect(status_registration_handler) logger.debug("Checking for /dev/bus/usb/002/003") - usb = Path("/dev/bus/usb/002/003") - if not usb.exists(): - logger.debug("Usb is not ready. Marking container as unhealthy") - if settings.IN_DOCKER: - Path(settings.SDR_HEALTHCHECK_FILE).touch() + check_for_usb() action_loader = ActionLoader() logger.debug("test") logger.debug(f"Actions ActionLoader has {len(action_loader.actions)} actions") diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 0f0eb8f8..3f23f728 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -453,7 +453,7 @@ SIGAN_POWER_SWITCH = env("SIGAN_POWER_SWITCH", default=None) MAX_FAILURES = env("MAX_FAILURES", default=2) os.environ["RUNNING_TESTS"] = str(RUNNING_TESTS) - +USB_PATH=env("USB_PATH", default=None) From defdc781dd31315462a4b720828095ee69731497 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 22 Jan 2024 09:56:40 -0700 Subject: [PATCH 209/255] Use an app for user initialization instead of script to avoid setting up django again. --- entrypoints/api_entrypoint.sh | 2 -- src/sensor/settings.py | 1 + scripts/create_superuser.py => src/users/__init__.py | 5 +---- src/users/apps.py | 5 +++++ 4 files changed, 7 insertions(+), 6 deletions(-) rename scripts/create_superuser.py => src/users/__init__.py (95%) mode change 100755 => 100644 create mode 100644 src/users/apps.py diff --git a/entrypoints/api_entrypoint.sh b/entrypoints/api_entrypoint.sh index 983a567d..5d80d15a 100644 --- a/entrypoints/api_entrypoint.sh +++ b/entrypoints/api_entrypoint.sh @@ -14,8 +14,6 @@ RUNNING_MIGRATIONS="True" export RUNNING_MIGRATIONS echo "Starting Migrations" python3.8 manage.py migrate -echo "Creating superuser (if managed)" -python3.8 /scripts/create_superuser.py RUNNING_MIGRATIONS="False" echo "Starting Gunicorn" exec gunicorn sensor.wsgi -c ../gunicorn/config.py & diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 2e46471b..2048e804 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -208,6 +208,7 @@ "scheduler.apps.SchedulerConfig", "status.apps.StatusConfig", "sensor.apps.SensorConfig", # global settings/utils, etc + "users.apps.UsersConfig", ] MIDDLEWARE = [ diff --git a/scripts/create_superuser.py b/src/users/__init__.py old mode 100755 new mode 100644 similarity index 95% rename from scripts/create_superuser.py rename to src/users/__init__.py index 522825b0..ac78069a --- a/scripts/create_superuser.py +++ b/src/users/__init__.py @@ -3,15 +3,12 @@ import os import sys -import django from django.contrib.auth import get_user_model # noqa -PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "src") +PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../..", "src") sys.path.append(PATH) -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "sensor.settings") -django.setup() UserModel = get_user_model() diff --git a/src/users/apps.py b/src/users/apps.py new file mode 100644 index 00000000..3ef1284a --- /dev/null +++ b/src/users/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class UsersConfig(AppConfig): + name = "users" From fb9a84a8affe0389df5704d0d786de3cdd12936f Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 22 Jan 2024 10:00:14 -0700 Subject: [PATCH 210/255] don't copy super user script. --- docker/Dockerfile-api | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker/Dockerfile-api b/docker/Dockerfile-api index ee69f497..858a04a5 100644 --- a/docker/Dockerfile-api +++ b/docker/Dockerfile-api @@ -27,9 +27,6 @@ COPY ./gunicorn /gunicorn RUN mkdir -p /entrypoints COPY ./entrypoints/api_entrypoint.sh /entrypoints -RUN mkdir -p /scripts -COPY ./scripts/create_superuser.py /scripts - RUN chmod +x /entrypoints/api_entrypoint.sh COPY ./configs /configs From 1311ca6e6f83f6096430e7a15ac7888b3d7db4e1 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 22 Jan 2024 10:04:55 -0700 Subject: [PATCH 211/255] convert usb_path to Path. --- src/initialization/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index a1905419..290f8075 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -12,10 +12,11 @@ status_monitor = StatusMonitor() + def check_for_usb(): logger.debug("Checking for USB...") if settings.USB_PATH is not None: - usb = settings.USB_PATH + usb = Path(settings.USB_PATH) if not usb.exists(): logger.debug("Usb is not ready. Marking container as unhealthy") if settings.IN_DOCKER: @@ -23,12 +24,15 @@ def check_for_usb(): else: logger.debug("Found USB") + def status_registration_handler(sender, **kwargs): try: logger.debug(f"Registering {sender} as status provider") status_monitor.add_component(kwargs["component"]) except: logger.exception("Error registering status component") + + try: register_component_with_status.connect(status_registration_handler) logger.debug("Checking for /dev/bus/usb/002/003") From d67bfebc919086e8d52d031bf8464c6d912fbece Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 22 Jan 2024 10:18:07 -0700 Subject: [PATCH 212/255] move adding of users into ready method. --- src/users/__init__.py | 64 --------------------------------------- src/users/apps.py | 69 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 64 deletions(-) diff --git a/src/users/__init__.py b/src/users/__init__.py index ac78069a..7eb80757 100644 --- a/src/users/__init__.py +++ b/src/users/__init__.py @@ -1,70 +1,6 @@ #!/usr/bin/env python3 -import os -import sys - -from django.contrib.auth import get_user_model # noqa PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../..", "src") sys.path.append(PATH) - - -UserModel = get_user_model() - - -def add_user(username, password, email=None): - try: - admin_user = UserModel._default_manager.get(username=username) - if email: - admin_user.email = email - admin_user.set_password(password) - admin_user.save() - print("Reset admin account password and email from environment") - except UserModel.DoesNotExist: - UserModel._default_manager.create_superuser(username, email, password) - print("Created admin account with password and email from environment") - - -try: - password = os.environ["ADMIN_PASSWORD"] - print("Retreived admin password from environment variable ADMIN_PASSWORD") - email = os.environ["ADMIN_EMAIL"] - print("Retreived admin email from environment variable ADMIN_EMAIL") - username = os.environ["ADMIN_NAME"] - print("Retreived admin name from environment variable ADMIN_NAME") - add_user(username.strip(), password.strip(), email.strip()) -except KeyError: - print("Not on a managed sensor, so not auto-generating admin account.") - print("You can add an admin later with `./manage.py createsuperuser`") - sys.exit(0) - -additional_user_names = "" -additional_user_password = "" -try: - additional_user_names = os.environ["ADDITIONAL_USER_NAMES"] - print( - "Retreived additional user names from environment variable ADDITIONAL_USER_NAMES" - ) - if ( - "ADDITIONAL_USER_PASSWORD" in os.environ - and os.environ["ADDITIONAL_USER_PASSWORD"] - ): - additional_user_password = os.environ["ADDITIONAL_USER_PASSWORD"].strip() - else: - # user will have unusable password - # https://docs.djangoproject.com/en/3.2/ref/contrib/auth/#django.contrib.auth.models.UserManager.create_user - additional_user_password = None - print( - "Retreived additional user password from environment variable ADDITIONAL_USER_PASSWORD" - ) -except KeyError: - print("Not creating any additonal users.") - - -if additional_user_names != "" and additional_user_password != "": - if "," in additional_user_names: - for additional_user_name in additional_user_names.split(","): - add_user(additional_user_name.strip(), additional_user_password) - else: - add_user(additional_user_names.strip(), additional_user_password) diff --git a/src/users/apps.py b/src/users/apps.py index 3ef1284a..81286392 100644 --- a/src/users/apps.py +++ b/src/users/apps.py @@ -1,5 +1,74 @@ from django.apps import AppConfig +import os +import sys + +from django.contrib.auth import get_user_model # noqa class UsersConfig(AppConfig): name = "users" + + def add_user(self, user_model, username, password, email=None): + try: + admin_user = user_model._default_manager.get(username=username) + if email: + admin_user.email = email + admin_user.set_password(password) + admin_user.save() + print("Reset admin account password and email from environment") + except user_model.DoesNotExist: + user_model._default_manager.create_superuser(username, email, password) + print("Created admin account with password and email from environment") + + def ready(self): + UserModel = get_user_model() + + try: + password = os.environ["ADMIN_PASSWORD"] + print("Retreived admin password from environment variable ADMIN_PASSWORD") + email = os.environ["ADMIN_EMAIL"] + print("Retreived admin email from environment variable ADMIN_EMAIL") + username = os.environ["ADMIN_NAME"] + print("Retreived admin name from environment variable ADMIN_NAME") + self.add_user(UserModel, username.strip(), password.strip(), email.strip()) + except KeyError: + print("Not on a managed sensor, so not auto-generating admin account.") + print("You can add an admin later with `./manage.py createsuperuser`") + sys.exit(0) + + additional_user_names = "" + additional_user_password = "" + try: + additional_user_names = os.environ["ADDITIONAL_USER_NAMES"] + print( + "Retreived additional user names from environment variable ADDITIONAL_USER_NAMES" + ) + if ( + "ADDITIONAL_USER_PASSWORD" in os.environ + and os.environ["ADDITIONAL_USER_PASSWORD"] + ): + additional_user_password = os.environ[ + "ADDITIONAL_USER_PASSWORD" + ].strip() + else: + # user will have unusable password + # https://docs.djangoproject.com/en/3.2/ref/contrib/auth/#django.contrib.auth.models.UserManager.create_user + additional_user_password = None + print( + "Retreived additional user password from environment variable ADDITIONAL_USER_PASSWORD" + ) + except KeyError: + print("Not creating any additonal users.") + + if additional_user_names != "" and additional_user_password != "": + if "," in additional_user_names: + for additional_user_name in additional_user_names.split(","): + self.add_user( + UserModel, + additional_user_name.strip(), + additional_user_password, + ) + else: + self.add_user( + UserModel, additional_user_names.strip(), additional_user_password + ) From 9a90ff7fe20bf45933c8e572ea21e6d13b80a3aa Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 22 Jan 2024 10:21:12 -0700 Subject: [PATCH 213/255] remove dead code. --- src/users/__init__.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/users/__init__.py b/src/users/__init__.py index 7eb80757..e69de29b 100644 --- a/src/users/__init__.py +++ b/src/users/__init__.py @@ -1,6 +0,0 @@ -#!/usr/bin/env python3 - - -PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../..", "src") - -sys.path.append(PATH) From d743c181de65fb05a3a9ebd9a1823730c213176a Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 22 Jan 2024 10:37:11 -0700 Subject: [PATCH 214/255] use logging in users app. --- src/sensor/settings.py | 1 + src/users/apps.py | 110 ++++++++++++++++++++++++----------------- 2 files changed, 65 insertions(+), 46 deletions(-) diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 2048e804..109967cb 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -405,6 +405,7 @@ "scos_usrp": {"handlers": ["console"], "level": LOGLEVEL}, "scos_sensor_keysight": {"handlers": ["console"], "level": LOGLEVEL}, "scos_tekrsa": {"handlers": ["console"], "level": LOGLEVEL}, + "users": {"handlers": ["console"], "level": LOGLEVEL}, }, } diff --git a/src/users/apps.py b/src/users/apps.py index 81286392..81d9339e 100644 --- a/src/users/apps.py +++ b/src/users/apps.py @@ -1,9 +1,13 @@ -from django.apps import AppConfig +import logging import os import sys +from django.apps import AppConfig +from django.conf import settings from django.contrib.auth import get_user_model # noqa +logger = logging.getLogger(__name__) + class UsersConfig(AppConfig): name = "users" @@ -15,60 +19,74 @@ def add_user(self, user_model, username, password, email=None): admin_user.email = email admin_user.set_password(password) admin_user.save() - print("Reset admin account password and email from environment") + logger.debug("Reset admin account password and email from environment") except user_model.DoesNotExist: user_model._default_manager.create_superuser(username, email, password) print("Created admin account with password and email from environment") def ready(self): - UserModel = get_user_model() + if not settings.RUNNING_MIGRATIONS: + UserModel = get_user_model() + try: + password = os.environ["ADMIN_PASSWORD"] + logger.debug( + "Retreived admin password from environment variable ADMIN_PASSWORD" + ) + email = os.environ["ADMIN_EMAIL"] + logger.debug( + "Retreived admin email from environment variable ADMIN_EMAIL" + ) + username = os.environ["ADMIN_NAME"] + logger.debug( + "Retreived admin name from environment variable ADMIN_NAME" + ) + self.add_user( + UserModel, username.strip(), password.strip(), email.strip() + ) + except KeyError: + logger.warning( + "Not on a managed sensor, so not auto-generating admin account." + ) + logger.warning( + "You can add an admin later with `./manage.py createsuperuser`" + ) - try: - password = os.environ["ADMIN_PASSWORD"] - print("Retreived admin password from environment variable ADMIN_PASSWORD") - email = os.environ["ADMIN_EMAIL"] - print("Retreived admin email from environment variable ADMIN_EMAIL") - username = os.environ["ADMIN_NAME"] - print("Retreived admin name from environment variable ADMIN_NAME") - self.add_user(UserModel, username.strip(), password.strip(), email.strip()) - except KeyError: - print("Not on a managed sensor, so not auto-generating admin account.") - print("You can add an admin later with `./manage.py createsuperuser`") - sys.exit(0) + additional_user_names = "" + additional_user_password = "" + try: + additional_user_names = os.environ["ADDITIONAL_USER_NAMES"] + print( + "Retreived additional user names from environment variable ADDITIONAL_USER_NAMES" + ) + if ( + "ADDITIONAL_USER_PASSWORD" in os.environ + and os.environ["ADDITIONAL_USER_PASSWORD"] + ): + logger.debug( + "Retreived additional user password from environment variable ADDITIONAL_USER_PASSWORD" + ) + additional_user_password = os.environ[ + "ADDITIONAL_USER_PASSWORD" + ].strip() + else: + # user will have unusable password + # https://docs.djangoproject.com/en/3.2/ref/contrib/auth/#django.contrib.auth.models.UserManager.create_user + additional_user_password = None - additional_user_names = "" - additional_user_password = "" - try: - additional_user_names = os.environ["ADDITIONAL_USER_NAMES"] - print( - "Retreived additional user names from environment variable ADDITIONAL_USER_NAMES" - ) - if ( - "ADDITIONAL_USER_PASSWORD" in os.environ - and os.environ["ADDITIONAL_USER_PASSWORD"] - ): - additional_user_password = os.environ[ - "ADDITIONAL_USER_PASSWORD" - ].strip() - else: - # user will have unusable password - # https://docs.djangoproject.com/en/3.2/ref/contrib/auth/#django.contrib.auth.models.UserManager.create_user - additional_user_password = None - print( - "Retreived additional user password from environment variable ADDITIONAL_USER_PASSWORD" - ) - except KeyError: - print("Not creating any additonal users.") + except KeyError: + logger.warning("Not creating any additonal users.") - if additional_user_names != "" and additional_user_password != "": - if "," in additional_user_names: - for additional_user_name in additional_user_names.split(","): + if additional_user_names != "" and additional_user_password != "": + if "," in additional_user_names: + for additional_user_name in additional_user_names.split(","): + self.add_user( + UserModel, + additional_user_name.strip(), + additional_user_password, + ) + else: self.add_user( UserModel, - additional_user_name.strip(), + additional_user_names.strip(), additional_user_password, ) - else: - self.add_user( - UserModel, additional_user_names.strip(), additional_user_password - ) From 117d978ee9845e0b32449fc880e8d963411250b6 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 22 Jan 2024 10:48:17 -0700 Subject: [PATCH 215/255] Only continue initialization if usb is found when required. --- src/initialization/__init__.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 290f8075..ef1e0009 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -13,16 +13,12 @@ status_monitor = StatusMonitor() -def check_for_usb(): +def usb_exists() -> bool: logger.debug("Checking for USB...") if settings.USB_PATH is not None: usb = Path(settings.USB_PATH) - if not usb.exists(): - logger.debug("Usb is not ready. Marking container as unhealthy") - if settings.IN_DOCKER: - Path(settings.SDR_HEALTHCHECK_FILE).touch() - else: - logger.debug("Found USB") + return usb.exists() + return True def status_registration_handler(sender, **kwargs): @@ -36,12 +32,17 @@ def status_registration_handler(sender, **kwargs): try: register_component_with_status.connect(status_registration_handler) logger.debug("Checking for /dev/bus/usb/002/003") - check_for_usb() - action_loader = ActionLoader() - logger.debug("test") - logger.debug(f"Actions ActionLoader has {len(action_loader.actions)} actions") - capabilities_loader = CapabilitiesLoader() - logger.debug("Calling sensor loader.") - sensor_loader = SensorLoader(capabilities_loader.capabilities) + usb_exists = usb_exists() + if usb_exists: + action_loader = ActionLoader() + logger.debug("test") + logger.debug(f"Actions ActionLoader has {len(action_loader.actions)} actions") + capabilities_loader = CapabilitiesLoader() + logger.debug("Calling sensor loader.") + sensor_loader = SensorLoader(capabilities_loader.capabilities) + else: + logger.warning("Usb is not ready. Marking container as unhealthy") + if settings.IN_DOCKER: + Path(settings.SDR_HEALTHCHECK_FILE).touch() except Exception as ex: logger.error(f"Error during initialization: {ex}") From cb4226884ec3acdc6ff8144adea011ba925a139b Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 22 Jan 2024 11:07:10 -0700 Subject: [PATCH 216/255] initialize objects to allow system to continue to start when not healthy. --- src/initialization/__init__.py | 12 ++++++++++++ src/initialization/capabilities_loader.py | 14 ++++++++------ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index ef1e0009..cb91fd10 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -1,4 +1,5 @@ import logging +import types from pathlib import Path from django.conf import settings from .action_loader import ActionLoader @@ -6,6 +7,7 @@ from .sensor_loader import SensorLoader from .status_monitor import StatusMonitor + from utils.signals import register_component_with_status logger = logging.getLogger(__name__) @@ -41,6 +43,16 @@ def status_registration_handler(sender, **kwargs): logger.debug("Calling sensor loader.") sensor_loader = SensorLoader(capabilities_loader.capabilities) else: + action_loader = types.SimpleNamespace() + action_loader.actions = {} + capabilities_loader = types.SimpleNamespace() + capabilities_loader.capabilities = {} + sensor_loader = types.SimpleNamespace() + sensor_loader.sensor = types.SimpleNamespace() + sensor_loader.sensor.signal_analyzer = None + sensor_loader.preselector = None + sensor_loader.switches = {} + sensor_loader.capabilities = {} logger.warning("Usb is not ready. Marking container as unhealthy") if settings.IN_DOCKER: Path(settings.SDR_HEALTHCHECK_FILE).touch() diff --git a/src/initialization/capabilities_loader.py b/src/initialization/capabilities_loader.py index cc7bc099..4db42640 100644 --- a/src/initialization/capabilities_loader.py +++ b/src/initialization/capabilities_loader.py @@ -6,7 +6,8 @@ logger = logging.getLogger(__name__) -class CapabilitiesLoader(object): + +class CapabilitiesLoader: _instance = None def __init__(self): @@ -18,12 +19,12 @@ def __init__(self): def __new__(cls): if cls._instance is None: - logger.debug('Creating the ActionLoader') - cls._instance = super(CapabilitiesLoader, cls).__new__(cls) + logger.debug("Creating the ActionLoader") + cls._instance = super().__new__(cls) return cls._instance -def load_capabilities(sensor_definition_file): +def load_capabilities(sensor_definition_file) -> dict: capabilities = {} sensor_definition_hash = None sensor_location = None @@ -43,12 +44,13 @@ def load_capabilities(sensor_definition_file): try: if "sensor_sha512" not in capabilities["sensor"]: sensor_def = json.dumps(capabilities["sensor"], sort_keys=True) - sensor_definition_hash = hashlib.sha512(sensor_def.encode("UTF-8")).hexdigest() + sensor_definition_hash = hashlib.sha512( + sensor_def.encode("UTF-8") + ).hexdigest() capabilities["sensor"]["sensor_sha512"] = sensor_definition_hash except: capabilities["sensor"]["sensor_sha512"] = "ERROR GENERATING HASH" # sensor_sha512 is None, do not raise Exception, but log it logger.exception(f"Unable to generate sensor definition hash") - return capabilities From d94502b2831e4e2f23f967731d1937c4f33e8375 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 22 Jan 2024 11:15:18 -0700 Subject: [PATCH 217/255] try making /dev/bus/usb a volume. --- docker-compose.yml | 1 + src/initialization/__init__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9ae223de..d93cd12f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,6 +71,7 @@ services: - ${REPO_ROOT}/configs:/configs:rw - ${REPO_ROOT}/drivers:/drivers:ro - ${REPO_ROOT}/files:/files:rw + - /dev/bus/usb tmpfs: - /scos_tmp cap_add: diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index cb91fd10..43f55c65 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -18,6 +18,7 @@ def usb_exists() -> bool: logger.debug("Checking for USB...") if settings.USB_PATH is not None: + logger.debug("Checking for " + settings.USB_PATH) usb = Path(settings.USB_PATH) return usb.exists() return True @@ -33,7 +34,6 @@ def status_registration_handler(sender, **kwargs): try: register_component_with_status.connect(status_registration_handler) - logger.debug("Checking for /dev/bus/usb/002/003") usb_exists = usb_exists() if usb_exists: action_loader = ActionLoader() From 552aececd1c08307dc1ddded2aae5c32e05d69ee Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 22 Jan 2024 11:25:46 -0700 Subject: [PATCH 218/255] use lsusb to check for required usb device. --- docker-compose.yml | 2 +- src/initialization/__init__.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index d93cd12f..78736e0b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -64,7 +64,7 @@ services: - SIGAN_POWER_SWITCH - SIGAN_POWER_CYCLE_STATES - RUNNING_MIGRATIONS - - USB_PATH + - USB_DEVICE expose: - '8000' volumes: diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 43f55c65..78ee701e 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -6,7 +6,7 @@ from .capabilities_loader import CapabilitiesLoader from .sensor_loader import SensorLoader from .status_monitor import StatusMonitor - +from subprocess import check_output from utils.signals import register_component_with_status @@ -17,10 +17,10 @@ def usb_exists() -> bool: logger.debug("Checking for USB...") - if settings.USB_PATH is not None: - logger.debug("Checking for " + settings.USB_PATH) - usb = Path(settings.USB_PATH) - return usb.exists() + if settings.USB_DEVICE is not None: + usb_devices = check_output("lsusb") + logger.debug("Checking for " + settings.USB_DEVICE) + return settings.USB_DEVICE in usb_devices return True From a9d0a2edc72020fb268a4fefc531ed6e50dd74c3 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 22 Jan 2024 11:30:24 -0700 Subject: [PATCH 219/255] add USB_DEVICE to settings. --- src/sensor/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 109967cb..20154a19 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -440,4 +440,4 @@ SIGAN_POWER_SWITCH = env("SIGAN_POWER_SWITCH", default=None) MAX_FAILURES = env("MAX_FAILURES", default=2) os.environ["RUNNING_TESTS"] = str(RUNNING_TESTS) -USB_PATH = env("USB_PATH", default=None) +USB_DEVICE = env("USB_DEVICE", default=None) From 2a1d1d87399cd184ac3d2887285e32aa67a39481 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 22 Jan 2024 11:43:59 -0700 Subject: [PATCH 220/255] decode lsusb output. Add usbutils to docker container. --- docker/Dockerfile-api | 2 +- src/initialization/__init__.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/docker/Dockerfile-api b/docker/Dockerfile-api index 858a04a5..8c4e3b8f 100644 --- a/docker/Dockerfile-api +++ b/docker/Dockerfile-api @@ -6,7 +6,7 @@ RUN apt-get update -q && \ apt-get install -qy --no-install-recommends \ libusb-1.0-0 libpython3.8 \ git smartmontools \ - python3-pip python3.8 python3.8-dev && \ + python3-pip python3.8 python3.8-dev usbutils && \ apt-get clean && rm -rf /var/lib/apt/lists/* ENV PYTHONUNBUFFERED 1 diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 78ee701e..c34ea755 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -1,14 +1,17 @@ import logging +import sys import types from pathlib import Path +from subprocess import check_output + from django.conf import settings + +from utils.signals import register_component_with_status + from .action_loader import ActionLoader from .capabilities_loader import CapabilitiesLoader from .sensor_loader import SensorLoader from .status_monitor import StatusMonitor -from subprocess import check_output - -from utils.signals import register_component_with_status logger = logging.getLogger(__name__) @@ -18,8 +21,9 @@ def usb_exists() -> bool: logger.debug("Checking for USB...") if settings.USB_DEVICE is not None: - usb_devices = check_output("lsusb") + usb_devices = check_output("lsusb").decode(sys.stdout.encoding) logger.debug("Checking for " + settings.USB_DEVICE) + logger.debug("Found " + usb_devices) return settings.USB_DEVICE in usb_devices return True From 5b60c1015bc7e72853a9d9a54dcc409ef3a41d62 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 22 Jan 2024 11:55:24 -0700 Subject: [PATCH 221/255] add USB_DEVICE to env.template and reset container if sigan is not healthy at start. --- env.template | 1 + src/initialization/__init__.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/env.template b/env.template index 2f6429c4..9ba976c5 100644 --- a/env.template +++ b/env.template @@ -76,3 +76,4 @@ PATH_TO_VERIFY_CERT=scos_test_ca.crt # set to CERT to enable scos-sensor certificate authentication AUTHENTICATION=TOKEN +USB_DEVICE=Tektronix diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index c34ea755..6ad8c256 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -46,6 +46,12 @@ def status_registration_handler(sender, **kwargs): capabilities_loader = CapabilitiesLoader() logger.debug("Calling sensor loader.") sensor_loader = SensorLoader(capabilities_loader.capabilities) + if not sensor_loader.sensor.signal_analyzer.healthy(): + if settings.IN_DOCKER: + logger.warning( + "Signal analyzer is not healthy. Marking container for restart." + ) + Path(settings.SDR_HEALTHCHECK_FILE).touch() else: action_loader = types.SimpleNamespace() action_loader.actions = {} From 8ee8c6410e2ac3e2b890f0fa8658eeb24dcfb417 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 22 Jan 2024 12:38:32 -0700 Subject: [PATCH 222/255] cleanup --- src/initialization/__init__.py | 26 ++-- src/initialization/action_loader.py | 65 ++++++--- src/initialization/apps.py | 7 + src/initialization/capabilities_loader.py | 9 +- src/initialization/sensor_loader.py | 155 ++++++++++++++-------- src/initialization/status_monitor.py | 24 +++- 6 files changed, 189 insertions(+), 97 deletions(-) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 6ad8c256..d0e8e320 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -18,7 +18,7 @@ status_monitor = StatusMonitor() -def usb_exists() -> bool: +def usb_device_exists() -> bool: logger.debug("Checking for USB...") if settings.USB_DEVICE is not None: usb_devices = check_output("lsusb").decode(sys.stdout.encoding) @@ -36,22 +36,27 @@ def status_registration_handler(sender, **kwargs): logger.exception("Error registering status component") +def set_container_unhealthy(): + if settings.IN_DOCKER: + logger.warning("Signal analyzer is not healthy. Marking container for restart.") + Path(settings.SDR_HEALTHCHECK_FILE).touch() + + try: register_component_with_status.connect(status_registration_handler) - usb_exists = usb_exists() - if usb_exists: + usb_device_exists = usb_device_exists() + if usb_device_exists: action_loader = ActionLoader() logger.debug("test") logger.debug(f"Actions ActionLoader has {len(action_loader.actions)} actions") capabilities_loader = CapabilitiesLoader() logger.debug("Calling sensor loader.") sensor_loader = SensorLoader(capabilities_loader.capabilities) - if not sensor_loader.sensor.signal_analyzer.healthy(): - if settings.IN_DOCKER: - logger.warning( - "Signal analyzer is not healthy. Marking container for restart." - ) - Path(settings.SDR_HEALTHCHECK_FILE).touch() + if ( + not settings.RUNNING_MIGRATIONS + and not sensor_loader.sensor.signal_analyzer.healthy() + ): + set_container_unhealthy() else: action_loader = types.SimpleNamespace() action_loader.actions = {} @@ -64,7 +69,6 @@ def status_registration_handler(sender, **kwargs): sensor_loader.switches = {} sensor_loader.capabilities = {} logger.warning("Usb is not ready. Marking container as unhealthy") - if settings.IN_DOCKER: - Path(settings.SDR_HEALTHCHECK_FILE).touch() + set_container_unhealthy() except Exception as ex: logger.error(f"Error during initialization: {ex}") diff --git a/src/initialization/action_loader.py b/src/initialization/action_loader.py index fa3289d8..d58fdd6b 100644 --- a/src/initialization/action_loader.py +++ b/src/initialization/action_loader.py @@ -4,32 +4,56 @@ import os import pkgutil import shutil +from typing import Dict + from django.conf import settings from scos_actions.actions import action_classes -from scos_actions.discover import test_actions -from scos_actions.discover import init +from scos_actions.discover import init, test_actions +from scos_actions.interfaces.action import Action logger = logging.getLogger(__name__) -class ActionLoader(object): + +class ActionLoader: + """ + Loads actions from scos_ plugins and any yaml configurations + in the configs/actions directory. Note: this class is a + singleton so other applications may safely create an instance + and reference the .actions property. + """ + _instance = None def __init__(self): if not hasattr(self, "actions"): logger.debug("Actions have not been loaded. Loading actions...") - self.actions = load_actions_and_sigan(settings.MOCK_SIGAN, settings.RUNNING_TESTS, settings.DRIVERS_DIR, - settings.ACTIONS_DIR) + self._actions = load_actions( + settings.MOCK_SIGAN, + settings.RUNNING_TESTS, + settings.DRIVERS_DIR, + settings.ACTIONS_DIR, + ) else: logger.debug("Already loaded actions. ") def __new__(cls): if cls._instance is None: - logger.debug('Creating the ActionLoader') - cls._instance = super(ActionLoader, cls).__new__(cls) - logger.debug(f"Calling load_actions with {settings.MOCK_SIGAN}, {settings.RUNNING_TESTS}, {settings.DRIVERS_DIR}, {settings.ACTIONS_DIR}") + logger.debug("Creating the ActionLoader") + cls._instance = super().__new__(cls) + logger.debug( + f"Calling load_actions with {settings.MOCK_SIGAN}, {settings.RUNNING_TESTS}, {settings.DRIVERS_DIR}, {settings.ACTIONS_DIR}" + ) return cls._instance -def copy_driver_files(driver_dir): + @property + def actions(self) -> Dict[str, Action]: + """ + Returns all sensor actions configured in the system. + """ + return self._actions + + +def copy_driver_files(driver_dir: str): """Copy driver files where they need to go""" logger.debug(f"Copying driver files in {driver_dir}") for root, dirs, files in os.walk(driver_dir): @@ -43,9 +67,7 @@ def copy_driver_files(driver_dir): if type(json_data) == dict and "scos_files" in json_data: scos_files = json_data["scos_files"] for scos_file in scos_files: - source_path = os.path.join( - driver_dir, scos_file["source_path"] - ) + source_path = os.path.join(driver_dir, scos_file["source_path"]) if not os.path.isfile(source_path): logger.error(f"Unable to find file at {source_path}") continue @@ -60,8 +82,10 @@ def copy_driver_files(driver_dir): logger.error(f"Failed to copy {source_path} to {dest_path}") logger.error(e) -def load_actions_and_sigan(mock_sigan, running_tests, driver_dir, action_dir): +def load_actions( + mock_sigan: bool, running_tests: bool, driver_dir: str, action_dir: str +): logger.debug("********** Initializing actions **********") copy_driver_files(driver_dir) # copy driver files before loading plugins discovered_plugins = { @@ -71,7 +95,6 @@ def load_actions_and_sigan(mock_sigan, running_tests, driver_dir, action_dir): } logger.debug(discovered_plugins) actions = {} - signal_analyzer = None if mock_sigan or running_tests: logger.debug(f"Loading {len(test_actions)} test actions.") actions.update(test_actions) @@ -82,16 +105,16 @@ def load_actions_and_sigan(mock_sigan, running_tests, driver_dir, action_dir): if hasattr(discover, "actions"): logger.debug(f"loading {len(discover.actions)} actions.") actions.update(discover.actions) - if hasattr(discover, "action_classes") and discover.action_classes is not None: + if ( + hasattr(discover, "action_classes") + and discover.action_classes is not None + ): action_classes.update(discover.action_classes) - # if hasattr(discover, "signal_analyzer") and discover.signal_analyzer is not None: - # logger.debug(f"Found signal_analyzer: {discover.signal_analyzer}") - # signal_analyzer = discover.signal_analyzer - # else: - # logger.debug(f"{discover} has no signal_analyzer attribute") logger.debug(f"Loading actions in {action_dir}") - yaml_actions, yaml_test_actions = init(action_classes=action_classes, yaml_dir=action_dir) + yaml_actions, yaml_test_actions = init( + action_classes=action_classes, yaml_dir=action_dir + ) actions.update(yaml_actions) logger.debug("Finished loading and registering actions") return actions diff --git a/src/initialization/apps.py b/src/initialization/apps.py index 46e8f1e4..6dc552bc 100644 --- a/src/initialization/apps.py +++ b/src/initialization/apps.py @@ -2,4 +2,11 @@ class InitializationConfig(AppConfig): + """ + The first application to load. This application is responsible + for initializing the hardware components and loading actions. + This ensures the components are initialized in the appropriate + order and available for the other applications. + """ + name = "initialization" diff --git a/src/initialization/capabilities_loader.py b/src/initialization/capabilities_loader.py index 4db42640..4c8d2059 100644 --- a/src/initialization/capabilities_loader.py +++ b/src/initialization/capabilities_loader.py @@ -1,6 +1,7 @@ import hashlib import json import logging + from django.conf import settings from scos_actions.utils import load_from_json @@ -13,7 +14,7 @@ class CapabilitiesLoader: def __init__(self): if not hasattr(self, "capabilities"): logger.debug("Capabilities have not been loaded. Loading...") - self.capabilities = load_capabilities(settings.SENSOR_DEFINITION_FILE) + self._capabilities = load_capabilities(settings.SENSOR_DEFINITION_FILE) else: logger.debug("Already loaded capabilities. ") @@ -23,8 +24,12 @@ def __new__(cls): cls._instance = super().__new__(cls) return cls._instance + @property + def capabilities(self) -> dict: + return self._capabilities + -def load_capabilities(sensor_definition_file) -> dict: +def load_capabilities(sensor_definition_file: str) -> dict: capabilities = {} sensor_definition_hash = None sensor_location = None diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index 538ec101..1cb86e91 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -1,41 +1,49 @@ import importlib import logging import signal -from django.conf import settings -from scos_actions.hardware.sensor import Sensor -from scos_actions.hardware.sigan_iface import SignalAnalyzerInterface -from scos_actions.metadata.utils import construct_geojson_point from os import path from pathlib import Path + +from django.conf import settings from its_preselector.configuration_exception import ConfigurationException from its_preselector.controlbyweb_web_relay import ControlByWebWebRelay +from its_preselector.preselector import Preselector from scos_actions import utils from scos_actions.calibration.calibration import Calibration, load_from_json +from scos_actions.hardware.sensor import Sensor +from scos_actions.metadata.utils import construct_geojson_point + from utils.signals import register_component_with_status logger = logging.getLogger(__name__) -class SensorLoader(object): + +class SensorLoader: _instance = None - def __init__(self, sensor_capabilities): + def __init__(self, sensor_capabilities: dict): if not hasattr(self, "sensor"): logger.debug("Sensor has not been loaded. Loading...") - self.sensor = load_sensor(sensor_capabilities) + self._sensor = load_sensor(sensor_capabilities) else: logger.debug("Already loaded sensor. ") def __new__(cls, sensor_capabilities): if cls._instance is None: - logger.debug('Creating the SensorLoader') - cls._instance = super(SensorLoader, cls).__new__(cls) + logger.debug("Creating the SensorLoader") + cls._instance = super().__new__(cls) return cls._instance -def load_sensor(sensor_capabilities): + @property + def sensor(self) -> Sensor: + return self._sensor + + +def load_sensor(sensor_capabilities: dict) -> Sensor: location = None - #Remove location from sensor definition and convert to geojson. - #Db may have an updated location, but status module will update it - #if needed. + # Remove location from sensor definition and convert to geojson. + # Db may have an updated location, but status module will update it + # if needed. if "location" in sensor_capabilities["sensor"]: sensor_loc = sensor_capabilities["sensor"].pop("location") location = construct_geojson_point( @@ -44,54 +52,72 @@ def load_sensor(sensor_capabilities): sensor_loc["z"] if "z" in sensor_loc else None, ) switches = load_switches(settings.SWITCH_CONFIGS_DIR) - sensor_cal = get_sensor_calibration(settings.SENSOR_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) - sigan_cal = get_sigan_calibration(settings.SIGAN_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE) + sensor_cal = get_sensor_calibration( + settings.SENSOR_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE + ) + sigan_cal = get_sigan_calibration( + settings.SIGAN_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE + ) sigan = None try: if not settings.RUNNING_MIGRATIONS: sigan_module_setting = settings.SIGAN_MODULE sigan_module = importlib.import_module(sigan_module_setting) - logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE) + logger.info( + "Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE + ) sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS) - sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal, switches = switches) + sigan = sigan_constructor( + sensor_cal=sensor_cal, sigan_cal=sigan_cal, switches=switches + ) register_component_with_status.send(sigan, component=sigan) else: logger.info("Running migrations. Not loading signal analyzer.") except Exception as ex: logger.warning(f"unable to create signal analyzer: {ex}") - preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELECTOR_MODULE, - settings.PRESELECTOR_CLASS, sensor_capabilities["sensor"]) - - sensor = Sensor(signal_analyzer=sigan, preselector=preselector, switches=switches, capabilities=sensor_capabilities, - location=location) + preselector = load_preselector( + settings.PRESELECTOR_CONFIG, + settings.PRESELECTOR_MODULE, + settings.PRESELECTOR_CLASS, + sensor_capabilities["sensor"], + ) + + sensor = Sensor( + signal_analyzer=sigan, + preselector=preselector, + switches=switches, + capabilities=sensor_capabilities, + location=location, + ) return sensor - def load_switches(switch_dir: Path) -> dict: - logger.debug(f"Loading switches in {switch_dir}") - switch_dict = {} - try: - if switch_dir is not None and switch_dir.is_dir(): - for f in switch_dir.iterdir(): - file_path = f.resolve() - logger.debug(f"loading switch config {file_path}") - conf = utils.load_from_json(file_path) - try: - switch = ControlByWebWebRelay(conf) - logger.debug(f"Adding {switch.id}") - switch_dict[switch.id] = switch - logger.debug(f"Registering switch status for {switch.name}") - register_component_with_status.send(__name__, component=switch) - except ConfigurationException: - logger.error(f"Unable to configure switch defined in: {file_path}") - except Exception as ex: - logger.error(f"Unable to load switches {ex}") - return switch_dict - - -def load_preselector_from_file(preselector_module, preselector_class, preselector_config_file: Path): + logger.debug(f"Loading switches in {switch_dir}") + switch_dict = {} + try: + if switch_dir is not None and switch_dir.is_dir(): + for f in switch_dir.iterdir(): + file_path = f.resolve() + logger.debug(f"loading switch config {file_path}") + conf = utils.load_from_json(file_path) + try: + switch = ControlByWebWebRelay(conf) + logger.debug(f"Adding {switch.id}") + switch_dict[switch.id] = switch + logger.debug(f"Registering switch status for {switch.name}") + register_component_with_status.send(__name__, component=switch) + except ConfigurationException: + logger.error(f"Unable to configure switch defined in: {file_path}") + except Exception as ex: + logger.error(f"Unable to load switches {ex}") + return switch_dict + + +def load_preselector_from_file( + preselector_module, preselector_class, preselector_config_file: Path +): if preselector_config_file is None: return None else: @@ -107,8 +133,15 @@ def load_preselector_from_file(preselector_module, preselector_class, preselecto return None -def load_preselector(preselector_config: str, module: str, preselector_class_name: str, sensor_definition: dict): - logger.debug(f"loading {preselector_class_name} from {module} with config: {preselector_config}") +def load_preselector( + preselector_config: str, + module: str, + preselector_class_name: str, + sensor_definition: dict, +) -> Preselector: + logger.debug( + f"loading {preselector_class_name} from {module} with config: {preselector_config}" + ) if module is not None and preselector_class_name is not None: preselector_module = importlib.import_module(module) preselector_constructor = getattr(preselector_module, preselector_class_name) @@ -120,10 +153,9 @@ def load_preselector(preselector_config: str, module: str, preselector_class_nam return ps - - - -def get_sigan_calibration(sigan_cal_file_path: str, default_cal_file_path: str) -> Calibration: +def get_sigan_calibration( + sigan_cal_file_path: str, default_cal_file_path: str +) -> Calibration: """ Load signal analyzer calibration data from file. @@ -135,14 +167,19 @@ def get_sigan_calibration(sigan_cal_file_path: str, default_cal_file_path: str) try: sigan_cal = None if sigan_cal_file_path is None or sigan_cal_file_path == "": - logger.warning("No sigan calibration file specified. Not loading calibration file.") + logger.warning( + "No sigan calibration file specified. Not loading calibration file." + ) elif not path.exists(sigan_cal_file_path): logger.warning( - sigan_cal_file_path + " does not exist. Not loading sigan calibration file." + sigan_cal_file_path + + " does not exist. Not loading sigan calibration file." ) else: logger.debug(f"Loading sigan cal file: {sigan_cal_file_path}") - default = check_for_default_calibration(sigan_cal_file_path,default_cal_file_path, "Sigan") + default = check_for_default_calibration( + sigan_cal_file_path, default_cal_file_path, "Sigan" + ) sigan_cal = load_from_json(sigan_cal_file_path, default) sigan_cal.is_default = default except Exception: @@ -151,7 +188,9 @@ def get_sigan_calibration(sigan_cal_file_path: str, default_cal_file_path: str) return sigan_cal -def get_sensor_calibration(sensor_cal_file_path: str, default_cal_file_path: str) -> Calibration: +def get_sensor_calibration( + sensor_cal_file_path: str, default_cal_file_path: str +) -> Calibration: """ Load sensor calibration data from file. @@ -184,11 +223,13 @@ def get_sensor_calibration(sensor_cal_file_path: str, default_cal_file_path: str return sensor_cal -def check_for_default_calibration(cal_file_path: str,default_cal_path: str, cal_type: str) -> bool: +def check_for_default_calibration( + cal_file_path: str, default_cal_path: str, cal_type: str +) -> bool: default_cal = False if cal_file_path == default_cal_path: default_cal = True logger.warning( f"***************LOADING DEFAULT {cal_type} CALIBRATION***************" ) - return default_cal \ No newline at end of file + return default_cal diff --git a/src/initialization/status_monitor.py b/src/initialization/status_monitor.py index 611d62d9..64340c79 100644 --- a/src/initialization/status_monitor.py +++ b/src/initialization/status_monitor.py @@ -2,16 +2,29 @@ logger = logging.getLogger(__name__) -class StatusMonitor(object): + +class StatusMonitor: + """ + Singleton the keeps track of all components within the system that can provide + status. + """ + _instance = None def __new__(cls): if cls._instance is None: - logger.debug('Creating the ActionLoader') - cls._instance = super(StatusMonitor, cls).__new__(cls) - cls._instance.status_components = [] + logger.debug("Creating the ActionLoader") + cls._instance = super().__new__(cls) + cls._instance._status_components = [] return cls._instance + @property + def status_components(self): + """ + Returns any components that have been registered as status providing. + """ + return self._status_components + def add_component(self, component): """ Allows objects to be registered to provide status. Any object registered will @@ -21,5 +34,4 @@ def add_component(self, component): :param component: the object to add to the list of status providing objects. """ if hasattr(component, "get_status"): - self.status_components.append(component) - + self._status_components.append(component) From 6b6882a8155252e9e8ef241f6070348ee9461695 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 22 Jan 2024 12:42:41 -0700 Subject: [PATCH 223/255] correct Action import. --- src/initialization/action_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/initialization/action_loader.py b/src/initialization/action_loader.py index d58fdd6b..0f82c8e9 100644 --- a/src/initialization/action_loader.py +++ b/src/initialization/action_loader.py @@ -9,7 +9,7 @@ from django.conf import settings from scos_actions.actions import action_classes from scos_actions.discover import init, test_actions -from scos_actions.interfaces.action import Action +from scos_actions.actions.interfaces.action import Action logger = logging.getLogger(__name__) From 738693fad76c77f9039a81e73d2147565c37c13e Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 22 Jan 2024 14:45:19 -0700 Subject: [PATCH 224/255] Remove /dev/bus/usb volume. It makes no difference. --- docker-compose.yml | 1 - src/requirements.txt | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 78736e0b..db9f3a60 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,7 +71,6 @@ services: - ${REPO_ROOT}/configs:/configs:rw - ${REPO_ROOT}/drivers:/drivers:ro - ${REPO_ROOT}/files:/files:rw - - /dev/bus/usb tmpfs: - /scos_tmp cap_add: diff --git a/src/requirements.txt b/src/requirements.txt index e468330a..05770207 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -146,7 +146,7 @@ scipy==1.10.1 # via scos-actions scos-actions @ git+https://github.com/NTIA/scos-actions@discover_action_types # via scos-tekrsa -scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@discover_actions_types +scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@discover_action_types # via -r requirements.in sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive # via scos-actions From 562f461a66f5c98de8d4631ede36e99a5d89659d Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 22 Jan 2024 14:53:27 -0700 Subject: [PATCH 225/255] use sensor_loader sensor in test_db_location_deleted_handler. --- src/handlers/tests/test_handlers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/handlers/tests/test_handlers.py b/src/handlers/tests/test_handlers.py index 4af79eff..04f25777 100644 --- a/src/handlers/tests/test_handlers.py +++ b/src/handlers/tests/test_handlers.py @@ -7,6 +7,7 @@ logger = logging.getLogger(__name__) + @pytest.mark.django_db def test_db_location_update_handler(): location = construct_geojson_point(-105.7, 40.5, 0) @@ -26,7 +27,6 @@ def test_db_location_update_handler(): assert sensor.location["coordinates"][2] == 10 - @pytest.mark.django_db def test_db_location_update_handler_current_location_none(): sensor = sensor_loader.sensor @@ -68,7 +68,7 @@ def test_db_location_update_handler_not_active(): @pytest.mark.django_db def test_db_location_deleted_handler(): location = construct_geojson_point(-105.7, 40.5, 0) - sensor = Sensor(location=location) + sensor = sensor_loader.sensor location = Location() location.gps = False location.height = 10 From 6e3927e0ab75f60cc079544d848f603dfe2ddd7d Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 22 Jan 2024 17:05:36 -0700 Subject: [PATCH 226/255] Fix tests. remove dead code. --- src/conftest.py | 3 -- src/handlers/tests/test_handlers.py | 5 +- .../tests/test_measurement_result_handler.py | 1 + src/initialization/__init__.py | 2 +- src/initialization/sensor_loader.py | 51 ++++++++++--------- src/scheduler/scheduler.py | 2 +- 6 files changed, 34 insertions(+), 30 deletions(-) diff --git a/src/conftest.py b/src/conftest.py index 0675db34..87c114c6 100644 --- a/src/conftest.py +++ b/src/conftest.py @@ -40,9 +40,6 @@ def testclock(): def test_scheduler(rf, testclock): """Instantiate test scheduler with fake request context and testclock.""" s = scheduler.scheduler.Scheduler() - # mock_sigan = MockSignalAnalyzer() - # sensor = Sensor(signal_analyzer=mock_sigan) - # s.sensor = sensor s.request = rf.post("mock://cburl/schedule") return s diff --git a/src/handlers/tests/test_handlers.py b/src/handlers/tests/test_handlers.py index 04f25777..ca049547 100644 --- a/src/handlers/tests/test_handlers.py +++ b/src/handlers/tests/test_handlers.py @@ -1,9 +1,10 @@ import logging + import pytest +from scos_actions.metadata.utils import construct_geojson_point + from initialization import sensor_loader from status.models import Location -from scos_actions.hardware.sensor import Sensor -from scos_actions.metadata.utils import construct_geojson_point logger = logging.getLogger(__name__) diff --git a/src/handlers/tests/test_measurement_result_handler.py b/src/handlers/tests/test_measurement_result_handler.py index 2379c13e..be260bde 100644 --- a/src/handlers/tests/test_measurement_result_handler.py +++ b/src/handlers/tests/test_measurement_result_handler.py @@ -6,6 +6,7 @@ from rest_framework import status from scos_actions.signals import measurement_action_completed +from initialization import sensor_loader from tasks.models import Acquisition from test_utils.task_test_utils import ( HTTPS_KWARG, diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index d0e8e320..1f5585a3 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -20,7 +20,7 @@ def usb_device_exists() -> bool: logger.debug("Checking for USB...") - if settings.USB_DEVICE is not None: + if not settings.RUNNING_TESTS and settings.USB_DEVICE is not None: usb_devices = check_output("lsusb").decode(sys.stdout.encoding) logger.debug("Checking for " + settings.USB_DEVICE) logger.debug("Found " + usb_devices) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index 1cb86e91..af0094e4 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -40,24 +40,36 @@ def sensor(self) -> Sensor: def load_sensor(sensor_capabilities: dict) -> Sensor: + switches = {} + sigan_cal = None + sensor_cal = None + preselector = None location = None - # Remove location from sensor definition and convert to geojson. - # Db may have an updated location, but status module will update it - # if needed. - if "location" in sensor_capabilities["sensor"]: - sensor_loc = sensor_capabilities["sensor"].pop("location") - location = construct_geojson_point( - sensor_loc["x"], - sensor_loc["y"], - sensor_loc["z"] if "z" in sensor_loc else None, + if not settings.RUNNING_TESTS: + # Remove location from sensor definition and convert to geojson. + # Db may have an updated location, but status module will update it + # if needed. + if "location" in sensor_capabilities["sensor"]: + sensor_loc = sensor_capabilities["sensor"].pop("location") + location = construct_geojson_point( + sensor_loc["x"], + sensor_loc["y"], + sensor_loc["z"] if "z" in sensor_loc else None, + ) + switches = load_switches(settings.SWITCH_CONFIGS_DIR) + sensor_cal = get_sensor_calibration( + settings.SENSOR_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE ) - switches = load_switches(settings.SWITCH_CONFIGS_DIR) - sensor_cal = get_sensor_calibration( - settings.SENSOR_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE - ) - sigan_cal = get_sigan_calibration( - settings.SIGAN_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE - ) + sigan_cal = get_sigan_calibration( + settings.SIGAN_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE + ) + preselector = load_preselector( + settings.PRESELECTOR_CONFIG, + settings.PRESELECTOR_MODULE, + settings.PRESELECTOR_CLASS, + sensor_capabilities["sensor"], + ) + sigan = None try: if not settings.RUNNING_MIGRATIONS: @@ -76,13 +88,6 @@ def load_sensor(sensor_capabilities: dict) -> Sensor: except Exception as ex: logger.warning(f"unable to create signal analyzer: {ex}") - preselector = load_preselector( - settings.PRESELECTOR_CONFIG, - settings.PRESELECTOR_MODULE, - settings.PRESELECTOR_CLASS, - sensor_capabilities["sensor"], - ) - sensor = Sensor( signal_analyzer=sigan, preselector=preselector, diff --git a/src/scheduler/scheduler.py b/src/scheduler/scheduler.py index c64419a0..a264dd90 100644 --- a/src/scheduler/scheduler.py +++ b/src/scheduler/scheduler.py @@ -170,7 +170,7 @@ def _call_task_action(self): try: logger.debug( - f"running task {entry_name}/{task_id} with sigan: {self.sensor}" + f"running task {entry_name}/{task_id} with sigan: {self.sensor.signal_analyzer}" ) detail = self.task.action_caller(self.sensor, schedule_entry_json, task_id) self.delayfn(0) # let other threads run From 095c3c96bde5312ce9a8dd5540430aa825e120f5 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 23 Jan 2024 11:26:13 -0700 Subject: [PATCH 227/255] Remove an evil comma converting the string last_calibration_datetime to an array. --- src/status/views.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/status/views.py b/src/status/views.py index 19c1d92b..61c90b64 100644 --- a/src/status/views.py +++ b/src/status/views.py @@ -11,10 +11,7 @@ get_datetime_str_now, ) -from initialization import ( - status_monitor, - sensor_loader -) +from initialization import status_monitor, sensor_loader from scheduler import scheduler from . import start_time @@ -62,8 +59,16 @@ def status(request, version, format=None): "disk_usage": disk_usage(), "days_up": get_days_up(), } - if sensor_loader.sensor is not None and sensor_loader.sensor.signal_analyzer is not None and sensor_loader.sensor.signal_analyzer.sensor_calibration is not None : - status_json["last_calibration_datetime"] = sensor_loader.sensor.signal_analyzer.sensor_calibration.last_calibration_datetime, + if ( + sensor_loader.sensor is not None + and sensor_loader.sensor.signal_analyzer is not None + and sensor_loader.sensor.signal_analyzer.sensor_calibration is not None + ): + status_json[ + "last_calibration_datetime" + ] = ( + sensor_loader.sensor.signal_analyzer.sensor_calibration.last_calibration_datetime + ) for component in status_monitor.status_components: component_status = component.get_status() if isinstance(component, WebRelay): From 7b298673fa4b05f99c9359df584fa989f3bdd0d5 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 26 Jan 2024 11:59:11 -0700 Subject: [PATCH 228/255] move capabilities in sensor constructor. --- src/initialization/sensor_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index af0094e4..a4a82457 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -90,9 +90,9 @@ def load_sensor(sensor_capabilities: dict) -> Sensor: sensor = Sensor( signal_analyzer=sigan, + capabilities=sensor_capabilities, preselector=preselector, switches=switches, - capabilities=sensor_capabilities, location=location, ) return sensor From a2e4c345f3bdf2f9f337e134044cb69a7bb6fa41 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 26 Jan 2024 13:27:41 -0700 Subject: [PATCH 229/255] Fix location handler logging. --- src/handlers/location_handler.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/handlers/location_handler.py b/src/handlers/location_handler.py index 0afffc87..593bc634 100644 --- a/src/handlers/location_handler.py +++ b/src/handlers/location_handler.py @@ -1,10 +1,13 @@ import logging + +from scos_actions.metadata.utils import construct_geojson_point + from initialization import sensor_loader from status.models import GPS_LOCATION_DESCRIPTION, Location -from scos_actions.metadata.utils import construct_geojson_point logger = logging.getLogger(__name__) + def location_action_completed_callback(sender, **kwargs): """Update database when GPS is synced or database is updated""" logger.debug(f"Updating location from {sender}") @@ -33,9 +36,15 @@ def location_action_completed_callback(sender, **kwargs): def db_location_updated(sender, **kwargs): instance = kwargs["instance"] - logger.debug(f"DB location updated by {sender}") if isinstance(instance, Location) and instance.active: - geojson = construct_geojson_point(longitude = instance.longitude, latitude=instance.latitude, altitude= instance.height) + geojson = construct_geojson_point( + longitude=instance.longitude, + latitude=instance.latitude, + altitude=instance.height, + ) + logger.debug( + f"DB location updated to latitude:{instance.latitude}, longitude:{instance.longitude}, height:{instance.height}" + ) if sensor_loader.sensor: sensor_loader.sensor.location = geojson logger.debug(f"Updated {sensor_loader.sensor} location to {geojson}") @@ -51,4 +60,6 @@ def db_location_deleted(sender, **kwargs): sensor_loader.sensor.location = None logger.debug(f"Set {sensor_loader.sensor} location to None.") else: - logger.warning("No sensor registered. Unable to remove sensor location.") + logger.warning( + "No sensor registered. Unable to remove sensor location." + ) From 159cb90afe0713248c8478bb9f3d4c5191d67195 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Sat, 27 Jan 2024 15:14:16 -0700 Subject: [PATCH 230/255] scos-actions 8.0.0 --- src/requirements-dev.txt | 2 +- src/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index 18ed0aba..290f8c7c 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -306,7 +306,7 @@ scipy==1.10.1 # via # -r requirements.txt # scos-actions -scos-actions @ git+https://github.com/NTIA/scos-actions@discover_action_types +scos-actions @ git+https://github.com/NTIA/scos-actions@8.0.0 # via # -r requirements.txt # scos-tekrsa diff --git a/src/requirements.txt b/src/requirements.txt index 05770207..1890bf07 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -144,7 +144,7 @@ ruamel-yaml-clib==0.2.8 # via ruamel-yaml scipy==1.10.1 # via scos-actions -scos-actions @ git+https://github.com/NTIA/scos-actions@discover_action_types +scos-actions @ git+https://github.com/NTIA/scos-actions@8.0.0 # via scos-tekrsa scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@discover_action_types # via -r requirements.in From ef43af2434442e0428e11363d20860f068e4dbf2 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Mon, 29 Jan 2024 16:51:55 -0500 Subject: [PATCH 231/255] remove unused and duplicate imports --- src/capabilities/__init__.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/capabilities/__init__.py b/src/capabilities/__init__.py index c4f1e809..0d7a1d78 100644 --- a/src/capabilities/__init__.py +++ b/src/capabilities/__init__.py @@ -1,14 +1,6 @@ -import hashlib -import json import logging -from initialization import action_loader - -from django.conf import settings -from initialization import action_loader -from initialization import capabilities_loader - - +from initialization import action_loader, capabilities_loader logger = logging.getLogger(__name__) logger.debug("********** Initializing capabilities **********") From 5eab69a612e15ded71e12b871f354a801ec338fd Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Mon, 29 Jan 2024 17:02:00 -0500 Subject: [PATCH 232/255] scos-tekrsa 5.0.0 --- src/requirements-dev.txt | 13 +++++++++++-- src/requirements.in | 2 +- src/requirements.txt | 7 ++++++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index 290f8c7c..fc283de7 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -59,7 +59,9 @@ colorama==0.4.6 colorful==0.5.5 # via ray coverage[toml]==7.3.2 - # via pytest-cov + # via + # coverage + # pytest-cov cryptography==41.0.7 # via -r requirements.txt defusedxml==0.7.1 @@ -182,6 +184,10 @@ numpy==1.24.4 # tekrsa-api-wrap nvidia-ml-py==12.535.133 # via gpustat +oauthlib==3.2.2 + # via + # -r requirements.txt + # requests-oauthlib opencensus==0.11.3 # via ray opencensus-context==0.1.3 @@ -285,8 +291,11 @@ requests==2.31.0 # its-preselector # ray # requests-mock + # requests-oauthlib requests-mock==1.11.0 # via -r requirements.txt +requests-oauthlib==1.3.1 + # via -r requirements.txt rpds-py==0.13.2 # via # -r requirements.txt @@ -310,7 +319,7 @@ scos-actions @ git+https://github.com/NTIA/scos-actions@8.0.0 # via # -r requirements.txt # scos-tekrsa -scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@discover_action_types +scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@5.0.0 # via -r requirements.txt sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive # via diff --git a/src/requirements.in b/src/requirements.in index 3562dcd1..bc87c3d5 100644 --- a/src/requirements.in +++ b/src/requirements.in @@ -11,7 +11,7 @@ packaging>=23.0, <24.0 psycopg2-binary>=2.0, <3.0 requests-mock>=1.0, <2.0 requests_oauthlib>=1.0, <2.0 -scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@discover_action_types +scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@5.0.0 # The following are sub-dependencies for which SCOS Sensor enforces a # higher minimum patch version than the dependencies which require them. diff --git a/src/requirements.txt b/src/requirements.txt index 1890bf07..fffdf660 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -91,6 +91,8 @@ numpy==1.24.4 # scos-actions # sigmf # tekrsa-api-wrap +oauthlib==3.2.2 + # via requests-oauthlib packaging==23.2 # via # -r requirements.in @@ -132,8 +134,11 @@ requests==2.31.0 # its-preselector # ray # requests-mock + # requests-oauthlib requests-mock==1.11.0 # via -r requirements.in +requests-oauthlib==1.3.1 + # via -r requirements.in rpds-py==0.13.2 # via # jsonschema @@ -146,7 +151,7 @@ scipy==1.10.1 # via scos-actions scos-actions @ git+https://github.com/NTIA/scos-actions@8.0.0 # via scos-tekrsa -scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@discover_action_types +scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@5.0.0 # via -r requirements.in sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive # via scos-actions From cd920ac72583a449e3ba3fbafc39843fed239d08 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 29 Jan 2024 15:11:02 -0700 Subject: [PATCH 233/255] Update readme, env.template, and optimize imports. Update requirements to scos-tekrsa 5.0.0. --- README.md | 319 ++++++++++++------ env.template | 101 +++--- .../management/commands/get_auth_token.py | 2 +- src/authentication/migrations/0001_initial.py | 2 - src/capabilities/__init__.py | 10 +- src/conftest.py | 6 - src/handlers/location_handler.py | 2 +- .../tests/test_measurement_result_handler.py | 1 - src/initialization/action_loader.py | 2 +- src/initialization/sensor_loader.py | 1 - .../tests/test_initialization.py | 16 +- src/requirements-dev.txt | 2 +- src/requirements.txt | 2 +- src/schedule/__init__.py | 4 +- src/schedule/migrations/0001_initial.py | 1 - src/schedule/models/schedule_entry.py | 1 - src/schedule/serializers.py | 2 +- src/schedule/tests/test_user_views.py | 1 - src/scheduler/tests/test_scheduler.py | 1 - src/scheduler/tests/utils.py | 3 +- src/sensor/settings.py | 3 - src/sensor/urls.py | 1 - src/status/__init__.py | 3 +- src/status/apps.py | 13 +- src/status/views.py | 3 +- src/tasks/models/task.py | 1 - src/tasks/models/task_result.py | 2 +- src/tasks/tests/test_archive_download.py | 2 - src/tasks/views.py | 1 - src/users/apps.py | 1 - 30 files changed, 308 insertions(+), 201 deletions(-) diff --git a/README.md b/README.md index 97399b36..478f0a9a 100644 --- a/README.md +++ b/README.md @@ -309,17 +309,19 @@ settings in the environment file: - ADDITIONAL_USER_NAMES: Comma separated list of additional admin usernames. - ADDITIONAL_USER_PASSWORD: Password for additional admin users. -- ADMIN_NAME: Username for the admin user. - ADMIN_EMAIL: Email used to generate admin user. Change in production. +- ADMIN_NAME: Username for the admin user. - ADMIN_PASSWORD: Password used to generate admin user. Change in production. - AUTHENTICATION: Authentication method used for scos-sensor. Supports `TOKEN` or `CERT`. -- BASE_IMAGE: Base docker image used to build the API container. +- BASE_IMAGE: Base docker image used to build the API container. Note, this should + updated when switching signal analyzers. - CALLBACK_AUTHENTICATION: Sets how to authenticate to the callback URL. Supports `TOKEN` or `CERT`. - CALLBACK_SSL_VERIFICATION: Set to “true” in production environment. If false, the SSL certificate validation will be ignored when posting results to the callback URL. -- CALLBACK_TIMEOUT: The timeout for the requests sent to the callback URL. +- CALLBACK_TIMEOUT: The timeout for the posts sent to the callback URL when a scheduled + action is completed. - DEBUG: Django debug mode. Set to False in production. - DOCKER_TAG: Always set to “latest” to install newest version of docker containers. - DOMAINS: A space separated list of domain names. Used to generate [ALLOWED_HOSTS]( @@ -349,12 +351,30 @@ settings in the environment file: unpredictable value. See . The env.template file sets to a randomly generated value. +- SIGAN_CLASS: The name of the signal analyzer class to use. By default, this is + TekRSASigan to use a Tektronix signal analyzer. This must be changed to switch to + a different signal analyzer. +- SIGAN_MODULE: The name of the python module that provides the signal analyzer + implementation. This defaults to scos_tekrsa.hardware.tekrsa_sigan for the + Tektronix signal analyzers. This must be changed to switch to a different + signal analyzer. +- SIGAN_POWER_CYCLE_STATES: Optional setting to provide the name of the control_state + in the SIGAN_POWER_SWITCH that will power cycle the signal analyzer. +- SIGAN_POWER_SWITCH: Optional setting used to indicate the name of a + [WebRelay](https://github.com/NTIA/Preselector) that may be used to power cycle + the signal analyzer if necessary.Note: determinations of power cycling behavior + are implemented within the signal analyzer implementations or actions. +- SSL_CA_PATH: Path to a CA certificate used to verify scos-sensor client + certificate(s) when authentication is set to CERT. - SSL_CERT_PATH: Path to server SSL certificate. Replace the certificate in the scos-sensor repository with a valid certificate in production. - SSL_KEY_PATH: Path to server SSL private key. Use the private key for your valid certificate in production. -- SSL_CA_PATH: Path to a CA certificate used to verify scos-sensor client - certificate(s) when authentication is set to CERT. +- USB_DEVICE: Optional string used to search for available USB devices. By default, + this is set to Tektronix to see if the Tektronix signal analyzer is available. If + the specified value is not found in the output of lsusb, scos-sensor will attempt + to restart the api container. If switching to a different signal analyzer, this + setting should be updated or removed. ### Sensor Definition File @@ -435,132 +455,223 @@ per second at several frequencies with a signal analyzer reference level setting ```json { + "last_calibration_datetime": "2023-10-23T14:39:13.682Z", "calibration_parameters": [ "sample_rate", "frequency", - "reference_level" + "reference_level", + "preamp_enable", + "attenuation" ], - "calibration_datetime": "2020-11-18T23:13:09.156274Z", + "clock_rate_lookup_by_sample_rate": [], "calibration_data": { - "14000000.0":{ - "3555000000":{ - "-25":{ - "noise_figure": 46.03993010994134, - "enbw": 15723428.858731967, - "gain": 0.40803345928877379 + "14000000.0": { + "3545000000.0": { + "-25": { + "true": { + "0": { + "datetime": "2023-10-23T14:38:02.882Z", + "gain": 30.09194805857024, + "noise_figure": 4.741521295220736, + "temperature": 15.6 + } + } + } + }, + "3555000000.0": { + "-25": { + "true": { + "0": { + "datetime": "2023-10-23T14:38:08.022Z", + "gain": 30.401008416406599, + "noise_figure": 4.394893979804061, + "temperature": 15.6 + } + } } }, - "3565000000":{ - "-25":{ - "noise_figure": 46.03993010994134, - "enbw": 15723428.858731967, - "gain": 0.40803345928877379 + "3565000000.0": { + "-25": { + "true": { + "0": { + "datetime": "2023-10-23T14:38:11.922Z", + "gain": 30.848049817892105, + "noise_figure": 4.0751785215495819, + "temperature": 15.6 + } + } } }, - "3575000000":{ - "-25":{ - "noise_figure": 46.03993010994134, - "enbw": 15723428.858731967, - "gain": 0.40803345928877379 + "3575000000.0": { + "-25": { + "true": { + "0": { + "datetime": "2023-10-23T14:38:16.211Z", + "gain": 30.73297444891243, + "noise_figure": 4.090843866619065, + "temperature": 15.6 + } + } } }, - "3585000000":{ - "-25":{ - "noise_figure": 46.03993010994134, - "enbw": 15723428.858731967, - "gain": 0.40803345928877379 + "3585000000.0": { + "-25": { + "true": { + "0": { + "datetime": "2023-10-23T14:38:20.725Z", + "gain": 30.884253019974623, + "noise_figure": 3.934553150614483, + "temperature": 15.6 + } + } } }, - "3595000000":{ - "-25":{ - "noise_figure": 46.03993010994134, - "enbw": 15723428.858731967, - "gain": 0.40803345928877379 + "3595000000.0": { + "-25": { + "true": { + "0": { + "datetime": "2023-10-23T14:38:24.606Z", + "gain": 31.002780356672476, + "noise_figure": 3.940238988552726, + "temperature": 15.6 + } + } } }, - "3605000000":{ - "-25":{ - "noise_figure": 46.03993010994134, - "enbw": 15723428.858731967, - "gain": 0.40803345928877379 + "3605000000.0": { + "-25": { + "true": { + "0": { + "datetime": "2023-10-23T14:38:29.724Z", + "gain": 31.035560147646778, + "noise_figure": 3.9290832485193989, + "temperature": 15.6 + } + } } }, - "3615000000":{ - "-25":{ - "noise_figure": 46.03993010994134, - "enbw": 15723428.858731967, - "gain": 0.40803345928877379 + "3615000000.0": { + "-25": { + "true": { + "0": { + "datetime": "2023-10-23T14:38:33.614Z", + "gain": 30.935970273145274, + "noise_figure": 4.006672278350428, + "temperature": 15.6 + } + } } }, - "3625000000":{ - "-25":{ - "noise_figure": 46.03993010994134, - "enbw": 15723428.858731967, - "gain": 0.40803345928877379 + "3625000000.0": { + "-25": { + "true": { + "0": { + "datetime": "2023-10-23T14:38:37.592Z", + "gain": 30.682403307202095, + "noise_figure": 4.064067195729546, + "temperature": 15.6 + } + } } }, - "3635000000":{ - "-25":{ - "noise_figure": 46.03993010994134, - "enbw": 15723428.858731967, - "gain": 0.40803345928877379 + "3635000000.0": { + "-25": { + "true": { + "0": { + "datetime": "2023-10-23T14:38:41.889Z", + "gain": 30.980340383458626, + "noise_figure": 3.8826926914916726, + "temperature": 15.6 + } + } } }, - "3645000000":{ - "-25":{ - "noise_figure": 46.03993010994134, - "enbw": 15723428.858731967, - "gain": 0.40803345928877379 + "3645000000.0": { + "-25": { + "true": { + "0": { + "datetime": "2023-10-23T14:38:46.450Z", + "gain": 30.948486357958318, + "noise_figure": 3.917520438472101, + "temperature": 15.6 + } + } } }, - "3655000000":{ - "-25":{ - "noise_figure": 46.03993010994134, - "enbw": 15723428.858731967, - "gain": 0.40803345928877379 + "3655000000.0": { + "-25": { + "true": { + "0": { + "datetime": "2023-10-23T14:38:51.575Z", + "gain": 30.96859101287636, + "noise_figure": 3.926017393059535, + "temperature": 15.6 + } + } } }, - "3665000000":{ - "-25":{ - "noise_figure": 46.03993010994134, - "enbw": 15723428.858731967, - "gain": 0.40803345928877379 + "3665000000.0": { + "-25": { + "true": { + "0": { + "datetime": "2023-10-23T14:38:55.571Z", + "gain": 30.618851884796429, + "noise_figure": 4.225928655860898, + "temperature": 15.6 + } + } } }, - "3675000000":{ - "-25":{ - "noise_figure": 46.03993010994134, - "enbw": 15723428.858731967, - "gain": 0.40803345928877379 + "3675000000.0": { + "-25": { + "true": { + "0": { + "datetime": "2023-10-23T14:38:59.585Z", + "gain": 30.552247661443276, + "noise_figure": 4.171389553033189, + "temperature": 15.6 + } + } } }, - "3685000000":{ - "-25":{ - "noise_figure": 46.03993010994134, - "enbw": 15723428.858731967, - "gain": 0.40803345928877379 + "3685000000.0": { + "-25": { + "true": { + "0": { + "datetime": "2023-10-23T14:39:03.921Z", + "gain": 30.413415948237785, + "noise_figure": 4.42128294424517, + "temperature": 15.6 + } + } } }, - "3695000000":{ - "-25":{ - "noise_figure": 46.03993010994134, - "enbw": 15723428.858731967, - "gain": 0.40803345928877379 + "3695000000.0": { + "-25": { + "true": { + "0": { + "datetime": "2023-10-23T14:39:09.115Z", + "gain": 30.112285957420995, + "noise_figure": 4.576464590847393, + "temperature": 15.6 + } + } + } + }, + "3705000000.0": { + "-25": { + "true": { + "0": { + "datetime": "2023-10-23T14:39:13.682Z", + "gain": 29.120051306906164, + "noise_figure": 5.2261432005125709, + "temperature": 15.7 + } + } } } } - }, - "clock_rate_lookup_by_sample_rate": [ - { - "sample_rate": 3500000.0, - "clock_frequency": 3500000.0 - }, - { - "sample_rate": 28000000.0, - "clock_frequency": 28000000.0 - } - ], - "sensor_uid": "US55120115" + } } ``` @@ -838,17 +949,19 @@ repository. The scos-actions repository is intended to be a dependency for every as it contains the actions base class and signals needed to interface with scos-sensor. These actions use a common but flexible signal analyzer interface that can be implemented for new types of hardware. This allows for action re-use by passing the -signal analyzer interface implementation and the required hardware and measurement -parameters to the constructor of these actions. Alternatively, custom actions that -support unique hardware functionality can be added to the plugin. - -The scos-actions repository can also be installed as a plugin which uses a mock signal -analyzer. +measurement parameters to the constructor of these actions and supplying the +Sensor instance (including the signal analyzer) to the `__call__` method. +Alternatively, custom actions that support unique hardware functionality can be +added to the plugin. -scos-sensor uses the following convention to discover actions offered by plugins: if +Scos-sensor uses the following convention to discover actions offered by plugins: if any Python package begins with "scos_", and contains a dictionary of actions at the Python path `package_name.discover.actions`, these actions will automatically be -available for scheduling. +available for scheduling. Similarly, plugins may offer new action types be including +a dictionary of action classes at the Python path `package_name.discover.action_classes`. +Scos-sensor will load all plugin actions and action classes prior to creating the actions +defined in yaml file in `configs/actions` directory. In this manner, a plugin may add new +action types to scos-sensor and those new types my instantiated/parameterized with yaml The scos-usrp plugin adds support for the Ettus B2xx line of signal analyzers. It can also be used as an example of a plugin which adds new hardware support and diff --git a/env.template b/env.template index 9ba976c5..d4820693 100644 --- a/env.template +++ b/env.template @@ -6,74 +6,91 @@ # Mark all the following variables for export set -o allexport -# https://docs.djangoproject.com/en/1.11/ref/settings/#std:setting-SECRET_KEY -# SECURITY WARNING: generate unique key with `manage.py generate_secret_key` -SECRET_KEY="$(python3 -c 'import secrets; print(secrets.token_urlsafe(64))')" -ENCRYPT_DATA_FILES=true -ENCRYPTION_KEY="$(python3 -c 'import secrets; import base64; print(base64.b64encode(secrets.token_bytes(32)).decode("utf-8"))')" +#Any names here will be added as additional users with the +#specified additional user password +ADDITIONAL_USER_NAMES="" # comma separated +ADDITIONAL_USER_PASSWORD="" -# Get scos-sensor branch name -SCOS_SENSOR_GIT_TAG="$(git describe --tags)" +# If admin user email and password set, admin user will be generated. +ADMIN_EMAIL="admin@example.com" +ADMIN_NAME=admin +ADMIN_PASSWORD=password + +# set to CERT to enable scos-sensor certificate authentication +AUTHENTICATION=TOKEN + +BASE_IMAGE=ghcr.io/ntia/scos-tekrsa/tekrsa_usb:0.2.3 +# Default callback api/results +# Set to CERT for certificate authentication +CALLBACK_AUTHENTICATION=TOKEN +# Set to false to disable SSL cert verification in the callback POST request +CALLBACK_SSL_VERIFICATION=false + +#Set the number of seconds before timeout in postback when a scheduled +#action completes +CALLBACK_TIMEOUT=2 # SECURITY WARNING: don't run with debug turned on in production! # Use either true or false DEBUG=true +# Use latest as default for local development +DOCKER_TAG=latest + # A space-separated list of domain names and IPs DOMAINS="localhost $(hostname -d) $(hostname -s).local" -IPS="$(hostname -I) 127.0.0.1" + +ENCRYPT_DATA_FILES=true + +ENCRYPTION_KEY="$(python3 -c 'import secrets; import base64; print(base64.b64encode(secrets.token_bytes(32)).decode("utf-8"))')" + FQDN="$(hostname -f)" -# SECURITY WARNING: You should be using certs from a trusted authority. -# If you don't have any, try letsencrypt or a similar service. -# Provide the absolute path to your ssl certificate and key -# Paths relative to configs/certs -REPO_ROOT=$(git rev-parse --show-toplevel) -SSL_CERT_PATH=sensor01.pem -SSL_KEY_PATH=sensor01.pem -SSL_CA_PATH=scos_test_ca.crt -# Use latest as default for local development -DOCKER_TAG=latest GIT_BRANCH="git:$(git rev-parse --abbrev-ref HEAD)@$(git rev-parse --short HEAD)" -# If admin user email and password set, admin user will be generated. -ADMIN_EMAIL="admin@example.com" -ADMIN_PASSWORD=password -ADMIN_NAME=Admin -ADDITIONAL_USER_NAMES="" # comma separated -ADDITIONAL_USER_PASSWORD="" +IPS="$(hostname -I) 127.0.0.1" # Session password for Postgres. Username is "postgres". # SECURITY WARNING: generate unique key with something like # `openssl rand -base64 12` POSTGRES_PASSWORD="$(python3 -c 'import secrets; import base64; print(base64.b64encode(secrets.token_bytes(32)).decode("utf-8"))')" -if $DEBUG; then - GUNICORN_LOG_LEVEL=debug - RAY_record_ref_creation_sites=1 -else - GUNICORN_LOG_LEVEL=info -fi - -# Set to false to disable SSL cert verification in the callback POST request -CALLBACK_SSL_VERIFICATION=true - # set default manager FQDN and IP to this machine MANAGER_FQDN="$(hostname -f)" MANAGER_IP="$(hostname -I | cut -d' ' -f1)" -BASE_IMAGE=ghcr.io/ntia/scos-tekrsa/tekrsa_usb:0.2.3 -# Default callback api/results -# Set to CERT for certificate authentication -CALLBACK_AUTHENTICATION=TOKEN -CALLBACK_TIMEOUT=2 - # Sensor certificate with private key used as client cert for callback URL # Paths relative to configs/certs PATH_TO_CLIENT_CERT=sensor01.pem # Trusted Certificate Authority certificate to verify callback URL server certificate PATH_TO_VERIFY_CERT=scos_test_ca.crt -# set to CERT to enable scos-sensor certificate authentication -AUTHENTICATION=TOKEN +REPO_ROOT=$(git rev-parse --show-toplevel) +# Get scos-sensor branch name +SCOS_SENSOR_GIT_TAG="$(git describe --tags)" + +# https://docs.djangoproject.com/en/1.11/ref/settings/#std:setting-SECRET_KEY +# SECURITY WARNING: generate unique key with `manage.py generate_secret_key` +SECRET_KEY="$(python3 -c 'import secrets; print(secrets.token_urlsafe(64))')" + +SIGAN_CLASS=TekRSASigan +SIGAN_MODULE=scos_tekrsa.hardware.tekrsa_sigan + +# SECURITY WARNING: You should be using certs from a trusted authority. +# If you don't have any, try letsencrypt or a similar service. +# Provide the absolute path to your ssl certificate and key +# Paths relative to configs/certs +SSL_CA_PATH=scos_test_ca.crt +SSL_CERT_PATH=sensor01.pem +SSL_KEY_PATH=sensor01.pem + USB_DEVICE=Tektronix + + +# Debug dependant settings +if $DEBUG; then + GUNICORN_LOG_LEVEL=debug + RAY_record_ref_creation_sites=1 +else + GUNICORN_LOG_LEVEL=info +fi diff --git a/src/authentication/management/commands/get_auth_token.py b/src/authentication/management/commands/get_auth_token.py index 8101f9a5..2f483cfc 100644 --- a/src/authentication/management/commands/get_auth_token.py +++ b/src/authentication/management/commands/get_auth_token.py @@ -1,4 +1,4 @@ -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand from authentication.models import User diff --git a/src/authentication/migrations/0001_initial.py b/src/authentication/migrations/0001_initial.py index c80c2b10..0f9691f1 100644 --- a/src/authentication/migrations/0001_initial.py +++ b/src/authentication/migrations/0001_initial.py @@ -1,7 +1,5 @@ # Generated by Django 3.0.4 on 2020-03-18 16:31 -import django.contrib.auth.models -import django.contrib.auth.validators import django.utils.timezone from django.db import migrations, models diff --git a/src/capabilities/__init__.py b/src/capabilities/__init__.py index c4f1e809..0d7a1d78 100644 --- a/src/capabilities/__init__.py +++ b/src/capabilities/__init__.py @@ -1,14 +1,6 @@ -import hashlib -import json import logging -from initialization import action_loader - -from django.conf import settings -from initialization import action_loader -from initialization import capabilities_loader - - +from initialization import action_loader, capabilities_loader logger = logging.getLogger(__name__) logger.debug("********** Initializing capabilities **********") diff --git a/src/conftest.py b/src/conftest.py index 87c114c6..eeadc1c2 100644 --- a/src/conftest.py +++ b/src/conftest.py @@ -1,12 +1,7 @@ import shutil -import tempfile -from collections import namedtuple import pytest from django.conf import settings -from django.test.client import Client -from scos_actions.hardware.mocks.mock_sigan import MockSignalAnalyzer -from scos_actions.hardware.sensor import Sensor import scheduler from authentication.models import User @@ -134,7 +129,6 @@ def alt_admin_user(db, django_user_model, django_username_field): @pytest.fixture def alt_admin_client(db, alt_admin_user): """A Django test client logged in as an admin user.""" - from django.test.client import Client client = CertificateAuthClient() assert client.login(username=alt_admin_user.username, password="password") diff --git a/src/handlers/location_handler.py b/src/handlers/location_handler.py index 593bc634..fc66cf94 100644 --- a/src/handlers/location_handler.py +++ b/src/handlers/location_handler.py @@ -3,7 +3,7 @@ from scos_actions.metadata.utils import construct_geojson_point from initialization import sensor_loader -from status.models import GPS_LOCATION_DESCRIPTION, Location +from status.models import Location logger = logging.getLogger(__name__) diff --git a/src/handlers/tests/test_measurement_result_handler.py b/src/handlers/tests/test_measurement_result_handler.py index be260bde..2379c13e 100644 --- a/src/handlers/tests/test_measurement_result_handler.py +++ b/src/handlers/tests/test_measurement_result_handler.py @@ -6,7 +6,6 @@ from rest_framework import status from scos_actions.signals import measurement_action_completed -from initialization import sensor_loader from tasks.models import Acquisition from test_utils.task_test_utils import ( HTTPS_KWARG, diff --git a/src/initialization/action_loader.py b/src/initialization/action_loader.py index 0f82c8e9..f94fb5a1 100644 --- a/src/initialization/action_loader.py +++ b/src/initialization/action_loader.py @@ -8,8 +8,8 @@ from django.conf import settings from scos_actions.actions import action_classes -from scos_actions.discover import init, test_actions from scos_actions.actions.interfaces.action import Action +from scos_actions.discover import init, test_actions logger = logging.getLogger(__name__) diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index a4a82457..16c0c539 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -1,6 +1,5 @@ import importlib import logging -import signal from os import path from pathlib import Path diff --git a/src/initialization/tests/test_initialization.py b/src/initialization/tests/test_initialization.py index 2c3bb383..706a2e15 100644 --- a/src/initialization/tests/test_initialization.py +++ b/src/initialization/tests/test_initialization.py @@ -1,16 +1,22 @@ -from initialization.sensor_loader import load_preselector import logging import os +from initialization.sensor_loader import load_preselector + logger = logging.getLogger(__name__) + def test_load_preselector(): preselector_config = os.getcwd() index = preselector_config.index("src") - preselector_config = os.path.join( preselector_config[:index], "configs/preselector_config.json") + preselector_config = os.path.join( + preselector_config[:index], "configs/preselector_config.json" + ) logger.debug("Loading preselector config: " + preselector_config) - preselector = load_preselector(preselector_config=preselector_config, + preselector = load_preselector( + preselector_config=preselector_config, module="its_preselector.web_relay_preselector", - preselector_class_name = "WebRelayPreselector", sensor_definition={} + preselector_class_name="WebRelayPreselector", + sensor_definition={}, ) - assert preselector is not None \ No newline at end of file + assert preselector is not None diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index 290f8c7c..049ed192 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -310,7 +310,7 @@ scos-actions @ git+https://github.com/NTIA/scos-actions@8.0.0 # via # -r requirements.txt # scos-tekrsa -scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@discover_action_types +scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@5.0.0 # via -r requirements.txt sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive # via diff --git a/src/requirements.txt b/src/requirements.txt index 1890bf07..8853db10 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -146,7 +146,7 @@ scipy==1.10.1 # via scos-actions scos-actions @ git+https://github.com/NTIA/scos-actions@8.0.0 # via scos-tekrsa -scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@discover_action_types +scos-tekrsa @ git+https://github.com/NTIA/scos-tekrsa@5.0.0 # via -r requirements.in sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive # via scos-actions diff --git a/src/schedule/__init__.py b/src/schedule/__init__.py index cf61ad0b..cfd9f1f3 100644 --- a/src/schedule/__init__.py +++ b/src/schedule/__init__.py @@ -1,9 +1,8 @@ import logging +from initialization import action_loader from utils import get_summary -from django.conf import settings -from initialization import action_loader def get_action_with_summary(action): """Given an action, return the string 'action_name - summary'.""" @@ -18,4 +17,3 @@ def get_action_with_summary(action): logger = logging.getLogger(__name__) logger.debug("********** Initializing schedule **********") - diff --git a/src/schedule/migrations/0001_initial.py b/src/schedule/migrations/0001_initial.py index 4b0aa6c5..7938618c 100644 --- a/src/schedule/migrations/0001_initial.py +++ b/src/schedule/migrations/0001_initial.py @@ -1,6 +1,5 @@ # Generated by Django 3.0.4 on 2020-03-18 16:31 -import django.core.validators import django.db.models.deletion from django.conf import settings from django.db import migrations, models diff --git a/src/schedule/models/schedule_entry.py b/src/schedule/models/schedule_entry.py index dfba1c1a..1d4cdbe4 100644 --- a/src/schedule/models/schedule_entry.py +++ b/src/schedule/models/schedule_entry.py @@ -9,7 +9,6 @@ from initialization import action_loader from scheduler import utils - logger = logging.getLogger(__name__) logger.debug( "************** scos-sensor/schedule/models/schedule_entry.py *****************" diff --git a/src/schedule/serializers.py b/src/schedule/serializers.py index 03b13a59..2cd83769 100644 --- a/src/schedule/serializers.py +++ b/src/schedule/serializers.py @@ -6,6 +6,7 @@ convert_datetime_to_millisecond_iso_format, parse_datetime_iso_format_str, ) + from initialization import action_loader from sensor import V1 from sensor.utils import get_datetime_from_timestamp, get_timestamp_from_datetime @@ -13,7 +14,6 @@ from . import get_action_with_summary from .models import DEFAULT_PRIORITY, ScheduleEntry - action_help = "[Required] The name of the action to be scheduled" priority_help = f"Lower number is higher priority (default={DEFAULT_PRIORITY})" CHOICES = [] diff --git a/src/schedule/tests/test_user_views.py b/src/schedule/tests/test_user_views.py index 10b8c511..e0fe89e3 100644 --- a/src/schedule/tests/test_user_views.py +++ b/src/schedule/tests/test_user_views.py @@ -1,4 +1,3 @@ -import pytest from rest_framework import status from rest_framework.reverse import reverse diff --git a/src/scheduler/tests/test_scheduler.py b/src/scheduler/tests/test_scheduler.py index 195708d1..0dbede07 100644 --- a/src/scheduler/tests/test_scheduler.py +++ b/src/scheduler/tests/test_scheduler.py @@ -1,4 +1,3 @@ -import base64 import threading import time diff --git a/src/scheduler/tests/utils.py b/src/scheduler/tests/utils.py index 912dbdea..660a30af 100644 --- a/src/scheduler/tests/utils.py +++ b/src/scheduler/tests/utils.py @@ -4,12 +4,11 @@ import time from itertools import chain, count, islice -from initialization import action_loader from authentication.models import User +from initialization import action_loader from schedule.models import Request, ScheduleEntry from scheduler.scheduler import Scheduler from sensor import V1 -from scos_actions.hardware.mocks.mock_sigan import MockSignalAnalyzer logger = logging.getLogger(__name__) actions = action_loader.actions diff --git a/src/sensor/settings.py b/src/sensor/settings.py index 20154a19..cde89b2e 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -9,9 +9,6 @@ https://docs.djangoproject.com/en/1.11/ref/settings/ """ -import hashlib -import importlib -import json import logging import os import sys diff --git a/src/sensor/urls.py b/src/sensor/urls.py index 70f6ac23..772301a4 100644 --- a/src/sensor/urls.py +++ b/src/sensor/urls.py @@ -18,7 +18,6 @@ """ from django.conf import settings -from django.conf.urls.static import static from django.contrib import admin from django.urls import include, path, re_path from django.views.generic import RedirectView diff --git a/src/status/__init__.py b/src/status/__init__.py index 65d0294d..b6279e2e 100644 --- a/src/status/__init__.py +++ b/src/status/__init__.py @@ -1,5 +1,6 @@ import datetime import logging + from initialization import sensor_loader logger = logging.getLogger(__name__) @@ -7,4 +8,4 @@ if sensor_loader.sensor.signal_analyzer is not None: start_time = sensor_loader.sensor.start_time else: - start_time = datetime.datetime.utcnow() \ No newline at end of file + start_time = datetime.datetime.utcnow() diff --git a/src/status/apps.py b/src/status/apps.py index 0f213ea0..166ad0fd 100644 --- a/src/status/apps.py +++ b/src/status/apps.py @@ -1,24 +1,27 @@ import logging + from django.apps import AppConfig -from initialization import sensor_loader from scos_actions.metadata.utils import construct_geojson_point +from initialization import sensor_loader logger = logging.getLogger(__name__) + class StatusConfig(AppConfig): name = "status" def ready(self): from .models import Location + try: location = Location.objects.get(active=True) db_location_geojson = construct_geojson_point( - location.longitude, - location.latitude, - location.height + location.longitude, location.latitude, location.height + ) + logger.debug( + f"Location found in DB. Updating sensor location to {location}." ) - logger.debug(f"Location found in DB. Updating sensor location to {location}.") if sensor_loader.sensor is not None: sensor_loader.sensor.location = db_location_geojson except: diff --git a/src/status/views.py b/src/status/views.py index 61c90b64..db3abfdb 100644 --- a/src/status/views.py +++ b/src/status/views.py @@ -1,6 +1,7 @@ import datetime import logging import shutil + from its_preselector.preselector import Preselector from its_preselector.web_relay import WebRelay from rest_framework.decorators import api_view @@ -11,7 +12,7 @@ get_datetime_str_now, ) -from initialization import status_monitor, sensor_loader +from initialization import sensor_loader, status_monitor from scheduler import scheduler from . import start_time diff --git a/src/tasks/models/task.py b/src/tasks/models/task.py index 8e148073..b9d2c893 100644 --- a/src/tasks/models/task.py +++ b/src/tasks/models/task.py @@ -9,7 +9,6 @@ from initialization import action_loader - logger = logging.getLogger(__name__) logger.debug("*********** scos-sensor/models/task.py ****************") diff --git a/src/tasks/models/task_result.py b/src/tasks/models/task_result.py index b004bc69..4e106b80 100644 --- a/src/tasks/models/task_result.py +++ b/src/tasks/models/task_result.py @@ -3,11 +3,11 @@ import os import shutil +from django.conf import settings from django.db import models from django.utils import timezone from schedule.models import ScheduleEntry -from django.conf import settings from tasks.consts import MAX_DETAIL_LEN UTC = timezone.timezone.utc diff --git a/src/tasks/tests/test_archive_download.py b/src/tasks/tests/test_archive_download.py index 3ee78fe7..08899e49 100644 --- a/src/tasks/tests/test_archive_download.py +++ b/src/tasks/tests/test_archive_download.py @@ -1,7 +1,5 @@ -import os import tempfile -import numpy as np import sigmf.sigmffile from rest_framework import status diff --git a/src/tasks/views.py b/src/tasks/views.py index 56d63048..b48e9d42 100644 --- a/src/tasks/views.py +++ b/src/tasks/views.py @@ -1,5 +1,4 @@ import logging -import os import tempfile from functools import partial diff --git a/src/users/apps.py b/src/users/apps.py index 81d9339e..8b1a78a2 100644 --- a/src/users/apps.py +++ b/src/users/apps.py @@ -1,6 +1,5 @@ import logging import os -import sys from django.apps import AppConfig from django.conf import settings From e8ed7ba509829b8cc90cca4335e625ea670c2928 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 29 Jan 2024 15:23:15 -0700 Subject: [PATCH 234/255] update readme. --- README.md | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 478f0a9a..85357216 100644 --- a/README.md +++ b/README.md @@ -957,15 +957,19 @@ added to the plugin. Scos-sensor uses the following convention to discover actions offered by plugins: if any Python package begins with "scos_", and contains a dictionary of actions at the Python path `package_name.discover.actions`, these actions will automatically be -available for scheduling. Similarly, plugins may offer new action types be including +available for scheduling. Similarly, plugins may offer new action types by including a dictionary of action classes at the Python path `package_name.discover.action_classes`. -Scos-sensor will load all plugin actions and action classes prior to creating the actions -defined in yaml file in `configs/actions` directory. In this manner, a plugin may add new -action types to scos-sensor and those new types my instantiated/parameterized with yaml - -The scos-usrp plugin adds support for the Ettus B2xx line of signal analyzers. -It can also be used as an example of a plugin which adds new hardware support and -re-uses the common actions in scos-actions. +Scos-sensor will load all plugin actions and action classes prior to creating actions +defined in yaml files in `configs/actions` directory. In this manner, a plugin may add new +action types to scos-sensor and those new types may be instantiated/parameterized with yaml +config files. + +The [scos-usrp](https://github.com/ntia/scos-usrp) plugin adds support for the Ettus B2xx +line of signal analyzers and [scos-tekrsa](https://github.com/ntia/scos-tekrsa) adss +support for Tektronix RSA306, RSA306B, RSA503A, +RSA507A, RSA513A, RSA518A, RSA603A, and RSA607A real-time spectrum analyzers. +These repositories may also be used as examples of plugins which provide new hardware +support and re-use the common actions in scos-actions. For more information on adding actions and hardware support, see [scos-actions]( ). From 338eb0a2d2577c355ad0760e7d898518adc12a23 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 29 Jan 2024 15:37:32 -0700 Subject: [PATCH 235/255] readme --- README.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 85357216..8533f8b4 100644 --- a/README.md +++ b/README.md @@ -314,8 +314,13 @@ settings in the environment file: - ADMIN_PASSWORD: Password used to generate admin user. Change in production. - AUTHENTICATION: Authentication method used for scos-sensor. Supports `TOKEN` or `CERT`. -- BASE_IMAGE: Base docker image used to build the API container. Note, this should - updated when switching signal analyzers. +- BASE_IMAGE: Base docker image used to build the API container. These docker + images, combined with any drivers found in the signal analyzer repos, are + responsible for providing the operating system suitable for the chosen signal + analyzer. Note, this should be updated when switching signal analyzers. + By default, this is configured to + use a version of `ghcr.io/ntia/scos-tekrsa/tekrsa_usb` to use a Tektronix + signal analyzer. - CALLBACK_AUTHENTICATION: Sets how to authenticate to the callback URL. Supports `TOKEN` or `CERT`. - CALLBACK_SSL_VERIFICATION: Set to “true” in production environment. If false, the SSL @@ -352,17 +357,17 @@ settings in the environment file: . The env.template file sets to a randomly generated value. - SIGAN_CLASS: The name of the signal analyzer class to use. By default, this is - TekRSASigan to use a Tektronix signal analyzer. This must be changed to switch to +`TekRSASigan` to use a Tektronix signal analyzer. This must be changed to switch to a different signal analyzer. - SIGAN_MODULE: The name of the python module that provides the signal analyzer - implementation. This defaults to scos_tekrsa.hardware.tekrsa_sigan for the + implementation. This defaults to `scos_tekrsa.hardware.tekrsa_sigan` for the Tektronix signal analyzers. This must be changed to switch to a different signal analyzer. - SIGAN_POWER_CYCLE_STATES: Optional setting to provide the name of the control_state in the SIGAN_POWER_SWITCH that will power cycle the signal analyzer. - SIGAN_POWER_SWITCH: Optional setting used to indicate the name of a [WebRelay](https://github.com/NTIA/Preselector) that may be used to power cycle - the signal analyzer if necessary.Note: determinations of power cycling behavior + the signal analyzer if necessary. Note: specifics of power cycling behavior are implemented within the signal analyzer implementations or actions. - SSL_CA_PATH: Path to a CA certificate used to verify scos-sensor client certificate(s) when authentication is set to CERT. From 183e82c917855fee54530aef0f4d2dfd219b5e2f Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 30 Jan 2024 07:22:54 -0700 Subject: [PATCH 236/255] Update readme to describe changing sigan. --- README.md | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 8533f8b4..a3d3968f 100644 --- a/README.md +++ b/README.md @@ -351,16 +351,17 @@ settings in the environment file: - POSTGRES_PASSWORD: Sets password for the Postgres database for the “postgres” user. Change in production. The env.template file sets to a randomly generated value. - REPO_ROOT: Root folder of the repository. Should be correctly set by default. -- SCOS_SENSOR_GIT_TAG: The scos-sensor branch name. +- SCOS_SENSOR_GIT_TAG: The scos-sensor branch name. This value may be used in action + metadata to capture the version of the software that produced the sigmf archive. - SECRET_KEY: Used by Django to provide cryptographic signing. Change to a unique, unpredictable value. See . The env.template file sets to a randomly generated value. - SIGAN_CLASS: The name of the signal analyzer class to use. By default, this is -`TekRSASigan` to use a Tektronix signal analyzer. This must be changed to switch to - a different signal analyzer. + set to `TekRSASigan` to use a Tektronix signal analyzer. This must be changed + to switch to a different signal analyzer. - SIGAN_MODULE: The name of the python module that provides the signal analyzer - implementation. This defaults to `scos_tekrsa.hardware.tekrsa_sigan` for the + implementation. This defaults to `scos_tekrsa.hardware.tekrsa_sigan` for the Tektronix signal analyzers. This must be changed to switch to a different signal analyzer. - SIGAN_POWER_CYCLE_STATES: Optional setting to provide the name of the control_state @@ -979,6 +980,35 @@ support and re-use the common actions in scos-actions. For more information on adding actions and hardware support, see [scos-actions]( ). +### Switching Signal Analyzers + +Scos-sensor currently supports Ettus B2xx signal analyzers through +the [scos-usrp](https://github.com/ntia/scos-usrp) plugin and +Tektronix RSA306, RSA306B, RSA503A, RSA507A, RSA513A, +RSA518A, RSA603A, and RSA607A real-time spectrum analyzers through +the [scos-tekrsa](https://github.com/ntia/scos-tekrsa) plugin. To +configure scos-sensor for the desired signal analyzer review the +instructions in the plugin repository. Generally, +switching signal analyzers involves updating the `BASE_IMAGE` +setting, updating the requirements, and updating the `SIGAN_MODULE` +and `SIGAN_CLASS` settings. To identify the `BASE_IMAGE`, +go to the preferred plugin repository and find the latest docker image. +For example, see +[scos-tekrsa base images](https://github.com/NTIA/scos-tekrsa/pkgs/container/scos-tekrsa%2Ftekrsa_usb) +or +[scos-usrp base images](https://github.com/NTIA/scos-usrp/pkgs/container/scos-usrp%2Fscos_usrp_uhd). +Update the `BASE_IMAGE` setting in env file to the desired base image. +Then update the `SIGAN_MODULE` and `SIGAN_CLASS` settings with +the appropriate Python module and class that provide +an implementation of the `SignalAnalyzerInterface` +(you will have to look in the plugin repo to identify the correct module and class). Finally, +update the requirements with the selected plugin repo. +See [Requirements and Configuration](https://github.com/NTIA/scos-sensor?tab=readme-ov-file#requirements-and-configuration) +and [Using pip-tools](https://github.com/NTIA/scos-sensor?tab=readme-ov-file#using-pip-tools) +for additional information. Be sure to re-source the environment file, update the +requirements files, and prune any existing containers +before rebuilding scos-sensor. + ## Preselector Support Scos-sensor can be configured to support From c7f040e3911e18eb0d9dd5b602afa3529e16fd87 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 30 Jan 2024 07:27:16 -0700 Subject: [PATCH 237/255] Add note to update USB_DEVICE when switching signal analyzers. --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a3d3968f..47f1fbfe 100644 --- a/README.md +++ b/README.md @@ -990,10 +990,10 @@ the [scos-tekrsa](https://github.com/ntia/scos-tekrsa) plugin. To configure scos-sensor for the desired signal analyzer review the instructions in the plugin repository. Generally, switching signal analyzers involves updating the `BASE_IMAGE` -setting, updating the requirements, and updating the `SIGAN_MODULE` -and `SIGAN_CLASS` settings. To identify the `BASE_IMAGE`, -go to the preferred plugin repository and find the latest docker image. -For example, see +setting, updating the requirements, and updating the `SIGAN_MODULE`, +`SIGAN_CLASS`, and `USB_DEVICE` settings. To identify the +`BASE_IMAGE`, go to the preferred plugin repository and find +the latest docker image. For example, see [scos-tekrsa base images](https://github.com/NTIA/scos-tekrsa/pkgs/container/scos-tekrsa%2Ftekrsa_usb) or [scos-usrp base images](https://github.com/NTIA/scos-usrp/pkgs/container/scos-usrp%2Fscos_usrp_uhd). From db2e52532fcbbd3b5281950d5c15d6ede14cb855 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 30 Jan 2024 07:30:30 -0700 Subject: [PATCH 238/255] Remove unused imports and logger. --- gunicorn/config.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index 2d82f58a..5efc64fb 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -1,18 +1,15 @@ -import importlib -import logging import os import sys from multiprocessing import cpu_count - bind = ":8000" workers = 1 worker_class = "gthread" threads = cpu_count() loglevel = os.environ.get("GUNICORN_LOG_LEVEL", "info") -logger = logging.getLogger(__name__) + def _modify_path(): """Ensure Django project is on sys.path.""" @@ -28,8 +25,10 @@ def post_worker_init(worker): _modify_path() os.environ.setdefault("DJANGO_SETTINGS_MODULE", "sensor.settings") import django + django.setup() from scheduler import scheduler + scheduler.thread.start() From 274ceffec6ad219fafbaa430d8602c1f669327d6 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Tue, 30 Jan 2024 07:32:35 -0700 Subject: [PATCH 239/255] Add create_superuser.py back to allow manual creation of additional users. --- scripts/create_superuser.py | 73 +++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 scripts/create_superuser.py diff --git a/scripts/create_superuser.py b/scripts/create_superuser.py new file mode 100644 index 00000000..522825b0 --- /dev/null +++ b/scripts/create_superuser.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 + +import os +import sys + +import django +from django.contrib.auth import get_user_model # noqa + +PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "src") + +sys.path.append(PATH) + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "sensor.settings") +django.setup() + +UserModel = get_user_model() + + +def add_user(username, password, email=None): + try: + admin_user = UserModel._default_manager.get(username=username) + if email: + admin_user.email = email + admin_user.set_password(password) + admin_user.save() + print("Reset admin account password and email from environment") + except UserModel.DoesNotExist: + UserModel._default_manager.create_superuser(username, email, password) + print("Created admin account with password and email from environment") + + +try: + password = os.environ["ADMIN_PASSWORD"] + print("Retreived admin password from environment variable ADMIN_PASSWORD") + email = os.environ["ADMIN_EMAIL"] + print("Retreived admin email from environment variable ADMIN_EMAIL") + username = os.environ["ADMIN_NAME"] + print("Retreived admin name from environment variable ADMIN_NAME") + add_user(username.strip(), password.strip(), email.strip()) +except KeyError: + print("Not on a managed sensor, so not auto-generating admin account.") + print("You can add an admin later with `./manage.py createsuperuser`") + sys.exit(0) + +additional_user_names = "" +additional_user_password = "" +try: + additional_user_names = os.environ["ADDITIONAL_USER_NAMES"] + print( + "Retreived additional user names from environment variable ADDITIONAL_USER_NAMES" + ) + if ( + "ADDITIONAL_USER_PASSWORD" in os.environ + and os.environ["ADDITIONAL_USER_PASSWORD"] + ): + additional_user_password = os.environ["ADDITIONAL_USER_PASSWORD"].strip() + else: + # user will have unusable password + # https://docs.djangoproject.com/en/3.2/ref/contrib/auth/#django.contrib.auth.models.UserManager.create_user + additional_user_password = None + print( + "Retreived additional user password from environment variable ADDITIONAL_USER_PASSWORD" + ) +except KeyError: + print("Not creating any additonal users.") + + +if additional_user_names != "" and additional_user_password != "": + if "," in additional_user_names: + for additional_user_name in additional_user_names.split(","): + add_user(additional_user_name.strip(), additional_user_password) + else: + add_user(additional_user_names.strip(), additional_user_password) From f0e922df12df6f3b2346826799687541958dd0d3 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 30 Jan 2024 12:46:59 -0500 Subject: [PATCH 240/255] fix typo, reduce size of calibration example --- README.md | 172 +----------------------------------------------------- 1 file changed, 2 insertions(+), 170 deletions(-) diff --git a/README.md b/README.md index 47f1fbfe..8e07fc03 100644 --- a/README.md +++ b/README.md @@ -422,7 +422,7 @@ specific to the sensor you are using. By default, scos-sensor will use `configs/default_calibration.json` as the sensor calibration file. However, if`configs/sensor_calibration.json` or `configs/sigan_calibration.json` exist they will be used instead of the default -calibration file. Sensor calibration files allow scos-sensor to pply a gain based +calibration file. Sensor calibration files allow scos-sensor to apply a gain based on a laboratory calibration of the sensor and may also contain other useful metadata that characterizes the sensor performance. For additional information on the calibration data, see the @@ -507,181 +507,13 @@ per second at several frequencies with a signal analyzer reference level setting } } } - }, - "3575000000.0": { - "-25": { - "true": { - "0": { - "datetime": "2023-10-23T14:38:16.211Z", - "gain": 30.73297444891243, - "noise_figure": 4.090843866619065, - "temperature": 15.6 - } - } - } - }, - "3585000000.0": { - "-25": { - "true": { - "0": { - "datetime": "2023-10-23T14:38:20.725Z", - "gain": 30.884253019974623, - "noise_figure": 3.934553150614483, - "temperature": 15.6 - } - } - } - }, - "3595000000.0": { - "-25": { - "true": { - "0": { - "datetime": "2023-10-23T14:38:24.606Z", - "gain": 31.002780356672476, - "noise_figure": 3.940238988552726, - "temperature": 15.6 - } - } - } - }, - "3605000000.0": { - "-25": { - "true": { - "0": { - "datetime": "2023-10-23T14:38:29.724Z", - "gain": 31.035560147646778, - "noise_figure": 3.9290832485193989, - "temperature": 15.6 - } - } - } - }, - "3615000000.0": { - "-25": { - "true": { - "0": { - "datetime": "2023-10-23T14:38:33.614Z", - "gain": 30.935970273145274, - "noise_figure": 4.006672278350428, - "temperature": 15.6 - } - } - } - }, - "3625000000.0": { - "-25": { - "true": { - "0": { - "datetime": "2023-10-23T14:38:37.592Z", - "gain": 30.682403307202095, - "noise_figure": 4.064067195729546, - "temperature": 15.6 - } - } - } - }, - "3635000000.0": { - "-25": { - "true": { - "0": { - "datetime": "2023-10-23T14:38:41.889Z", - "gain": 30.980340383458626, - "noise_figure": 3.8826926914916726, - "temperature": 15.6 - } - } - } - }, - "3645000000.0": { - "-25": { - "true": { - "0": { - "datetime": "2023-10-23T14:38:46.450Z", - "gain": 30.948486357958318, - "noise_figure": 3.917520438472101, - "temperature": 15.6 - } - } - } - }, - "3655000000.0": { - "-25": { - "true": { - "0": { - "datetime": "2023-10-23T14:38:51.575Z", - "gain": 30.96859101287636, - "noise_figure": 3.926017393059535, - "temperature": 15.6 - } - } - } - }, - "3665000000.0": { - "-25": { - "true": { - "0": { - "datetime": "2023-10-23T14:38:55.571Z", - "gain": 30.618851884796429, - "noise_figure": 4.225928655860898, - "temperature": 15.6 - } - } - } - }, - "3675000000.0": { - "-25": { - "true": { - "0": { - "datetime": "2023-10-23T14:38:59.585Z", - "gain": 30.552247661443276, - "noise_figure": 4.171389553033189, - "temperature": 15.6 - } - } - } - }, - "3685000000.0": { - "-25": { - "true": { - "0": { - "datetime": "2023-10-23T14:39:03.921Z", - "gain": 30.413415948237785, - "noise_figure": 4.42128294424517, - "temperature": 15.6 - } - } - } - }, - "3695000000.0": { - "-25": { - "true": { - "0": { - "datetime": "2023-10-23T14:39:09.115Z", - "gain": 30.112285957420995, - "noise_figure": 4.576464590847393, - "temperature": 15.6 - } - } - } - }, - "3705000000.0": { - "-25": { - "true": { - "0": { - "datetime": "2023-10-23T14:39:13.682Z", - "gain": 29.120051306906164, - "noise_figure": 5.2261432005125709, - "temperature": 15.7 - } - } - } } } } } ``` -When an action is run with the above calibration, scos will expect the action to have +When an action is run with the above calibration, SCOS will expect the action to have a sample_rate, frequency, and reference_level specified in the action config. The values specified for these parameters will then be used to retrieve the calibration data. From 9d42b8d8a9e7d71e5218d7d555c521977ec58694 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 30 Jan 2024 13:09:27 -0500 Subject: [PATCH 241/255] remove unused code --- src/handlers/tests/test_handlers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/handlers/tests/test_handlers.py b/src/handlers/tests/test_handlers.py index ca049547..f8e5bf17 100644 --- a/src/handlers/tests/test_handlers.py +++ b/src/handlers/tests/test_handlers.py @@ -11,7 +11,6 @@ @pytest.mark.django_db def test_db_location_update_handler(): - location = construct_geojson_point(-105.7, 40.5, 0) sensor = sensor_loader.sensor logger.debug(f"Sensor: {sensor}") location = Location() @@ -68,7 +67,6 @@ def test_db_location_update_handler_not_active(): @pytest.mark.django_db def test_db_location_deleted_handler(): - location = construct_geojson_point(-105.7, 40.5, 0) sensor = sensor_loader.sensor location = Location() location.gps = False From b0922e4bd16b2498d1e62d13fe5825586e0a56f5 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 30 Jan 2024 13:11:49 -0500 Subject: [PATCH 242/255] remove placeholder debug message --- src/initialization/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 1f5585a3..9d4b71e2 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -47,7 +47,6 @@ def set_container_unhealthy(): usb_device_exists = usb_device_exists() if usb_device_exists: action_loader = ActionLoader() - logger.debug("test") logger.debug(f"Actions ActionLoader has {len(action_loader.actions)} actions") capabilities_loader = CapabilitiesLoader() logger.debug("Calling sensor loader.") From b615e6c3fab2fbf03d7824cdba9fc4eb47ca7578 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 30 Jan 2024 13:15:48 -0500 Subject: [PATCH 243/255] avoid using function name as variable name --- src/initialization/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 9d4b71e2..723d58e6 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -18,7 +18,7 @@ status_monitor = StatusMonitor() -def usb_device_exists() -> bool: +def get_usb_device_exists() -> bool: logger.debug("Checking for USB...") if not settings.RUNNING_TESTS and settings.USB_DEVICE is not None: usb_devices = check_output("lsusb").decode(sys.stdout.encoding) @@ -44,7 +44,7 @@ def set_container_unhealthy(): try: register_component_with_status.connect(status_registration_handler) - usb_device_exists = usb_device_exists() + usb_device_exists = get_usb_device_exists() if usb_device_exists: action_loader = ActionLoader() logger.debug(f"Actions ActionLoader has {len(action_loader.actions)} actions") From ac153954e56127e57b7637e9441edd575e6cd830 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 30 Jan 2024 13:18:33 -0500 Subject: [PATCH 244/255] consistent use of logger.exception --- src/initialization/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/initialization/__init__.py b/src/initialization/__init__.py index 723d58e6..e97def1e 100644 --- a/src/initialization/__init__.py +++ b/src/initialization/__init__.py @@ -69,5 +69,5 @@ def set_container_unhealthy(): sensor_loader.capabilities = {} logger.warning("Usb is not ready. Marking container as unhealthy") set_container_unhealthy() -except Exception as ex: - logger.error(f"Error during initialization: {ex}") +except: + logger.exception("Error during initialization") From e9bb866dd2f50b35e73aed9b909fa061e0daff59 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 30 Jan 2024 13:54:31 -0500 Subject: [PATCH 245/255] load test actions from any SCOS plugin, or local --- src/initialization/action_loader.py | 49 ++++++++++++++++------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/src/initialization/action_loader.py b/src/initialization/action_loader.py index f94fb5a1..3753f2cc 100644 --- a/src/initialization/action_loader.py +++ b/src/initialization/action_loader.py @@ -9,7 +9,7 @@ from django.conf import settings from scos_actions.actions import action_classes from scos_actions.actions.interfaces.action import Action -from scos_actions.discover import init, test_actions +from scos_actions.discover import init logger = logging.getLogger(__name__) @@ -93,28 +93,35 @@ def load_actions( for finder, name, ispkg in pkgutil.iter_modules() if name.startswith("scos_") and name != "scos_actions" } - logger.debug(discovered_plugins) + logger.debug(f"Discovered SCOS plugins: {discovered_plugins}") actions = {} - if mock_sigan or running_tests: - logger.debug(f"Loading {len(test_actions)} test actions.") - actions.update(test_actions) - else: - for name, module in discovered_plugins.items(): - logger.debug("Looking for actions in " + name + ": " + str(module)) - discover = importlib.import_module(name + ".discover") - if hasattr(discover, "actions"): - logger.debug(f"loading {len(discover.actions)} actions.") - actions.update(discover.actions) - if ( - hasattr(discover, "action_classes") - and discover.action_classes is not None - ): - action_classes.update(discover.action_classes) - logger.debug(f"Loading actions in {action_dir}") - yaml_actions, yaml_test_actions = init( + for name, module in discovered_plugins.items(): + # Load action classes from discovered plugins + logger.debug(f"Looking for action classes in {name}: {module}") + discover = importlib.import_module(name + ".discover") + if hasattr(discover, "action_classes") and discover.action_classes is not None: + action_classes.update(discover.action_classes) + if mock_sigan or running_tests: + # Load test action YAMLs from discovered plugins + logger.debug(f"Looking for test actions in {name}: {module}") + if hasattr(discover, "test_actions") and discover.test_actions is not None: + actions.update(discover.test_actions) + else: + # Load non-testing action YAMLS from discovered plugins + logger.debug(f"Looking for actions in {name}: {module}") + if hasattr(discover, "actions") and discover.actions is not None: + actions.update(discover.actions) + # Load actions or test actions from local action_dir + local_actions, local_test_actions = init( action_classes=action_classes, yaml_dir=action_dir ) - actions.update(yaml_actions) - logger.debug("Finished loading and registering actions") + if mock_sigan or running_tests: + logger.debug(f"Loading test actions in {action_dir}") + actions.update(local_test_actions) + else: + logger.debug(f"Loading actions in {action_dir}") + actions.update(local_actions) + + logger.debug("Finished loading and registering actions") return actions From 5a7d3694e0f9ceabffc9fead588a26027b14d68d Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 30 Jan 2024 15:08:26 -0500 Subject: [PATCH 246/255] debug when registered component has no get_status --- src/initialization/status_monitor.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/initialization/status_monitor.py b/src/initialization/status_monitor.py index 64340c79..55a909e9 100644 --- a/src/initialization/status_monitor.py +++ b/src/initialization/status_monitor.py @@ -35,3 +35,8 @@ def add_component(self, component): """ if hasattr(component, "get_status"): self._status_components.append(component) + else: + logger.debug( + "Provided component has no `get_status` method and was not registered" + + f" with the status monitor: {component}" + ) From 9965e1a0b9045565ad6ed6c22c9d78415139fb78 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 30 Jan 2024 16:00:15 -0500 Subject: [PATCH 247/255] Revert "load test actions from any SCOS plugin, or local" This reverts commit e9bb866dd2f50b35e73aed9b909fa061e0daff59. --- src/initialization/action_loader.py | 49 +++++++++++++---------------- 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/src/initialization/action_loader.py b/src/initialization/action_loader.py index 3753f2cc..f94fb5a1 100644 --- a/src/initialization/action_loader.py +++ b/src/initialization/action_loader.py @@ -9,7 +9,7 @@ from django.conf import settings from scos_actions.actions import action_classes from scos_actions.actions.interfaces.action import Action -from scos_actions.discover import init +from scos_actions.discover import init, test_actions logger = logging.getLogger(__name__) @@ -93,35 +93,28 @@ def load_actions( for finder, name, ispkg in pkgutil.iter_modules() if name.startswith("scos_") and name != "scos_actions" } - logger.debug(f"Discovered SCOS plugins: {discovered_plugins}") + logger.debug(discovered_plugins) actions = {} - - for name, module in discovered_plugins.items(): - # Load action classes from discovered plugins - logger.debug(f"Looking for action classes in {name}: {module}") - discover = importlib.import_module(name + ".discover") - if hasattr(discover, "action_classes") and discover.action_classes is not None: - action_classes.update(discover.action_classes) - if mock_sigan or running_tests: - # Load test action YAMLs from discovered plugins - logger.debug(f"Looking for test actions in {name}: {module}") - if hasattr(discover, "test_actions") and discover.test_actions is not None: - actions.update(discover.test_actions) - else: - # Load non-testing action YAMLS from discovered plugins - logger.debug(f"Looking for actions in {name}: {module}") - if hasattr(discover, "actions") and discover.actions is not None: - actions.update(discover.actions) - # Load actions or test actions from local action_dir - local_actions, local_test_actions = init( - action_classes=action_classes, yaml_dir=action_dir - ) if mock_sigan or running_tests: - logger.debug(f"Loading test actions in {action_dir}") - actions.update(local_test_actions) + logger.debug(f"Loading {len(test_actions)} test actions.") + actions.update(test_actions) else: - logger.debug(f"Loading actions in {action_dir}") - actions.update(local_actions) + for name, module in discovered_plugins.items(): + logger.debug("Looking for actions in " + name + ": " + str(module)) + discover = importlib.import_module(name + ".discover") + if hasattr(discover, "actions"): + logger.debug(f"loading {len(discover.actions)} actions.") + actions.update(discover.actions) + if ( + hasattr(discover, "action_classes") + and discover.action_classes is not None + ): + action_classes.update(discover.action_classes) - logger.debug("Finished loading and registering actions") + logger.debug(f"Loading actions in {action_dir}") + yaml_actions, yaml_test_actions = init( + action_classes=action_classes, yaml_dir=action_dir + ) + actions.update(yaml_actions) + logger.debug("Finished loading and registering actions") return actions From 530b16fbec0c8e0c9b35ac7fad8d9c6cf1e3b347 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 30 Jan 2024 17:57:57 -0500 Subject: [PATCH 248/255] import settings from django.conf --- src/sensor/templatetags/sensor_tags.py | 5 ++--- src/status/migrations/0003_auto_20211217_2229.py | 5 ++--- src/tasks/tests/test_archive_download.py | 8 ++++---- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/sensor/templatetags/sensor_tags.py b/src/sensor/templatetags/sensor_tags.py index 3ebdbd1e..9c28caa4 100644 --- a/src/sensor/templatetags/sensor_tags.py +++ b/src/sensor/templatetags/sensor_tags.py @@ -1,10 +1,9 @@ from django import template - -from sensor.settings import VERSION_STRING +from django.conf import settings register = template.Library() @register.simple_tag def sensor_version_string(): - return VERSION_STRING + return settings.VERSION_STRING diff --git a/src/status/migrations/0003_auto_20211217_2229.py b/src/status/migrations/0003_auto_20211217_2229.py index 45c3648c..f8b9b164 100644 --- a/src/status/migrations/0003_auto_20211217_2229.py +++ b/src/status/migrations/0003_auto_20211217_2229.py @@ -2,13 +2,12 @@ import json +from django.conf import settings from django.db import migrations -from sensor.settings import SENSOR_DEFINITION_FILE - def load_location(apps, schema_editor): - with open(SENSOR_DEFINITION_FILE) as f: + with open(settings.SENSOR_DEFINITION_FILE) as f: sensor_def = json.load(f) if "location" in sensor_def: location = sensor_def["location"] diff --git a/src/tasks/tests/test_archive_download.py b/src/tasks/tests/test_archive_download.py index 08899e49..f031879b 100644 --- a/src/tasks/tests/test_archive_download.py +++ b/src/tasks/tests/test_archive_download.py @@ -1,9 +1,9 @@ import tempfile import sigmf.sigmffile +from django.conf import settings from rest_framework import status -import sensor.settings from test_utils.task_test_utils import ( HTTPS_KWARG, reverse_archive, @@ -18,7 +18,7 @@ def test_single_acquisition_archive_download(admin_client, test_scheduler): task_id = 1 url = reverse_archive(entry_name, task_id) disposition = 'attachment; filename="{}_test_acq_1.sigmf"' - disposition = disposition.format(sensor.settings.FQDN) + disposition = disposition.format(settings.FQDN) response = admin_client.get(url, **HTTPS_KWARG) assert response.status_code == status.HTTP_200_OK @@ -44,7 +44,7 @@ def test_multirec_acquisition_archive_download(admin_client, test_scheduler): task_id = 1 url = reverse_archive(entry_name, task_id) disposition = 'attachment; filename="{}_test_multirec_acq_1.sigmf"' - disposition = disposition.format(sensor.settings.FQDN) + disposition = disposition.format(settings.FQDN) response = admin_client.get(url, **HTTPS_KWARG) assert response.status_code == status.HTTP_200_OK @@ -64,7 +64,7 @@ def test_all_acquisitions_archive_download(admin_client, test_scheduler, tmpdir) entry_name = simulate_frequency_fft_acquisitions(admin_client, n=3) url = reverse_archive_all(entry_name) disposition = 'attachment; filename="{}_test_multiple_acq.sigmf"' - disposition = disposition.format(sensor.settings.FQDN) + disposition = disposition.format(settings.FQDN) response = admin_client.get(url, **HTTPS_KWARG) assert response.status_code == status.HTTP_200_OK From 4e86cf82d7b73bb5799cd01219b67b5c82e571fd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 30 Jan 2024 19:38:24 -0500 Subject: [PATCH 249/255] Bump aiohttp from 3.9.1 to 3.9.2 in /src (#268) * Bump aiohttp from 3.9.1 to 3.9.2 in /src Bumps [aiohttp](https://github.com/aio-libs/aiohttp) from 3.9.1 to 3.9.2. - [Release notes](https://github.com/aio-libs/aiohttp/releases) - [Changelog](https://github.com/aio-libs/aiohttp/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/aiohttp/compare/v3.9.1...v3.9.2) --- updated-dependencies: - dependency-name: aiohttp dependency-type: direct:development ... Signed-off-by: dependabot[bot] * Update requirements-dev.txt and recompile --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Anthony Romaniello --- src/requirements-dev.in | 2 +- src/requirements-dev.txt | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/requirements-dev.in b/src/requirements-dev.in index 3c586abb..8fe2d167 100644 --- a/src/requirements-dev.in +++ b/src/requirements-dev.in @@ -9,4 +9,4 @@ tox>=4.0,<5.0 # The following are sub-dependencies for which SCOS Sensor enforces a # higher minimum patch version than the dependencies which require them. # This is done to ensure the inclusion of specific security patches. -aiohttp>=3.9.0 # CVE-2023-37276 +aiohttp>=3.9.2 # CVE-2023-37276 diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index e8ef0921..3b95e482 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -4,7 +4,7 @@ # # pip-compile requirements-dev.in # -aiohttp==3.9.1 +aiohttp==3.9.3 # via # -r requirements-dev.in # aiohttp-cors @@ -59,7 +59,9 @@ colorama==0.4.6 colorful==0.5.5 # via ray coverage[toml]==7.3.2 - # via pytest-cov + # via + # coverage + # pytest-cov cryptography==41.0.7 # via -r requirements.txt defusedxml==0.7.1 @@ -362,7 +364,7 @@ virtualenv==20.21.0 # tox wcwidth==0.2.12 # via blessed -yarl==1.9.3 +yarl==1.9.4 # via aiohttp zipp==3.17.0 # via From 4dcfd4faeeb23a1587c1294161f549c3280cc55d Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Wed, 31 Jan 2024 10:25:24 -0700 Subject: [PATCH 250/255] Don't set default values for SIGAN_MODULE, SIGAN_CLASS, or DEVICE_MODEL. Add check and raise exception if SIGAN_MODULE or SIGAN_CLASS are not set. Don't set sensor_sha512 if unable to generate hash. Add DEVICE_MODEL to readme and env.template. --- README.md | 3 +++ env.template | 2 ++ src/initialization/capabilities_loader.py | 2 -- src/initialization/sensor_loader.py | 14 ++++++++++++++ src/sensor/settings.py | 6 +++--- 5 files changed, 22 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8e07fc03..4bd31ab4 100644 --- a/README.md +++ b/README.md @@ -328,6 +328,9 @@ settings in the environment file: - CALLBACK_TIMEOUT: The timeout for the posts sent to the callback URL when a scheduled action is completed. - DEBUG: Django debug mode. Set to False in production. +- DEVICE_MODEL: Optional setting indicating the model of the signal analyzer. The + TekRSASigan class will use this value to determine which action configs to load. + See [scos-tekrsa](https://github.com/ntia/scos-tekrsa) for additional details. - DOCKER_TAG: Always set to “latest” to install newest version of docker containers. - DOMAINS: A space separated list of domain names. Used to generate [ALLOWED_HOSTS]( ). diff --git a/env.template b/env.template index d4820693..f78e123e 100644 --- a/env.template +++ b/env.template @@ -34,6 +34,8 @@ CALLBACK_TIMEOUT=2 # Use either true or false DEBUG=true +DEVICE_MODEL=RSA507A + # Use latest as default for local development DOCKER_TAG=latest diff --git a/src/initialization/capabilities_loader.py b/src/initialization/capabilities_loader.py index 4c8d2059..4936c295 100644 --- a/src/initialization/capabilities_loader.py +++ b/src/initialization/capabilities_loader.py @@ -54,8 +54,6 @@ def load_capabilities(sensor_definition_file: str) -> dict: ).hexdigest() capabilities["sensor"]["sensor_sha512"] = sensor_definition_hash except: - capabilities["sensor"]["sensor_sha512"] = "ERROR GENERATING HASH" - # sensor_sha512 is None, do not raise Exception, but log it logger.exception(f"Unable to generate sensor definition hash") return capabilities diff --git a/src/initialization/sensor_loader.py b/src/initialization/sensor_loader.py index 16c0c539..4ce54936 100644 --- a/src/initialization/sensor_loader.py +++ b/src/initialization/sensor_loader.py @@ -72,6 +72,7 @@ def load_sensor(sensor_capabilities: dict) -> Sensor: sigan = None try: if not settings.RUNNING_MIGRATIONS: + check_for_required_sigan_settings() sigan_module_setting = settings.SIGAN_MODULE sigan_module = importlib.import_module(sigan_module_setting) logger.info( @@ -97,6 +98,19 @@ def load_sensor(sensor_capabilities: dict) -> Sensor: return sensor +def check_for_required_sigan_settings(): + error = "" + raise_exception = False + if settings.SIGAN_MODULE is None: + raise_exception = True + error = "SIGAN_MODULE environment variable must be set. " + if settings.SIGAN_CLASS is None: + raise_exception = True + error += "SIGAN_CLASS environment variable. " + if raise_exception: + raise Exception(error) + + def load_switches(switch_dir: Path) -> dict: logger.debug(f"Loading switches in {switch_dir}") switch_dict = {} diff --git a/src/sensor/settings.py b/src/sensor/settings.py index cde89b2e..e17c2c6a 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -345,9 +345,9 @@ "PORT": "5432", } } - DEVICE_MODEL = env("DEVICE_MODEL", default="RSA507A") - SIGAN_MODULE = env.str("SIGAN_MDOULE", default="scos_tekrsa.hardware.tekrsa_sigan") - SIGAN_CLASS = env.str("SIGAN_CLASS", default="TekRSASigan") + DEVICE_MODEL = env("DEVICE_MODEL", default=None) + SIGAN_MODULE = env.str("SIGAN_MDOULE", default=None) + SIGAN_CLASS = env.str("SIGAN_CLASS", default=None) if not IN_DOCKER: DATABASES["default"]["HOST"] = "localhost" From c3b95bdc0bd510233d1dfd5dea737c0be542ccec Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Wed, 31 Jan 2024 10:49:21 -0700 Subject: [PATCH 251/255] Add actions folder with readme. --- configs/actions/readme.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 configs/actions/readme.md diff --git a/configs/actions/readme.md b/configs/actions/readme.md new file mode 100644 index 00000000..f0d1e84c --- /dev/null +++ b/configs/actions/readme.md @@ -0,0 +1,3 @@ +# Actions + +Add yaml configs for actions in this directory to create additional actions. From 7de7c4ebd2c0c493a56eb0db2657a96b6932612b Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 1 Feb 2024 15:58:14 -0700 Subject: [PATCH 252/255] typo fix. --- src/sensor/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sensor/settings.py b/src/sensor/settings.py index e17c2c6a..1bc97746 100644 --- a/src/sensor/settings.py +++ b/src/sensor/settings.py @@ -346,7 +346,7 @@ } } DEVICE_MODEL = env("DEVICE_MODEL", default=None) - SIGAN_MODULE = env.str("SIGAN_MDOULE", default=None) + SIGAN_MODULE = env.str("SIGAN_MODULE", default=None) SIGAN_CLASS = env.str("SIGAN_CLASS", default=None) if not IN_DOCKER: From afcf2bb8900e9431b95644dedd6b6caebf062b6f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Feb 2024 13:08:26 -0500 Subject: [PATCH 253/255] Bump cryptography from 41.0.7 to 42.0.0 in /src (#269) * Bump cryptography from 41.0.7 to 42.0.0 in /src Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.7 to 42.0.0. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/41.0.7...42.0.0) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Bump cryptography minimum version --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Anthony Romaniello --- src/requirements-dev.txt | 2 +- src/requirements.in | 2 +- src/requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index e6587fc2..75b7616d 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -62,7 +62,7 @@ coverage[toml]==7.3.2 # via # coverage # pytest-cov -cryptography==41.0.7 +cryptography==42.0.0 # via -r requirements.txt defusedxml==0.7.1 # via diff --git a/src/requirements.in b/src/requirements.in index bc87c3d5..a5658a69 100644 --- a/src/requirements.in +++ b/src/requirements.in @@ -1,4 +1,4 @@ -cryptography>=41.0.6 +cryptography>=42.0.0 django>=3.2.23, <4.0 djangorestframework>=3.0, <4.0 django-session-timeout>=0.1, <1.0 diff --git a/src/requirements.txt b/src/requirements.txt index fffdf660..38812cc3 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -20,7 +20,7 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via ray -cryptography==41.0.7 +cryptography==42.0.0 # via -r requirements.in defusedxml==0.7.1 # via its-preselector From a3e8c7525add58efb70d9f94ec4d7a93388c02b0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Feb 2024 15:18:37 -0500 Subject: [PATCH 254/255] Bump cryptography from 42.0.0 to 42.0.4 in /src (#273) * Bump cryptography from 42.0.0 to 42.0.4 in /src Bumps [cryptography](https://github.com/pyca/cryptography) from 42.0.0 to 42.0.4. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/42.0.0...42.0.4) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Require cryptography>=42.0.4 --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Anthony Romaniello --- src/requirements-dev.txt | 2 +- src/requirements.in | 2 +- src/requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index 75b7616d..159e817f 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -62,7 +62,7 @@ coverage[toml]==7.3.2 # via # coverage # pytest-cov -cryptography==42.0.0 +cryptography==42.0.4 # via -r requirements.txt defusedxml==0.7.1 # via diff --git a/src/requirements.in b/src/requirements.in index a5658a69..03baa030 100644 --- a/src/requirements.in +++ b/src/requirements.in @@ -1,4 +1,4 @@ -cryptography>=42.0.0 +cryptography>=42.0.4 django>=3.2.23, <4.0 djangorestframework>=3.0, <4.0 django-session-timeout>=0.1, <1.0 diff --git a/src/requirements.txt b/src/requirements.txt index 38812cc3..e4ef831f 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -20,7 +20,7 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via ray -cryptography==42.0.0 +cryptography==42.0.4 # via -r requirements.in defusedxml==0.7.1 # via its-preselector From 7c23a7c6decfc8345c5685a289dde509d73ff82a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Feb 2024 15:30:02 -0500 Subject: [PATCH 255/255] Bump django from 3.2.23 to 3.2.24 in /src (#270) * Bump django from 3.2.23 to 3.2.24 in /src Bumps [django](https://github.com/django/django) from 3.2.23 to 3.2.24. - [Commits](https://github.com/django/django/compare/3.2.23...3.2.24) --- updated-dependencies: - dependency-name: django dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Require django>=3.2.24 --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Anthony Romaniello --- src/requirements-dev.txt | 2 +- src/requirements.in | 2 +- src/requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index 159e817f..6f50321a 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -70,7 +70,7 @@ defusedxml==0.7.1 # its-preselector distlib==0.3.7 # via virtualenv -django==3.2.23 +django==3.2.24 # via # -r requirements.txt # django-session-timeout diff --git a/src/requirements.in b/src/requirements.in index 03baa030..591bc41f 100644 --- a/src/requirements.in +++ b/src/requirements.in @@ -1,5 +1,5 @@ cryptography>=42.0.4 -django>=3.2.23, <4.0 +django>=3.2.24, <4.0 djangorestframework>=3.0, <4.0 django-session-timeout>=0.1, <1.0 drf-yasg>=1.0, <2.0 diff --git a/src/requirements.txt b/src/requirements.txt index e4ef831f..59c9699d 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -24,7 +24,7 @@ cryptography==42.0.4 # via -r requirements.in defusedxml==0.7.1 # via its-preselector -django==3.2.23 +django==3.2.24 # via # -r requirements.in # django-session-timeout