Merge pull request #51 from 4dn-dcic/local-check-runner

Local check runner

CHANGELOG.rst

@@ -6,6 +6,19 @@ foursight-core
 Change Log
 ----------
 
+4.5.0
+=====
+* 2023-08
+* Support local-check-runner utility.
+* Minor change to respect REDIS_HOST_LOCAL environment variable (for local dev/testing),
+  as well as allowing override of Auth0 client/secret (AUTH0_CLIENT_LOCAL, AUTH0_SECRET_LOCAL).
+* Miscellaneous changes to get foursight-smaht working properly.
+  * Support to get consorita/submission_centers, as well as for
+    awards/labs for foursight-fourfront (previous oversight).
+  * Fixed up users pages.
+* Added UI warning bar about inability to connect to ElasticSearch.
+
+
 4.4.0
 =====
 * 2023-06-20

foursight_core/app.py

@@ -1,2 +1,6 @@
+from typing import Optional
 from chalice import Chalice
 app = Chalice(app_name='foursight-core')
+app.request_args = lambda: app.current_request.to_dict().get("query_params", {})
+app.request_arg = lambda name, default = None: app.current_request.to_dict().get("query_params", {}).get(name, default)
+app.request = lambda: app.current_request

foursight_core/captured_output.py

@@ -0,0 +1,70 @@
+# TODO: Move to dcicutils.
+from collections import namedtuple
+from contextlib import contextmanager
+import io
+import sys
+from typing import Optional
+
+_real_stdout = sys.stdout
+_real_stderr = sys.stderr
+
+@contextmanager
+def captured_output(capture: bool = True):
+    """
+    Context manager to capture any/all output to stdout or stderr, and not actually output it to stdout
+    or stderr. Yields and object with a get_captured_output() method to get the output captured thus far,
+    and another uncaptured_print() method to actually print the given output to stdout, even though output
+    to stdout is being captured. Can be useful, for example, in creating command-line scripts which invoke
+    code which outputs a lot of info, warning, error, etc to stdout or stderr, and we want to suprress that
+    output; but with the yielded uncaptured_print() method output specific to the script can actually be
+    output (to stdout); and/or can also optionally output any/all captured output, e.g. for debugging or
+    troubleshooting purposes. Disable this capture, without having to restructure your code WRT the usage
+    of the with-clause with this context manager, pass False as an argument to this context manager. 
+    """
+
+    original_stdout = _real_stdout
+    original_stderr = _real_stderr
+    captured_output = io.StringIO()
+
+    def set_original_output() -> None:
+        sys.stdout = original_stdout
+        sys.stderr = original_stderr
+
+    def set_captured_output() -> None:
+        if capture:
+            sys.stdout = captured_output
+            sys.stderr = captured_output
+
+    def uncaptured_print(*args, **kwargs) -> None:
+        set_original_output()
+        print(*args, **kwargs)
+        set_captured_output()
+
+    def uncaptured_input(message: str) -> str:
+        set_original_output()
+        value = input(message)
+        set_captured_output()
+        return value
+
+    def get_captured_output() -> Optional[str]:
+        return captured_output.getvalue() if capture else None
+
+    try:
+        set_captured_output()
+        Result = namedtuple("Result", ["get_captured_output", "uncaptured_print", "uncaptured_input"])
+        yield Result(get_captured_output, uncaptured_print, uncaptured_input)
+    finally:
+        set_original_output()
+
+
+@contextmanager
+def uncaptured_output():
+    original_stdout = sys.stdout
+    original_stderr = sys.stderr
+    sys.stdout = _real_stdout
+    sys.stderr = _real_stderr
+    try:
+        yield
+    finally:
+        sys.stdout = original_stdout
+        sys.stderr = original_stderr

foursight_core/check_utils.py

@@ -1,8 +1,10 @@
-import os
-import importlib
+from collections import namedtuple
 import copy
+import importlib
 import json
 import logging
+import os
+from typing import Callable, Optional
 from dcicutils.env_base import EnvBase
 from dcicutils.env_utils import infer_foursight_from_env
 from dcicutils.misc_utils import json_leaf_subst
@@ -39,7 +41,7 @@ def __init__(self, foursight_prefix, check_package_name='foursight_core', check_
         # which calls back to the locate_check_setup_file function AppUtilsCore here in foursight-core).
         if not os.path.exists(check_setup_file):
             raise BadCheckSetup(f"Did not locate the specified check setup file: {check_setup_file}")
-        self.CHECK_SETUP_FILE = check_setup_file # for display/troubleshooting
+        self.CHECK_SETUP_FILE = check_setup_file  # for display/troubleshooting
         with open(check_setup_file, 'r') as jfile:
             self.CHECK_SETUP = json.load(jfile)
         logger.debug(f"foursight_core/CheckHandler: Loaded check_setup.json file: {check_setup_file} ...")
@@ -374,31 +376,138 @@ def run_check_or_action(self, connection, check_str, check_kwargs):
         Fetches the check function and runs it (returning whatever it returns)
         Return a string for failed results, CheckResult/ActionResult object otherwise.
         """
-        # make sure parameters are good
-        error_str = ' '.join(['Info: CHECK:', str(check_str), 'KWARGS:', str(check_kwargs)])
-        if len(check_str.strip().split('/')) != 2:
-            return ' '.join(['ERROR. Check string must be of form module/check_name.', error_str])
-        mod_name = check_str.strip().split('/')[0]
-        check_name = check_str.strip().split('/')[1]
+        check_method = None
+        try:
+            check_method = self._get_check_or_action_function(check_str)
+        except Exception as e:
+            return f"ERROR: {str(e)}"
         if not isinstance(check_kwargs, dict):
-            return ' '.join(['ERROR. Check kwargs must be a dict.', error_str])
-        check_mod = None
+            return "ERROR: Check kwargs must be a dictionary: {check_str}"
+        return check_method(connection, **check_kwargs)
+
+    def _get_check_or_action_function(self, check_or_action_string: str, check_or_action: str = "check") -> Callable:
+        if len(check_or_action_string.strip().split('/')) != 2:
+            raise Exception(f"{check_or_action.title()} string must be of form"
+                            "module_name/{check_or_action}_function_name: {check_or_action_string}")
+        module_name = check_or_action_string.strip().split('/')[0]
+        function_name = check_or_action_string.strip().split('/')[1]
+        module = None
         for package_name in [self.check_package_name, 'foursight_core']:
             try:
-                check_mod = self.import_check_module(package_name, mod_name)
+                module = self.import_check_module(package_name, module_name)
             except ModuleNotFoundError:
                 continue
             except Exception as e:
                 raise e
-        if not check_mod:
-            return ' '.join(['ERROR. Check module is not valid.', error_str])
-        check_method = check_mod.__dict__.get(check_name)
-        if not check_method:
-            return ' '.join(['ERROR. Check name is not valid.', error_str])
-        if not self.check_method_deco(check_method, self.CHECK_DECO) and \
-           not self.check_method_deco(check_method, self.ACTION_DECO):
-            return ' '.join(['ERROR. Check or action must use a decorator.', error_str])
-        return check_method(connection, **check_kwargs)
+        if not module:
+            raise Exception(f"Cannot find check module: {module_name}")
+        function = module.__dict__.get(function_name)
+        if not function:
+            raise Exception(f"Cannot find check function: {module_name}/{function_name}")
+        if not self.check_method_deco(function, self.CHECK_DECO) and \
+           not self.check_method_deco(function, self.ACTION_DECO):
+            raise Exception(f"{check_or_action.title()} function must use"
+                            "@{check_or_action}_function decorator: {module_name}/{function_name}")
+        return function
+
+    @staticmethod
+    def get_checks_info(search: str = None) -> list:
+        checks = []
+        registry = Decorators.get_registry()
+        for item in registry:
+            info = CheckHandler._create_check_or_action_info(registry[item])
+            if search and search not in info.qualified_name.lower():
+                continue
+            if info.is_check:
+                checks.append(info)
+        return sorted(checks, key=lambda item: item.qualified_name)
+
+    @staticmethod
+    def get_actions_info(search: str = None) -> list:
+        actions = []
+        registry = Decorators.get_registry()
+        for item in registry:
+            info = CheckHandler._create_check_or_action_info(registry[item])
+            if search and search not in info.qualified_name.lower():
+                continue
+            if info.is_action:
+                actions.append(info)
+        return sorted(actions, key=lambda item: item.qualified_name)
+
+    @staticmethod
+    def get_check_info(check_function_name: str, check_module_name: str = None) -> Optional[namedtuple]:
+        return CheckHandler._get_check_or_action_info(check_function_name, check_module_name, "check")
+
+    @staticmethod
+    def get_action_info(action_function_name: str, action_module_name: str = None) -> Optional[namedtuple]:
+        return CheckHandler._get_check_or_action_info(action_function_name, action_module_name, "action")
+
+    @staticmethod
+    def _get_check_or_action_info(function_name: str,
+                                  module_name: str = None, kind: str = None) -> Optional[namedtuple]:
+
+        function_name = function_name.strip();
+        if module_name:
+            module_name = module_name.strip();
+        if not module_name:
+            if len(function_name.split("/")) == 2:
+                module_name = function_name.split("/")[0].strip()
+                function_name = function_name.split("/")[1].strip()
+            elif len(function_name.split(".")) == 2:
+                module_name = function_name.split(".")[0].strip()
+                function_name = function_name.split(".")[1].strip()
+        registry = Decorators.get_registry()
+        for name in registry:
+            if not kind or registry[name]["kind"] == kind:
+                item = registry[name]
+                if item["name"] == function_name:
+                    if not module_name:
+                        return CheckHandler._create_check_or_action_info(item)
+                    if item["module"].endswith("." + module_name):
+                        return CheckHandler._create_check_or_action_info(item)
+
+    @staticmethod
+    def _create_check_or_action_info(info: dict) -> Optional[namedtuple]:
+
+        def unqualified_module_name(module_name: str) -> str:
+            return module_name.rsplit(".", 1)[-1] if "." in module_name else module_name
+
+        def qualified_check_or_action_name(check_or_action_name: str, module_name: str) -> str:
+            unqualified_module = unqualified_module_name(module_name)
+            return f"{unqualified_module}/{check_or_action_name}" if unqualified_module else check_or_action_name
+
+        Info = namedtuple("CheckInfo", ["kind",
+                                        "is_check",
+                                        "is_action",
+                                        "name",
+                                        "qualified_name",
+                                        "file",
+                                        "line",
+                                        "module",
+                                        "unqualified_module",
+                                        "package",
+                                        "github_url",
+                                        "args",
+                                        "kwargs",
+                                        "function",
+                                        "associated_action",
+                                        "associated_check"])
+        return Info(info["kind"],
+                    info["kind"] == "check",
+                    info["kind"] == "action",
+                    info["name"],
+                    qualified_check_or_action_name(info["name"], info["module"]),
+                    info["file"],
+                    info["line"],
+                    info["module"],
+                    unqualified_module_name(info["module"]),
+                    info["package"],
+                    info["github_url"],
+                    info["args"],
+                    info["kwargs"],
+                    info["function"],
+                    info.get("action"),
+                    info.get("check"))
 
     def init_check_or_action_res(self, connection, check):
         """

foursight_core/decorators.py

@@ -87,10 +87,16 @@ def create_registry_record(self, kind, func, default_args, default_kwargs) -> No
             "package": func_package,
             "github_url": get_github_url(func_package, func_file, func_line),
             "args": default_args,
-            "kwargs": default_kwargs
+            "kwargs": default_kwargs,
+            "function": func
         }
         if associated_action:
             registry_record["action"] = associated_action
+        elif kind == "action":
+            for name in _decorator_registry:
+                item = _decorator_registry[name]
+                if item.get("action") == func_name:
+                    registry_record["check"] = item["name"] 
         _decorator_registry[func_name] = registry_record
 
     def check_function(self, *default_args, **default_kwargs):

foursight_core/fs_connection.py

@@ -44,7 +44,7 @@ def __init__(self, fs_environ, fs_environ_info, test=False, use_es=True, host=No
         # FOURFRONT information
         self.ff_server = fs_environ_info['fourfront']
         self.ff_env = fs_environ_info['ff_env']
-        self.ff_es = fs_environ_info['es']
+        self.ff_es = fs_environ_info['es'] if not host else host
         self.ff_bucket = fs_environ_info['bucket']
         self.redis = None
         self.redis_url = None

foursight_core/identity.py

@@ -143,6 +143,30 @@ def set_elasticsearch_host_environment_variable() -> None:
         logger.info(f"Foursight ES_HOST environment variable value is: {os.environ.get('ES_HOST')}")
 
 
+def set_redis_host_environment_variable() -> None:
+    redis_host_local = os.environ.get("REDIS_HOST_LOCAL")
+    if redis_host_local:
+        os.environ["REDIS_HOST"] = redis_host_local
+        logger.info(f"Foursight REDIS_HOST local environment variable value is: {os.environ.get('REDIS_HOST')}")
+    else:
+        logger.info(f"Foursight REDIS_HOST environment variable value is: {os.environ.get('REDIS_HOST')}")
+
+
+def set_auth0_environment_variable() -> None:
+    auth0_client_local = os.environ.get("AUTH0_CLIENT_LOCAL")
+    if auth0_client_local:
+        os.environ["CLIENT_ID"] = auth0_client_local
+        logger.info(f"Foursight Auth0 CLIENT_ID local environment variable value is: {os.environ.get('CLIENT_ID')}")
+    else:
+        logger.info(f"Foursight Auth0 CLIENT_ID environment variable value is: {os.environ.get('CLIENT_ID')}")
+    auth0_secret_local = os.environ.get("AUTH0_SECRET_LOCAL")
+    if auth0_secret_local:
+        os.environ["CLIENT_SECRET"] = auth0_secret_local
+        logger.info("Foursight Auth0 CLIENT_SECRET local environment variable value is: REDACTED")
+    else:
+        logger.info("Foursight Auth0 CLIENT_SECRET environment variable value is: REDACTED")
+
+
 def apply_identity_globally():
 
     # Make sure the IDENTITY (environment variable) is set (via Foursight CloudFormation template);
@@ -173,3 +197,9 @@ def apply_identity_globally():
 
     # Set ES_HOST to proxy for local testing (e.g. http://localhost:9200) via ES_HOST_LOCAL environment variable.
     set_elasticsearch_host_environment_variable()
+
+    # Set REDIS_HOST to proxy for local testing (e.g. redis://localhost:6379) via REDIS_HOST_LOCAL environment variable.
+    set_redis_host_environment_variable()
+
+    # Set AUTH0_CLIENT_LOCAL/AUTH0_SECRET_LOCAL for local testing.
+    set_auth0_environment_variable()

foursight_core/react/api/auth.py

@@ -163,7 +163,7 @@ def create_authtoken(self, jwt: str, jwt_expires_at: int, domain: str, request:
                     # For testing only, we simulate a portal access key error (e.g. due to expiration),
                     # which would manifest itself, primarily and most importantly, here, on login.
                     raise Exception("test_mode_access_key_simulate_error")
-            allowed_envs, first_name, last_name = self._envs.get_user_auth_info(email, raise_exception=True)
+            allowed_envs, first_name, last_name = self._envs.get_user_auth_info(email)
             user_exception = False
         except Exception as e:
             #

foursight_core/react/api/envs.py

@@ -6,6 +6,7 @@
 from dcicutils.env_utils import foursight_env_name
 from dcicutils.function_cache_decorator import function_cache
 from dcicutils.misc_utils import find_association
+from ...app import app
 from .gac import Gac
 
 logging.basicConfig()
@@ -81,15 +82,19 @@ def get_user_auth_info(self, email: str, raise_exception: bool = False) -> Tuple
             try:
                 # Note we must lower case the email to find the user. This is because all emails
                 # in the database are lowercased; it causes issues with OAuth if we don't do this.
+                known_env_name = known_env["full_name"]
+                envs = app.core.init_environments(known_env_name)
+                connection = app.core.init_connection(known_env_name, envs)
                 user = ff_utils.get_metadata('users/' + email.lower(),
-                                             ff_env=known_env["full_name"], add_on="frame=object&datastore=database")
+                                             key=connection.ff_keys,
+                                             add_on="frame=object&datastore=database")
                 if self._is_user_allowed_access(user):
                     # Since this is in a loop, for each env, this setup here will end up getting first/last name
                     # from the last env in the loop; doesn't really matter, just pick one set; this is just for
                     # informational/display purposes in the UI.
                     first_name = user.get("first_name")
                     last_name = user.get("last_name")
-                    allowed_envs.append(known_env["full_name"])
+                    allowed_envs.append(known_env_name)
             except Exception as e:
                 if raise_exception:
                     raise