Merge pull request #52 from 4dn-dcic/upgrade-python-20230920

Upgrade to Python 3.11

.github/workflows/main-CI.yml

@@ -18,15 +18,15 @@ jobs:
   # This workflow contains a single job called "build"
   build:
     # The type of runner that the job will run on
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.7'
+          python-version: '3.11'
 
       - name: Build
         run: |

.github/workflows/main-deploy-docs.yml

@@ -12,7 +12,7 @@ jobs:
   # This workflow contains a single job called "build"
   build:
     # The type of runner that the job will run on
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:

.github/workflows/main-publish.yml

@@ -18,17 +18,18 @@ jobs:
   # This workflow contains a single job called "build"
   build:
     # The type of runner that the job will run on
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.7'
+          python-version: '3.11'
+      # Do not yet know why it is necessary to manually install dcicutils but it is.
       - name: Install Python dependencies for publish
-        run: python -m pip install dcicutils==7.3.0.1b38
+        run: python -m pip install dcicutils==7.12.0.1b5
       - name: Publish
         env:
           PYPI_USER: ${{ secrets.PYPI_USER }}

CHANGELOG.rst

@@ -6,6 +6,12 @@ foursight-core
 Change Log
 ----------
 
+5.0.0
+=====
+* Update to Python 3.11.
+* Minor UI updates.
+
+
 4.5.0
 =====
 * 2023-08
@@ -17,6 +23,7 @@ Change Log
     awards/labs for foursight-fourfront (previous oversight).
   * Fixed up users pages.
 * Added UI warning bar about inability to connect to ElasticSearch.
+* A few minor UI tweaks; one WRT showing ff_link (like pre-React version) for check results.
 
 
 4.4.0

Makefile

@@ -12,6 +12,10 @@ build:  react
 	make configure
 	poetry install
 
+build-noreact:
+	make configure
+	poetry install
+
 update:
 	poetry update
 

foursight_core/app.py

@@ -1,6 +1,9 @@
+from json import loads as load_json
 from typing import Optional
 from chalice import Chalice
 app = Chalice(app_name='foursight-core')
-app.request_args = lambda: app.current_request.to_dict().get("query_params", {})
-app.request_arg = lambda name, default = None: app.current_request.to_dict().get("query_params", {}).get(name, default)
-app.request = lambda: app.current_request
+app.request = lambda: app.current_request.to_dict()
+app.request_args = lambda: app.request().get("query_params", {}) or {}
+app.request_arg = lambda name, default = None: app.request_args().get(name, default)
+app.request_body = lambda: load_json(app.current_request.raw_body.decode())
+app.request_method = lambda: app.current_request.method.upper()

foursight_core/app_utils.py

@@ -50,7 +50,7 @@
 from .react.api.react_api import ReactApi
 from .react.api.datetime_utils import (
     convert_time_t_to_utc_datetime_string,
-    convert_utc_datetime_to_utc_datetime_string
+    convert_datetime_to_utc_datetime_string
 )
 from .routes import Routes
 from .route_prefixes import CHALICE_LOCAL
@@ -788,10 +788,10 @@ def get_lambda_last_modified(self, lambda_name: str = None) -> Optional[str]:
                 lambda_tags = boto_lambda.list_tags(Resource=lambda_arn)["Tags"]
                 lambda_last_modified_tag = lambda_tags.get("last_modified")
                 if lambda_last_modified_tag:
-                    lambda_last_modified = convert_utc_datetime_to_utc_datetime_string(lambda_last_modified_tag)
+                    lambda_last_modified = convert_datetime_to_utc_datetime_string(lambda_last_modified_tag)
                 else:
                     lambda_last_modified = lambda_info["Configuration"]["LastModified"]
-                    lambda_last_modified = convert_utc_datetime_to_utc_datetime_string(lambda_last_modified)
+                    lambda_last_modified = convert_datetime_to_utc_datetime_string(lambda_last_modified)
                 return lambda_last_modified
         except Exception as e:
             logger.warning(f"Error getting lambda ({lambda_name}) last modified time: {e}")
@@ -868,7 +868,9 @@ def get_unique_annotated_environment_names(self):
              "short_name": short_env_name(env),
              "full_name": full_env_name(env),
              "public_name": public_env_name(env) if public_env_name(env) else short_env_name(env),
-             "foursight_name": infer_foursight_from_env(envname=env)} for env in unique_environment_names]
+             "foursight_name": infer_foursight_from_env(envname=env),
+             "portal_url": self.get_portal_url(env)}
+            for env in unique_environment_names]
         return sorted(unique_annotated_environment_names, key=lambda key: key["public_name"])
 
     def view_foursight(self, request, environ, is_admin=False, domain="", context="/"):
@@ -1145,7 +1147,7 @@ def view_users(self, request, environ, is_admin=False, domain="", context="/"):
                 "first_name": user_record.get("first_name"),
                 "last_name": user_record.get("last_name"),
                 "uuid": user_record.get("uuid"),
-                "modified": convert_utc_datetime_to_utc_datetime_string(last_modified)})
+                "modified": convert_datetime_to_utc_datetime_string(last_modified)})
         users = sorted(users, key=lambda key: key["email_address"])
         template = self.jin_env.get_template('users.html')
         html_resp.body = template.render(

foursight_core/captured_output.py

@@ -22,6 +22,9 @@ def captured_output(capture: bool = True):
     of the with-clause with this context manager, pass False as an argument to this context manager. 
     """
 
+    if capture and "--debug" in sys.argv[1:]:
+        capture = False
+
     original_stdout = _real_stdout
     original_stderr = _real_stderr
     captured_output = io.StringIO()

foursight_core/decorators.py

@@ -88,7 +88,7 @@ def create_registry_record(self, kind, func, default_args, default_kwargs) -> No
             "github_url": get_github_url(func_package, func_file, func_line),
             "args": default_args,
             "kwargs": default_kwargs,
-            "function": func
+            "function": func.__name__
         }
         if associated_action:
             registry_record["action"] = associated_action

foursight_core/react/api/auth.py

@@ -30,20 +30,22 @@ def __init__(self, auth0_client: str, auth0_secret: str, envs: Envs):
         self._auth0_client = auth0_client
         self._auth0_secret = auth0_secret
         self._envs = envs
-        # acquired from identity or env variable locally
-        try:
-            self._redis = RedisBase(create_redis_client(
-                url=os.environ['REDIS_HOST'])
-            ) if 'REDIS_HOST' in os.environ else None
-        except redis.exceptions.ConnectionError:
-            PRINT('Cannot connect to Redis')
-            PRINT('This error is expected when deploying with remote (ElastiCache) Redis')
-            self._redis = None
+        self._redis = None
 
     def get_redis_handler(self):
         """
         Returns a handler to Redis or None if not in use
         """
+        if not self._redis:
+            # 2023-09-21: Moved this from __init__ to here;
+            # it speeds up provision/deploy from 4dn-cloud-infra.
+            try:
+                self._redis = RedisBase(create_redis_client(
+                    url=os.environ['REDIS_HOST'])
+                ) if 'REDIS_HOST' in os.environ else None
+            except (redis.exceptions.ConnectionError, redis.exceptions.TimeoutError):
+                PRINT('Cannot connect to Redis')
+                PRINT('This error is expected when deploying with remote (ElastiCache) Redis')
         return self._redis
 
     @classmethod

foursight_core/react/api/aws_network.py

@@ -77,7 +77,7 @@ def _filter_boto_description_list(description: dict,
         predicate_filter = lambda tag: tag and predicate.match(tag) is not None
     elif isinstance(predicate, Callable):
         predicate_filter = lambda tag: tag and predicate(tag)
-    else:
+    elif predicate is not None:
         raise Exception(f"Unknown predicate type {type(predicate)} passed to filter_boto_description.")
 
     if isinstance(description.get(name), list):

foursight_core/react/api/aws_s3.py

@@ -1,6 +1,6 @@
 import logging
 from typing import Optional
-from .datetime_utils import convert_utc_datetime_to_utc_datetime_string
+from .datetime_utils import convert_datetime_to_utc_datetime_string
 from ...boto_s3 import boto_s3_client, boto_s3_resource
 
 logging.basicConfig()
@@ -32,7 +32,7 @@ def get_bucket_keys(cls, bucket_name: str) -> list:
                         results.append({
                             "key": bucket_key["Key"],
                             "size": bucket_key["Size"],
-                            "modified": convert_utc_datetime_to_utc_datetime_string(bucket_key["LastModified"])
+                            "modified": convert_datetime_to_utc_datetime_string(bucket_key["LastModified"])
                         })
 
         except Exception as e:

foursight_core/react/api/aws_stacks.py

@@ -1,5 +1,5 @@
 import boto3
-from .datetime_utils import convert_utc_datetime_to_utc_datetime_string
+from .datetime_utils import convert_datetime_to_utc_datetime_string
 from .yaml_utils import load_yaml
 from collections import OrderedDict
 from dcicutils.function_cache_decorator import function_cache
@@ -35,8 +35,8 @@ def _create_aws_stack_info(stack: object):
         "description": stack.description,
         "role_arn": stack.role_arn,
         "status": stack.stack_status,
-        "updated": convert_utc_datetime_to_utc_datetime_string(stack.last_updated_time),
-        "created": convert_utc_datetime_to_utc_datetime_string(stack.creation_time)
+        "updated": convert_datetime_to_utc_datetime_string(stack.last_updated_time),
+        "created": convert_datetime_to_utc_datetime_string(stack.creation_time)
     }
 
 

foursight_core/react/api/datetime_utils.py

@@ -7,8 +7,7 @@
 EPOCH = datetime.datetime.utcfromtimestamp(0)  # I.e.: 1970-01-01 00:00:00 UTC
 
 
-def _convert_utc_datetime_to_datetime_string(t: Union[datetime.datetime, str],
-                                             tzname: Optional[str] = None) -> Optional[str]:
+def convert_datetime_to_string(t: Union[datetime.datetime, str], tzname: Optional[str] = None) -> Optional[str]:
     """
     Converts the given datetime object OR string, which is ASSUMED to by in the UTC timezone,
     into a datetime string in the given/named timezone, and returns its value in a form that looks
@@ -20,34 +19,24 @@ def _convert_utc_datetime_to_datetime_string(t: Union[datetime.datetime, str],
     :param tzname: A timezone name (string); default to UTC if unspecified.
     :return: A datetime string in the given timezone formatted like: 2022-08-22 13:25:34 EDT
     """
-    def make_utc_aware_datetime(t: datetime) -> datetime:
-        return t.replace(tzinfo=pytz.UTC)
+    def make_aware_datetime(t: datetime) -> datetime:
+        return pytz.UTC.localize(t) if t.tzinfo is None else t
     try:
+        if not t:
+            return None
         if isinstance(t, str):
-            #
-            # Can sometimes get dates (from user ElasticSearch index)
-            # which look like this: 2019-06-20T00:00:00.0000000+00:00
-            # i.e. with 7-digits for ms which does not parse (up to 6).
-            # Was doing this hack below but found that the Python
-            # dateutil.parser is more forgiving so using that now.
-            #
-            # if ".0000000" in t:
-            #     t = t.replace(".0000000", ".000000")
-            # t = datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S.%f%z")
-            #
             t = dateutil_parser.parse(t)
+        t = make_aware_datetime(t)
         tz = pytz.timezone(tzname) if tzname else pytz.UTC
-        t = make_utc_aware_datetime(t).astimezone(tz)
+        if t.tzinfo != tz:
+            t = t.astimezone(tz)
         return t.strftime("%Y-%m-%d %H:%M:%S %Z")
-    except Exception as e:
+    except Exception:
         return None
 
 
-def convert_utc_datetime_to_utc_datetime_string(t: Union[datetime.datetime, str]) -> Optional[str]:
-    """
-    Same as _convert_utc_datetime_to_datetime_string (above) but explicitly for the UTC timezone.
-    """
-    return _convert_utc_datetime_to_datetime_string(t)
+def convert_datetime_to_utc_datetime_string(t: Union[datetime.datetime, str]):
+    return convert_datetime_to_string(t)
 
 
 def convert_time_t_to_datetime_string(time_t: int, tzname: Optional[str] = None) -> Optional[str]:
@@ -60,7 +49,7 @@ def convert_time_t_to_datetime_string(time_t: int, tzname: Optional[str] = None)
     :param tzname: A timezone name (string); default to UTC if unspecified.
     :return: A datetime string in the given timezone formatted like: 2022-08-22 13:25:34 EDT
     """
-    return _convert_utc_datetime_to_datetime_string(convert_time_t_to_datetime(time_t), tzname)
+    return convert_datetime_to_datetime_string(convert_time_t_to_datetime(time_t), tzname)
 
 
 def convert_time_t_to_utc_datetime_string(time_t: int) -> Optional[str]:

foursight_core/react/api/envs.py

@@ -27,6 +27,17 @@ def __init__(self, known_envs: list):
         # as returned by app_utils.get_unique_annotated_environment_names, where each
         # object contains these fields: name, short_name, full_name, public_name, foursight_name
         self._known_envs = known_envs
+        # Set any green/blue production/staging info.
+        for known_env in self._known_envs:
+            if self._env_contains(known_env, "blue"):
+                known_env["color"] = "blue"
+            elif self._env_contains(known_env, "green"):
+                known_env["color"] = "green"
+            if known_env.get("color"):
+                if self._env_contains(known_env, "stage") or self._env_contains(known_env, "staging"):
+                    known_env["is_staging"] = True
+                else:
+                    known_env["is_production"] = True
 
     def get_known_envs(self) -> list:
         return self._known_envs
@@ -109,3 +120,66 @@ def _is_user_allowed_access(user: Optional[dict]) -> bool:
     def _is_user_in_one_or_more_groups(user: Optional[dict], allowed_groups: list) -> bool:
         user_groups = user.get("groups") if user else None
         return user_groups and any(allowed_group in user_groups for allowed_group in allowed_groups or [])
+
+    @staticmethod
+    def _env_contains(env: dict, value: str, ignore_case: bool = True) -> bool:
+        if ignore_case:
+            value = value.lower()
+            return (value in env["full_name"].lower() or
+                    value in env["short_name"].lower() or
+                    value in env["public_name"].lower() or
+                    value in env["foursight_name"].lower())
+        else:
+            return (value in env["full_name"] or
+                    value in env["short_name"] or
+                    value in env["public_name"] or
+                    value in env["foursight_name"])
+
+    @staticmethod
+    def _env_contained_within(env: dict, value: str) -> bool:
+        """
+        Returns True iff the given environment (dictionary) is contained or somehow represented
+        within the given string value. Originally created for determining (sort of heuristically)
+        the environment to which an AWS task definition name should be associated. For example,
+        the name "c4-ecs-fourfront-hotseat-stack-FourfrontDeployment-xTDwbIYxIZh7" would belong
+        to the "hotseat" environment.
+        """
+        value = value.lower()
+        if "color" in env and env["color"] in ["blue", "green"] and env["color"] in value:
+            # Handle situations like this where both blue and green appear, but green appears twice:
+            # c4-ecs-blue-green-smaht-production-stack-SmahtgreenDeployment-mIHBLXIQ1pok
+            blue_count = value.count("blue")
+            green_count = value.count("green")
+            if env["color"] == "blue":
+                if blue_count > green_count:
+                    return True
+            elif green_count > blue_count:
+                return True
+        result = (env["full_name"].lower() in value or
+                  env["short_name"].lower() in value or
+                  env["public_name"].lower() in value or
+                  env["foursight_name"].lower() in value)
+        return result
+
+    def get_production_color(self) -> Tuple[Optional[str], Optional[str]]:
+        for known_env in self._known_envs:
+            if known_env.get("is_production"):
+                return (known_env["color"], known_env)
+        return (None, None)
+
+    def get_staging_color(self) -> Tuple[Optional[str], Optional[str]]:
+        for known_env in self._known_envs:
+            if known_env.get("is_staging"):
+                return (known_env["color"], known_env)
+        return (None, None)
+
+    def get_associated_env(self, name: str) -> Optional[dict]:
+        known_envs_with_colors = [env for env in self._known_envs if env.get("color")]
+        known_envs_sans_colors = [env for env in self._known_envs if not env.get("color")]
+        for known_env in known_envs_with_colors:
+            if self._env_contained_within(known_env, name):
+                return known_env
+        for known_env in known_envs_sans_colors:
+            if self._env_contained_within(known_env, name):
+                return known_env
+        return None