diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7be835cbe5..ca4560665f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,7 +44,7 @@ jobs: node-version: ${{ matrix.node_version }} check-latest: false - - name: Install/Link Postgres + - name: Install/Link Postgres and Redis if: ${{ matrix.test_type == 'NPM' || matrix.test_type == 'UNIT' }} run: | sudo apt-get install curl ca-certificates gnupg @@ -54,6 +54,9 @@ jobs: sudo apt-get install postgresql-14 postgresql-client-14 echo "/usr/lib/postgresql/14/bin" >> $GITHUB_PATH sudo ln -s /usr/lib/postgresql/14/bin/initdb /usr/local/bin/initdb + sudo apt install redis-server + sudo ln -s /usr/bin/redis-server /usr/local/bin/redis-server + sudo ln -s /usr/bin/redis-cli /usr/local/bin/redis-cli - name: Install Deps if: ${{ matrix.test_type == 'NPM' || matrix.test_type == 'UNIT' }} run: | diff --git a/.gitignore b/.gitignore index 1528a2d771..10f1273b6a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .coverage coverage.xml beanstalk.cfg +dump.rdb /coverage /.installed.cfg /.mr.developer.cfg diff --git a/Makefile b/Makefile index 1d7689490e..df550f6c90 100644 --- a/Makefile +++ b/Makefile @@ -111,8 +111,9 @@ kibana-stop: kill: # kills back-end processes associated with the application. Use with care. pkill -f postgres & - pkill -f elasticsearch & + pkill -f opensearch & pkill -f moto_server & + pkill -f redis-server & clean-python: @echo -n "Are you sure? This will wipe all libraries installed on this virtualenv [y/N] " && read ans && [ $${ans:-N} = y ] diff --git a/base.ini b/base.ini index dbdffccfcd..9acc76cf25 100644 --- a/base.ini +++ b/base.ini @@ -30,9 +30,9 @@ multiauth.policy.remoteuser.use = encoded.authentication.NamespacedAuthenticatio multiauth.policy.remoteuser.base = pyramid.authentication.RemoteUserAuthenticationPolicy multiauth.policy.accesskey.namespace = accesskey -multiauth.policy.accesskey.use = encoded.authentication.NamespacedAuthenticationPolicy -multiauth.policy.accesskey.base = encoded.authentication.BasicAuthAuthenticationPolicy -multiauth.policy.accesskey.check = encoded.authentication.basic_auth_check +multiauth.policy.accesskey.use = snovault.authentication.NamespacedAuthenticationPolicy +multiauth.policy.accesskey.base = snovault.authentication.BasicAuthAuthenticationPolicy +multiauth.policy.accesskey.check = snovault.authentication.basic_auth_check multiauth.policy.auth0.use = encoded.authentication.NamespacedAuthenticationPolicy multiauth.policy.auth0.namespace = auth0 diff --git a/deploy/docker/production/fourfront_any_alpha.ini b/deploy/docker/production/fourfront_any_alpha.ini index d643a472a2..64367576c1 100644 --- a/deploy/docker/production/fourfront_any_alpha.ini +++ b/deploy/docker/production/fourfront_any_alpha.ini @@ -1,6 +1,7 @@ [app:app] use = config:base.ini#app session.secret = %(here)s/session-secret.b64 +auth0.domain = ${AUTH0_DOMAIN} auth0.client = ${AUTH0_CLIENT} auth0.secret = ${AUTH0_SECRET} file_upload_bucket = ${FILE_UPLOAD_BUCKET} @@ -31,6 +32,9 @@ elasticsearch.aws_auth = true production = true load_test_data = snovault.loadxl:load_${DATA_SET}_data sqlalchemy.url = postgresql://${RDS_USERNAME}:${RDS_PASSWORD}@${RDS_HOSTNAME}:${RDS_PORT}/${RDS_DB_NAME} +redis.server = ${REDIS_SERVER} +g.recaptcha.key = ${g.recaptcha.key} +g.recaptcha.secret = ${g.recaptcha.secret} [composite:indexer] use = config:base.ini#indexer diff --git a/development.ini.template b/development.ini.template index b0d477e18b..8e8f406168 100644 --- a/development.ini.template +++ b/development.ini.template @@ -6,6 +6,7 @@ [app:app] use = config:base.ini#app sqlalchemy.url = postgresql://postgres@localhost:5441/postgres?host=/tmp/snovault/pgdata +redis.server = redis://localhost:6379 blob_bucket = encoded-4dn-blobs metadata_bundles_bucket = metadata-bundles-fourfront-local-test load_test_only = true diff --git a/docs/source/index.rst b/docs/source/index.rst index d1af38abdc..9faec1725c 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -54,7 +54,7 @@ Install or update dependencies:: $ brew install libevent libmagic libxml2 libxslt openssl postgresql graphviz nginx python3 $ brew install freetype libjpeg libtiff littlecms webp # Required by Pillow $ brew cask install adoptopenjdk8 - $ brew install opensearch node@16 + $ brew install opensearch node@16 redis NOTES: diff --git a/poetry.lock b/poetry.lock index 46ffd8529d..24d962aee6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1857,6 +1857,20 @@ files = [ {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, ] +[[package]] +name = "mirakuru" +version = "2.5.2" +description = "Process executor (not only) for tests." +optional = false +python-versions = ">=3.8" +files = [ + {file = "mirakuru-2.5.2-py3-none-any.whl", hash = "sha256:90c2d90a8cf14349b2f33e6db30a16acd855499811e0312e56cf80ceacf2d3e5"}, + {file = "mirakuru-2.5.2.tar.gz", hash = "sha256:41ca583d355eb7a6cfdc21c1aea549979d685c27b57239b88725434f115a7132"}, +] + +[package.dependencies] +psutil = {version = ">=4.0.0", markers = "sys_platform != \"cygwin\""} + [[package]] name = "mock" version = "5.1.0" @@ -2648,6 +2662,9 @@ files = [ {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, ] +[package.dependencies] +cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"crypto\""} + [package.extras] crypto = ["cryptography (>=3.4.0)"] dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] @@ -2855,6 +2872,26 @@ pytest = ">=6.2.5" [package.extras] dev = ["pre-commit", "pytest-asyncio", "tox"] +[[package]] +name = "pytest-redis" +version = "2.4.0" +description = "Redis fixtures and fixture factories for Pytest." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-redis-2.4.0.tar.gz", hash = "sha256:8a07520abed3cd341e8da1793059aa5717b02e56c43e7c76435db682cede10aa"}, + {file = "pytest_redis-2.4.0-py3-none-any.whl", hash = "sha256:3cf00ad3f7241e38ce6f1bcb66af11b91956a889f1e216cfc026e81aa638a4e7"}, +] + +[package.dependencies] +mirakuru = "*" +port-for = ">=0.6.0" +pytest = ">=6.2.0" +redis = "*" + +[package.extras] +tests = ["mock", "pytest-cov", "pytest-xdist"] + [[package]] name = "pytest-timeout" version = "2.3.1" diff --git a/pyproject.toml b/pyproject.toml index e81f4d4b13..14f77d5c98 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] # Note: Various modules refer to this system as "encoded", not "fourfront". name = "encoded" -version = "8.1.3" +version = "9.0.0" # TODO: To become 9.0.0 description = "4DN-DCIC Fourfront" authors = ["4DN-DCIC Team "] license = "MIT" @@ -151,6 +151,7 @@ pytest = "^7.2.1" pytest-cov = ">=2.2.1" pytest-instafail = ">=0.3.0" pytest-mock = ">=0.11.0" +pytest-redis = "^2.0.0" pytest-timeout = ">=1.0.0" pytest-xdist = ">=1.14" "repoze.debug" = ">=1.0.2" diff --git a/pytest.ini b/pytest.ini index 32a94e40d2..295ac3a811 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,8 @@ [pytest] timeout_func_only = true +redis_exec = /usr/local/bin/redis-server addopts = + --basetemp=/tmp/pytest -p encoded.tests.datafixtures -p snovault.tests.serverfixtures --instafail diff --git a/src/encoded/__init__.py b/src/encoded/__init__.py index 82af56dc11..f3d4bbf29c 100644 --- a/src/encoded/__init__.py +++ b/src/encoded/__init__.py @@ -1,4 +1,5 @@ import encoded.project_defs +import base64 import hashlib import logging import json # used only in Fourfront, not CGAP @@ -6,9 +7,13 @@ import netaddr import os import pkg_resources +import requests import sentry_sdk import subprocess - +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.backends import default_backend +from dcicutils.misc_utils import PRINT from codeguru_profiler_agent import Profiler from dcicutils.ecs_utils import ECSUtils from dcicutils.env_utils import EnvUtils, get_mirror_env_from_context @@ -148,6 +153,58 @@ def init_code_guru(*, group_name, region=ECSUtils.REGION): Profiler(profiling_group_name=group_name, region_name=region).start() +def jwk_to_pem(jwk): + """Converts JSON Web Key (JWK) to PEM format. + + A JSON Web Key (JWK) contains various fields depending on the key type ('kty'). + For an RSA key ('kty': 'RSA'), it must include: + - 'n': Base64url encoding of the RSA modulus (often called 'n' in mathematical terms). + - 'e': Base64url encoding of the RSA public exponent (often called 'e' in mathematical terms). + + Args: + jwk (dict): JSON Web Key in dictionary format. + + Returns: + bytes: PEM encoded key. + + Raises: + ValueError: If the 'kty' field is missing or if the key type is unsupported. + ValueError: If the 'n' or 'e' fields are missing in the JWK for an RSA key. + """ + + # Example usage: + # jwk = { + # "kty": "RSA", + # "n": "base64_encoded_value_of_n", + # "e": "base64_encoded_value_of_e" + # } + # pem_key = jwk_to_pem(jwk) + # print(pem_key.decode()) + + #TODO Move it into snovault for possible RAS integration of CGAP or SMaHT + + if 'kty' not in jwk: + raise ValueError("JWK must have a 'kty' field") + kty = jwk['kty'] + + if kty == 'RSA': + if 'n' not in jwk or 'e' not in jwk: + raise ValueError("JWK RSA key must have 'n' and 'e' fields") + + n = int.from_bytes(base64.urlsafe_b64decode(jwk['n'] + '=='), byteorder='big') + e = int.from_bytes(base64.urlsafe_b64decode(jwk['e'] + '=='), byteorder='big') + + public_key = rsa.RSAPublicNumbers(e, n).public_key(default_backend()) + pem = public_key.public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo + ) + + return pem + + # You can add more cases for other key types (e.g., 'EC' for elliptic curve keys) + raise ValueError("Unsupported 'kty': {}".format(kty)) + def main(global_config, **local_config): """ This function returns a Pyramid WSGI application. @@ -180,20 +237,42 @@ def main(global_config, **local_config): settings['auth0.domain'] = settings.get('auth0.domain', os.environ.get('Auth0Domain', DEFAULT_AUTH0_DOMAIN)) settings['auth0.client'] = settings.get('auth0.client', os.environ.get('Auth0Client')) settings['auth0.secret'] = settings.get('auth0.secret', os.environ.get('Auth0Secret')) - settings['auth0.options'] = { - 'auth': { - 'sso': False, - 'redirect': False, - 'responseType': 'token', - 'params': { - 'scope': 'openid email', - 'prompt': 'select_account' + + #options + if 'auth0' in settings['auth0.domain']: # Auth0 + settings['auth0.options'] = { + 'auth': { + 'sso': False, + 'redirect': False, + 'responseType': 'token', + 'params': { + 'scope': 'openid email', + 'prompt': 'select_account' + } + }, + 'allowedConnections': ['github', 'google-oauth2'] # TODO: make at least this part configurable + } + elif 'nih.gov' in settings['auth0.domain']: # RAS + # we are still keeping the Auth0 structure for compatibility (SPC) + settings['auth0.options'] = { + 'auth': { + 'responseType': 'code', + 'params': { + 'scope': 'openid profile email ga4gh_passport_v1', + 'prompt': 'login consent' + } } - }, - 'allowedConnections': [ # TODO: make at least this part configurable - 'github', 'google-oauth2' - ] - } + } + auth0Domain = settings['auth0.domain'] + # get public key from jwks uri + response = requests.get(url=f'https://{auth0Domain}/openid/connect/jwks.json') + jwks = response.json() + # gives the set of jwks keys.the keys has to be passed as it is to jwt.decode() for signature verification. + settings['auth0.public.key'] = jwk_to_pem(jwks['keys'][0]) + else: + # Unknown + settings['auth0.options'] = {} + # ga4 api secret if 'IDENTITY' in os.environ: identity = assume_identity() @@ -203,8 +282,8 @@ def main(global_config, **local_config): settings['ga4.secret'] = settings.get('ga4.secret', os.environ.get('GA4Secret')) # set google reCAPTCHA keys # TODO propagate from GAC - settings['g.recaptcha.key'] = os.environ.get('reCaptchaKey') - settings['g.recaptcha.secret'] = os.environ.get('reCaptchaSecret') + settings['g.recaptcha.key'] = settings.get('g.recaptcha.key', os.environ.get('reCaptchaKey')) + settings['g.recaptcha.secret'] = settings.get('g.recaptcha.secret', os.environ.get('reCaptchaSecret')) # enable invalidation scope settings[INVALIDATION_SCOPE_ENABLED] = True @@ -223,6 +302,8 @@ def main(global_config, **local_config): config.include('pyramid_multiauth') # must be before calling set_authorization_policy # Override default authz policy set by pyramid_multiauth config.set_authorization_policy(LocalRolesAuthorizationPolicy()) + + # This creates a session factory (from definition in Snovault/app.py) config.include(session) # must include, as tm.attempts was removed from pyramid_tm @@ -253,6 +334,9 @@ def main(global_config, **local_config): config.include('snovault.elasticsearch') config.include('.search') + if 'redis.server' in config.registry.settings: + config.include('snovault.redis') + # this contains fall back url, so make sure it comes just before static_resoruces config.include('.types.page') config.include(static_resources) diff --git a/src/encoded/authentication.py b/src/encoded/authentication.py index a41d682846..bfbd3de1a3 100644 --- a/src/encoded/authentication.py +++ b/src/encoded/authentication.py @@ -1,10 +1,272 @@ -from snovault.authentication import ( - Auth0AuthenticationPolicy, - basic_auth_check, - BasicAuthAuthenticationPolicy, - CRYPT_CONTEXT, - generate_password, - generate_user, - NamespacedAuthenticationPolicy, - session_properties +import jwt +import logging + +from pyramid.authentication import ( + BasicAuthAuthenticationPolicy as _BasicAuthAuthenticationPolicy, + CallbackAuthenticationPolicy ) +import requests +from pyramid.path import ( + DottedNameResolver, + caller_package, +) +from pyramid.httpexceptions import HTTPUnauthorized +from snovault.authentication import get_jwt +from snovault.redis.interfaces import REDIS +from dcicutils.redis_tools import RedisSessionToken + + +logger = logging.getLogger(__name__) + + +JWT_ENCODING_ALGORITHM = 'HS256' + +# Might need to keep a list of previously used algorithms here, not just the one we use now. +# Decryption algorithm used to default to a long list, but more recent versions of jwt library +# say we should stop assuming that. +# +# In case it goes away, as far as I can tell, the default for decoding from their +# default_algorithms() method used to be what we've got in JWT_ALL_ALGORITHMS here. +# -kmp 15-May-2020 + +JWT_ALL_ALGORITHMS = ['ES512', 'RS384', 'HS512', 'ES256', 'none', + 'RS256', 'PS512', 'ES384', 'HS384', 'ES521', + 'PS384', 'HS256', 'PS256', 'RS512'] + +# Probably we could get away with fewer, but I think not as few as just our own encoding algorithm, +# so for now I believe the above list was the default, and this just rearranges it to prefer the one +# we use for encoding. -kmp 19-Jan-2021 + +JWT_DECODING_ALGORITHMS = [JWT_ENCODING_ALGORITHM] + + + +def redis_is_active(request): + """ Quick helper to standardize detecting whether redis is in use """ + return 'redis.server' in request.registry.settings + + +class NamespacedAuthenticationPolicy(object): + """ Wrapper for authentication policy classes + + As userids are included in the list of principals, it seems good practice + to namespace them to avoid clashes. + + Constructor Arguments + + ``namespace`` + + The namespace used (string). + + ``base`` + + The base authentication policy (class or dotted name). + + Remaining arguments are passed to the ``base`` constructor. + + Example + + To make a ``REMOTE_USER`` 'admin' be 'user.admin' + + .. code-block:: python + + policy = NamespacedAuthenticationPolicy('user', + 'pyramid.authentication.RemoteUserAuthenticationPolicy') + """ + + def __new__(cls, namespace, base, *args, **kw): + # Dotted name support makes it easy to configure with pyramid_multiauth + name_resolver = DottedNameResolver(caller_package()) + base = name_resolver.maybe_resolve(base) + # Dynamically create a subclass + name = 'Namespaced_%s_%s' % (namespace, base.__name__) + klass = type(name, (cls, base), {'_namespace_prefix': namespace + '.'}) + return super(NamespacedAuthenticationPolicy, klass).__new__(klass) + + def __init__(self, namespace, base, *args, **kw): + super(NamespacedAuthenticationPolicy, self).__init__(*args, **kw) + + def unauthenticated_userid(self, request): + cls = super(NamespacedAuthenticationPolicy, self) + userid = super(NamespacedAuthenticationPolicy, self) \ + .unauthenticated_userid(request) + if userid is not None: + userid = self._namespace_prefix + userid + return userid + + def authenticated_userid(self, request): + """ + Adds `request.user_info` for all authentication types. + Fetches and returns some user details if called. + """ + namespaced_userid = super().authenticated_userid(request) + + if namespaced_userid is not None: + # userid, if present, may be in form of UUID (if remoteuser) or an email (if Auth0). + namespace, userid = namespaced_userid.split(".", 1) + + # Allow access basic user credentials from request obj after authenticating & saving request + def get_user_info(request): + user_props = request.embed('/session-properties', as_user=userid) # Performs an authentication against DB for user. + if not user_props.get('details'): + raise HTTPUnauthorized( + title="Could not find user info for {}".format(userid), + headers={ + 'WWW-Authenticate': + "Bearer realm=\"{}\"; Basic realm=\"{}\"".format(request.domain, request.domain) + } + ) + return user_props + + # If not authenticated (not in our DB), request.user_info will throw an HTTPUnauthorized error. + request.set_property(get_user_info, "user_info", True) + + return namespaced_userid + + def remember(self, request, principal, **kw): + if not principal.startswith(self._namespace_prefix): + return [] + principal = principal[len(self._namespace_prefix):] + return super(NamespacedAuthenticationPolicy, self) \ + .remember(request, principal, **kw) + + +class BasicAuthAuthenticationPolicy(_BasicAuthAuthenticationPolicy): + def __init__(self, check, *args, **kw): + # Dotted name support makes it easy to configure with pyramid_multiauth + name_resolver = DottedNameResolver(caller_package()) + check = name_resolver.maybe_resolve(check) + super(BasicAuthAuthenticationPolicy, self).__init__(check, *args, **kw) + + +class LoginDenied(HTTPUnauthorized): + title = 'Login Failure' + + def __init__(self, domain=None, *args, **kwargs): + super(LoginDenied, self).__init__(*args, **kwargs) + if not self.headers.get('WWW-Authenticate') and domain: + # headers['WWW-Authenticate'] might be set in constructor thru headers + self.headers['WWW-Authenticate'] = "Bearer realm=\"{}\"; Basic realm=\"{}\"".format(domain, domain) + + +_fake_user = object() + + +class Auth0AuthenticationPolicy(CallbackAuthenticationPolicy): + + login_path = '/login' + method = 'POST' + + def unauthenticated_userid(self, request): + ''' + So basically this is used to do a login, instead of the actual + login view... not sure why, but yeah.. + ''' + # we will cache it for the life of this request, cause pyramids does traversal + cached = getattr(request, '_auth0_authenticated', _fake_user) + if cached is not _fake_user: + return cached + + # try to find the token in the request (should be in the header) + id_token = get_jwt(request) + if not id_token: + # can I thrown an 403 here? + # print('Missing assertion.', 'unauthenticated_userid', request) + return None + + if redis_is_active(request): + session_token = RedisSessionToken.from_redis( + redis_handler=request.registry[REDIS], + namespace=request.registry.settings['env.name'], + token=id_token + ) + if not session_token: + return None + + auth0_domain = request.registry.settings['auth0.domain'] + if 'auth0' in auth0_domain: + secret = request.registry.settings['auth0.secret'] + algorithms = JWT_DECODING_ALGORITHMS + else: + # RAS + secret = request.registry.settings['auth0.public.key'] + algorithms = ['RS256'] + + + jwt_info = session_token.decode_jwt( + audience=request.registry.settings['auth0.client'], + secret=secret, + algorithms=algorithms + ) + if jwt_info.get('email') is None: + jwt_info['email'] = session_token.get_email() + if session_token.get_email() is not None: + request.set_property(lambda r: False, 'auth0_expired') + else: + jwt_info = self.get_token_info(id_token, request) + if not jwt_info: + return None + + email = request._auth0_authenticated = jwt_info['email'].lower() + + # At this point, email has been authenticated with their Auth0 provider, but we don't know yet if this email is in our database. + # If not authenticated (not in our DB), request.user_info will throw an HTTPUnauthorized error. + def get_user_info(request): + # This indirection is necessary, otherwise needed parameters don't make it + return self.get_token_info(request, email, id_token) + + request.set_property(get_user_info, "user_info", True) + return email + + @staticmethod + def get_user_info(request, email, id_token): + """ + Previously an inner method, redefined here so can be used outside, but can only be used within a route + Allow access basic user credentials from request obj after authenticating & saving request + """ + user_props = request.embed('/session-properties', as_user=email) # Performs an authentication against DB for user. + if not user_props.get('details'): + raise HTTPUnauthorized( + title="Could not find user info for {}".format(email), + headers={'WWW-Authenticate': "Bearer realm=\"{}\"; Basic realm=\"{}\"".format(request.domain, request.domain) } + ) + user_props['id_token'] = id_token + return user_props + + @staticmethod + def get_token_info(token, request): + ''' + Given a jwt get token info from auth0, handle retrying and whatnot. + This is only called if we receive a Bearer token in Authorization header. + ''' + try: + + # lets see if we have an auth0 token or our own + registry = request.registry + auth0_client = registry.settings.get('auth0.client') + auth0_secret = registry.settings.get('auth0.secret') + if auth0_client and auth0_secret: + # leeway accounts for clock drift between us and auth0 + payload = jwt.decode(token, auth0_secret, + algorithms=JWT_DECODING_ALGORITHMS, + audience=auth0_client, leeway=30) + if 'email' in payload and payload.get('email_verified') is True: + request.set_property(lambda r: False, 'auth0_expired') + return payload + + else: # we don't have the key, let auth0 do the work for us + user_url = "https://{domain}/tokeninfo".format(domain='hms-dbmi.auth0.com') + resp = requests.post(user_url, {'id_token':token}) + payload = resp.json() + if 'email' in payload and payload.get('email_verified') is True: + request.set_property(lambda r: False, 'auth0_expired') + return payload + + except (ValueError, jwt.exceptions.InvalidTokenError, jwt.exceptions.InvalidKeyError) as e: + # Catch errors from decoding JWT + print('Invalid JWT assertion : %s (%s)', (e, type(e).__name__)) + request.set_property(lambda r: True, 'auth0_expired') # Allow us to return 403 code &or unset cookie in renderers.py + return None + + print("didn't get email or email is not verified") + return None diff --git a/src/encoded/project/authentication.py b/src/encoded/project/authentication.py index 24cea131d1..754cd9795d 100644 --- a/src/encoded/project/authentication.py +++ b/src/encoded/project/authentication.py @@ -1,13 +1,24 @@ from dcicutils.misc_utils import ignored +from pyramid.view import HTTPTemporaryRedirect from pyramid.httpexceptions import HTTPUnauthorized from snovault.project.authentication import SnovaultProjectAuthentication + class FourfrontProjectAuthentication(SnovaultProjectAuthentication): def login(self, context, request, *, samesite): - ignored(samesite) - samesite = "lax" - return super().login(context, request, samesite=samesite) + domain = request.registry.settings['auth0.domain'] + if 'auth0' in domain: + ignored(samesite) + samesite = "lax" + return super().login(context, request, samesite=samesite) + else: # do RAS handshake + ras_client_id = request.registry.settings['auth0.client'] + redir_url = f"{domain}" \ + f"?client_id={ras_client_id}&prompt=login+consent&" \ + f"redirect_uri={request.scheme}://{request.host}/callback&response_type=code" \ + f"&scope=openid+profile+email+ga4gh_passport_v1" + raise HTTPTemporaryRedirect(location=redir_url) def namespaced_authentication_policy_authenticated_userid(self, namespaced_authentication_policy, request, set_user_info_property): set_user_info_property = True diff --git a/src/encoded/static/components/app.js b/src/encoded/static/components/app.js index 534931a30f..c7fc661022 100644 --- a/src/encoded/static/components/app.js +++ b/src/encoded/static/components/app.js @@ -1173,11 +1173,11 @@ export default class App extends React.PureComponent { "img-src 'self' https://* data: www.google-analytics.com", "child-src blob:", "frame-src www.google.com/recaptcha/", - "script-src 'self' https://www.google-analytics.com https://www.googletagmanager.com https://cdn.auth0.com https://hms-dbmi.auth0.com https://secure.gravatar.com https://www.gstatic.com/recaptcha/ https://www.google.com/recaptcha/ 'unsafe-eval'", // + (typeof BUILDTYPE === "string" && BUILDTYPE === "quick" ? " 'unsafe-eval'" : ""), + "script-src 'self' https://www.google-analytics.com https://www.googletagmanager.com https://cdn.auth0.com https://hms-dbmi.auth0.com *.nih.gov https://secure.gravatar.com https://www.gstatic.com/recaptcha/ https://www.google.com/recaptcha/ 'unsafe-eval'", // + (typeof BUILDTYPE === "string" && BUILDTYPE === "quick" ? " 'unsafe-eval'" : ""), "style-src 'self' 'unsafe-inline' https://fonts.googleapis.com https://unpkg.com https://www.googletagmanager.com", "font-src 'self' https://fonts.gstatic.com", "worker-src 'self' blob:", - "connect-src 'self' * blob: https://raw.githubusercontent.com https://higlass.4dnucleome.org https://*.s3.amazonaws.com https://s3.amazonaws.com/4dn-dcic-public/ https://www.encodeproject.org https://rest.ensembl.org https://www.google-analytics.com https://www.googletagmanager.com https://o427308.ingest.sentry.io https://www.gstatic.com/recaptcha/ https://www.google.com/recaptcha/ 'unsafe-inline' 'unsafe-eval'" + "connect-src 'self' * blob: https://raw.githubusercontent.com https://higlass.4dnucleome.org https://*.s3.amazonaws.com https://s3.amazonaws.com/4dn-dcic-public/ https://www.encodeproject.org https://rest.ensembl.org https://www.google-analytics.com https://www.googletagmanager.com https://o427308.ingest.sentry.io https://www.gstatic.com/recaptcha/ https://www.google.com/recaptcha/ *.nih.gov 'unsafe-inline' 'unsafe-eval'" ].join("; "); // `lastCSSBuildTime` is used for both CSS and JS because is most likely they change at the same time on production from recompiling diff --git a/src/encoded/static/components/forms/UserRegistrationForm.js b/src/encoded/static/components/forms/UserRegistrationForm.js index 2292052b2b..c5be44dc87 100644 --- a/src/encoded/static/components/forms/UserRegistrationForm.js +++ b/src/encoded/static/components/forms/UserRegistrationForm.js @@ -175,7 +175,9 @@ export default class UserRegistrationForm extends React.PureComponent { ajax.load( endpoint, (resp) => { - onComplete(); // <- Do request to login, then hide/unmount this component. + if (onComplete && typeof onComplete === 'function') { + onComplete(); // <- Do request to login, then hide/unmount this component. + } }, 'POST', (err) => { diff --git a/src/encoded/static/components/index.js b/src/encoded/static/components/index.js index 0b3754bce1..c94e471107 100644 --- a/src/encoded/static/components/index.js +++ b/src/encoded/static/components/index.js @@ -6,6 +6,15 @@ * Here we import all of our Content Views (Page Views) and register them * to `globals.content_views` so that they may be picked up and routed to in * the root `App` component. + * + * VERY IMPORTANT: The routing mechanism dispatching from the back-end is based on the + * "title" field ie: if the back-end returns JSON with title=login-success that will trigger + * the rendering of the LoginSuccessView. In addition, there is some weirdness with + * "npm run dev-quick" where it will not reload new components into the system, nor + * can they be imported directly from SPC in this file - so you need to override all components + * from SPC in this repo before configuring the dispatch below. See LoginSuccessView.js and + * authentication.py for a basic illustration of how it works. - Will March 17 2023 + * */ import { content_views } from './globals'; @@ -52,6 +61,10 @@ import PublicationSearchView from './browse/PublicationSearchView'; import SubscriptionsView from './browse/SubscriptionsView'; import FileSearchView from './browse/FileSearchView'; +// auth related views +import LoginSuccessView from './navigation/components/LoginSuccessView'; +import UserRegistrationView from './navigation/components/UserRegistrationView'; + content_views.register(StaticPage, 'StaticPage'); content_views.register(DirectoryPage, 'DirectoryPage'); @@ -111,6 +124,9 @@ content_views.register(SearchView, 'PublicationSearchResults', 'mul content_views.register(SubscriptionsView, 'Submissions'); // TODO: Rename 'Submissions' to 'Subscriptions' on back-end (?) content_views.register(FileSearchView, 'FileSearchResults'); +content_views.register(LoginSuccessView, 'callback'); +content_views.register(UserRegistrationView, 'registration'); + // Fallback for anything we haven't registered content_views.fallback = function () { return FallbackView; diff --git a/src/encoded/static/components/navigation/components/AccountNav.js b/src/encoded/static/components/navigation/components/AccountNav.js index 8210fbb19c..362b94c5de 100644 --- a/src/encoded/static/components/navigation/components/AccountNav.js +++ b/src/encoded/static/components/navigation/components/AccountNav.js @@ -12,7 +12,8 @@ import { LoginNavItem } from './LoginNavItem'; import { BigDropdownNavItem, BigDropdownIntroductionWrapper } from './BigDropdown'; -const auth0Options = { +/** Specific to 4DN */ +export const auth0Options = { auth: { sso: false, redirect: false, @@ -50,6 +51,7 @@ const auth0Options = { export const AccountNav = React.memo(function AccountNav(props){ const { session, updateAppSessionState, schemas, ...passProps } = props; const { windowWidth, href } = passProps; + updateAppSessionState(); // call this in an attempt before rendering the component if (!session) { // Render login button //TODO remove custom info after RAS transition completed @@ -57,7 +59,7 @@ export const AccountNav = React.memo(function AccountNav(props){ return ( diff --git a/src/encoded/static/components/navigation/components/LoginSuccessView.js b/src/encoded/static/components/navigation/components/LoginSuccessView.js new file mode 100644 index 0000000000..4f48b2c6a1 --- /dev/null +++ b/src/encoded/static/components/navigation/components/LoginSuccessView.js @@ -0,0 +1,149 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import _ from 'underscore'; +import * as JWT from '@hms-dbmi-bgm/shared-portal-components/es/components/util/json-web-token'; +import { event as trackEvent, setUserID } from '@hms-dbmi-bgm/shared-portal-components/es/components/util/analytics'; +import { load, fetch, promise as ajaxPromise } from '@hms-dbmi-bgm/shared-portal-components/es/components/util/ajax'; +import { Alerts } from '@hms-dbmi-bgm/shared-portal-components/es/components/ui/Alerts'; +import { navigate } from '@hms-dbmi-bgm/shared-portal-components/es/components/util'; +import { ItemDetailList } from '@hms-dbmi-bgm/shared-portal-components/es/components/ui/ItemDetailList'; + + +/** + * Component that essentially handles the "callback" interaction previously done by + * the UI. The interaction in the Redis state now proceeds as follows: + * 1. The UI renders the Auth0Lock component in 'code' mode + * 2. The user logs in via the Auth0Lock and transmits the code to the back-end + * via GET /callback?code=abcdefg... + * 3. The back-end calls into Auth0 to get JWT and returns a session token to + * the browser, returning a success response + * 4. This component loads, triggering the acquisition of /session-properties + * and thus the population of the user_info in local storage + */ +export default class LoginSuccessView extends React.PureComponent { + + static propTypes = { + 'readyToRedirect': PropTypes.bool + }; + + constructor(props) { + super(props); + this.state = { + readyToRedirect: false + }; + } + + /** + * This component is meant to be loaded upon navigating from backend on authentication to the /callback + * Once that has happened, we should have stored a jwtToken as a cookie that + * we can use to make authenticated requests ie: the below calls should return + * clean responses if the user successfully logged in and error out appropriately if + * they did not. + */ + componentDidMount() { + Promise.race([ + // At this point a http-only cookie session token will be stored under jwtToken + fetch('/session-properties'), + new Promise(function(resolve, reject){ + setTimeout(function(){ reject({ 'description' : 'timed out', 'type' : 'timed-out' }); }, 30000); /* 30 seconds */ + }) + ]) + .then((response) => { + // this may not be needed? + console.log('got response from session properties', response); + if (response.code || response.status) throw response; + return response; + }) + .then((userInfoResponse) => { + const { + details: { + email: userEmail = null + } = {}, + user_actions = [] + } = userInfoResponse; + + if (!userEmail) { + throw new Error("Did not receive user details from /session-properties, login failed."); + } + + // Fetch user profile and (outdated/to-revisit-later) use their primary lab as the eventLabel. + const profileURL = (_.findWhere(user_actions, { 'id': 'profile' }) || {}).href; + if (profileURL) { + this.setState({ "isLoading": false }); + + JWT.saveUserInfoLocalStorage(userInfoResponse); + updateAppSessionState(); // <- this function (in App.js) is now expected to call `Alerts.deQueue(Alerts.LoggedOut);` + console.info('Login completed'); + + // Register an analytics event for UI login. + // This is used to segment public vs internal audience in Analytics dashboards. + load(profileURL, (profile) => { + if (typeof successCallback === 'function') { + successCallback(profile); + } + if (typeof onLogin === 'function') { + onLogin(profile); + } + + const { uuid: userId, groups = null } = profile; + + setUserID(userId); + + trackEvent('Authentication', 'UILogin', { + eventLabel: "Authenticated ClientSide", + name: userId, + userId, + userGroups: groups && (JSON.stringify(groups.sort())) + }); + + }, 'GET', () => { + throw new Error('Request to profile URL failed.'); + }); + } else { + console.log('in failed user profile fetch'); + throw new Error('No profile URL found in user_actions.'); + } + }).catch((error) => { + // Handle Errors + console.log(error); + + this.setState({ "isLoading" : false }); + setUserID(null); + + if (typeof errorCallback === "function") { + errorCallback(error); + } + }); + this.setState({ + readyToRedirect: true + }); + } + + getReturnUrl() { + const url = document.cookie.split("; ").find((row) => row.startsWith("returnUrl="))?.split("=")[1]; + return (url && decodeURIComponent(url)) || '/'; + } + + render() { + var { context, schemas } = this.props; + if (this.state.readyToRedirect) { + navigate(this.getReturnUrl(), {}, (resp)=>{ + // Show alert on new Item page + Alerts.queue({ + 'title' : 'Success', + 'message' : 'You are now logged in.', + 'style' : 'success' + }); + //remove return url cookie + document.cookie = `returnUrl=; expires=${new Date(0).toUTCString()}; path=/;`; + }); + } + // This needs styling, maybe a spinning loader? Will look into later + return ( +
+ {typeof context.description == "string" ?

{context.description}

: null} + +
+ ); + } +} diff --git a/src/encoded/static/components/navigation/components/UserRegistrationView.js b/src/encoded/static/components/navigation/components/UserRegistrationView.js new file mode 100644 index 0000000000..36ed156c3d --- /dev/null +++ b/src/encoded/static/components/navigation/components/UserRegistrationView.js @@ -0,0 +1,149 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import _ from 'underscore'; +import { UserRegistrationModal } from './UserRegistrationModal'; +import { onLoginNavItemClick } from './LoginNavItem'; +import { JWT, console, navigate } from '@hms-dbmi-bgm/shared-portal-components/es/components/util'; +import { performLogout } from '@hms-dbmi-bgm/shared-portal-components/es/components/navigation/components/LoginController'; +import { event as trackEvent, setUserID } from '@hms-dbmi-bgm/shared-portal-components/es/components/util/analytics'; +import { load, fetch } from '@hms-dbmi-bgm/shared-portal-components/es/components/util/ajax'; + +/** + * Component that is rendered when a user does not successfully log in and needs + * to be shown the registration modal. The caller should have an email in the @graph + * of the response (see /callback in authentication.py) + */ +export default class UserRegistrationView extends React.PureComponent { + + static propTypes = { + ...UserRegistrationModal.propTypes + }; + + constructor(props) { + super(props); + this.onRegistrationComplete = this.onRegistrationComplete.bind(this); + this.onRegistrationCancel = this.onRegistrationCancel.bind(this); + this.onLogin = this.onLogin.bind(this); + this.showLock = this.showLock.bind(this); + this.state = { + id: 'loginbtn', + unverifiedUserEmail: props.context['@graph'][0], + showLock: false, + isLoading: false + }; + } + + onRegistrationComplete() { + const { updateAppSessionState } = this.props; + + Promise.race([ + // At this point a http-only cookie session token will be stored under jwtToken + fetch('/session-properties'), + new Promise(function (resolve, reject) { + setTimeout(function () { reject({ 'description': 'timed out', 'type': 'timed-out' }); }, 30000); /* 30 seconds */ + }) + ]) + .then((response) => { + // this may not be needed? + console.log('got response from session properties', response); + if (response.code || response.status) throw response; + return response; + }) + .then((userInfoResponse) => { + const { + details: { + email: userEmail = null + } = {}, + user_actions = [] + } = userInfoResponse; + + if (!userEmail) { + throw new Error("Did not receive user details from /session-properties, login failed."); + } + + // Fetch user profile and (outdated/to-revisit-later) use their primary lab as the eventLabel. + const profileURL = (_.findWhere(user_actions, { 'id': 'profile' }) || {}).href; + if (profileURL) { + this.setState({ "isLoading": false }); + + JWT.saveUserInfoLocalStorage(userInfoResponse); + updateAppSessionState(); // <- this function (in App.js) is now expected to call `Alerts.deQueue(Alerts.LoggedOut);` + console.info('Login completed'); + + // Register an analytics event for UI login. + // This is used to segment public vs internal audience in Analytics dashboards. + load(profileURL, (profile) => { + if (typeof successCallback === 'function') { + successCallback(profile); + } + if (typeof onLogin === 'function') { + onLogin(profile); + } + + const { uuid: userId, groups = null } = profile; + + setUserID(userId); + + trackEvent('Authentication', 'UILogin', { + eventLabel: "Authenticated ClientSide", + name: userId, + userId, + userGroups: groups && (JSON.stringify(groups.sort())) + }); + + // Attempt to preserve hash, if any, but don't scroll to it. + const windowHash = '/';//(window && window.location && window.location.hash) || ''; + navigate(windowHash, { "inPlace" : true, "dontScrollToTop" : !!(windowHash) }); + + }, 'GET', () => { + throw new Error('Request to profile URL failed.'); + }); + } else { + console.log('in failed user profile fetch'); + throw new Error('No profile URL found in user_actions.'); + } + }).catch((error) => { + // Handle Errors + console.log(error); + + this.setState({ "isLoading": false }); + setUserID(null); + + if (typeof errorCallback === "function") { + errorCallback(error); + } + }); + } + + onLogin(profile){ + console.log("Logged in", profile); + } + + onRegistrationCancel() { + // even user is not registered and logged in yet, jwtToken (having redis key) has already been created. + // performLogout clears any user-specific data stored during the oauth process. + performLogout().then(()=>{ + + this.setState({ "isLoading" : false }); + + // Remove from analytics session + setUserID(null); + + // Attempt to preserve hash, if any, but don't scroll to it. + const windowHash = '/';//(window && window.location && window.location.hash) || ''; + console.info("Logged out; re-loading context"); + navigate(windowHash, { "inPlace" : true, "dontScrollToTop" : !!(windowHash) }); + }); + } + + showLock(){ + onLoginNavItemClick(); + } + + render() { + return ( + + ); + } +} \ No newline at end of file diff --git a/src/encoded/tests/conftest.py b/src/encoded/tests/conftest.py index 12cc6c00f6..2ef189c21b 100644 --- a/src/encoded/tests/conftest.py +++ b/src/encoded/tests/conftest.py @@ -13,12 +13,14 @@ from dcicutils.ff_mocks import NO_SERVER_FIXTURES from dcicutils.qa_utils import notice_pytest_fixtures, MockFileSystem +from dcicutils.redis_utils import create_redis_client from pyramid.request import apply_request_extensions from pyramid.testing import DummyRequest from pyramid.threadlocal import get_current_registry, manager as threadlocal_manager from snovault import DBSESSION, ROOT, UPGRADER from snovault.elasticsearch import ELASTIC_SEARCH, create_mapping from snovault.util import generate_indexer_namespace_for_testing +from pytest_redis import factories from .conftest_settings import make_app_settings_dictionary from .. import main from snovault.loadxl import load_all @@ -76,6 +78,17 @@ def app_settings(request, wsgi_server_host_port, conn, DBSession): # noQA - We return settings +@pytest.fixture(scope='session') +def ras_app_settings(request, wsgi_server_host_port, conn, DBSession): # noQA - We didn't choose the fixture name. + notice_pytest_fixtures(request, wsgi_server_host_port, conn, DBSession) + settings = make_app_settings_dictionary() + settings['auth0.audiences'] = 'http://%s:%s' % wsgi_server_host_port + settings['auth0.domain'] = 'https://stsstg.nih.gov' + settings['auth0.client'] = 'dummy-client-id' + settings[DBSESSION] = DBSession + return settings + + INDEXER_NAMESPACE_FOR_TESTING = generate_indexer_namespace_for_testing('fourfront') @@ -97,6 +110,24 @@ def es_app_settings(wsgi_server_host_port, elasticsearch_server, postgresql_serv return settings +# special redis fixture for the session +redis_proc = factories.redis_proc(port=8888) + + +@pytest.fixture(scope='session') +def redis_app_settings(wsgi_server_host_port, postgresql_server, redis_proc): + settings = make_app_settings_dictionary() + settings['create_tables'] = True + settings['persona.audiences'] = 'http://%s:%s' % wsgi_server_host_port # 2-tuple such as: ('localhost', '5000') + settings['sqlalchemy.url'] = postgresql_server + settings['collection_datastore'] = 'elasticsearch' + settings['item_datastore'] = 'elasticsearch' + settings['indexer'] = True + settings['indexer.namespace'] = INDEXER_NAMESPACE_FOR_TESTING + settings['redis.server'] = f'redis://127.0.0.1:8888' + return settings + + def pytest_configure(): logging.basicConfig(format='%(message)s') logging.getLogger('sqlalchemy.engine').setLevel(logging.WARNING) @@ -158,6 +189,11 @@ def app(app_settings): return main({}, **app_settings) +@pytest.fixture(scope='session') +def ras_app(ras_app_settings): + return main({}, **ras_app_settings) + + @pytest.fixture(scope='session') def es_app(es_app_settings, **kwargs): """ @@ -170,6 +206,13 @@ def es_app(es_app_settings, **kwargs): return app +@pytest.fixture(scope='session') +def redis_app(redis_app_settings, **kwargs): + """ App that uses Redis and Postgres for testing session tokens """ + app = main({}, **redis_app_settings) + return app + + @pytest.fixture def registry(app): return app.registry @@ -284,6 +327,26 @@ def es_testapp(es_app): return webtest.TestApp(es_app, environ) +@pytest.fixture(scope='session') +def redis_testapp(redis_app): + """ Testapp with Postgres and Redis """ + environ = { + 'HTTP_ACCEPT': 'application/json', + 'REMOTE_USER': 'TEST', + } + return webtest.TestApp(redis_app, environ) + + +@pytest.fixture(scope='session') +def ras_testapp(ras_app): + """ Testapp for use with RAS """ + environ = { + 'HTTP_ACCEPT': 'application/json', + 'REMOTE_USER': 'TEST', + } + return webtest.TestApp(ras_app, environ) + + @pytest.fixture def html_es_testapp(es_app): """TestApp with ES + Postgres for TEST user, accepting text/html content.""" diff --git a/src/encoded/tests/data/inserts/experiment_seq.json b/src/encoded/tests/data/inserts/experiment_seq.json index f9c95735d9..f756e628e2 100644 --- a/src/encoded/tests/data/inserts/experiment_seq.json +++ b/src/encoded/tests/data/inserts/experiment_seq.json @@ -18,8 +18,7 @@ "antibody_lot_id": "11111111", "average_fragment_size": 100, "fragmentation_method": "chemical", - "library_preparation_date": "2017-01-01", - "disease_term": "COVID-19" + "library_preparation_date": "2017-01-01" }, { "accession":"4DNEXO6777B1", diff --git a/src/encoded/tests/test_ras_login.py b/src/encoded/tests/test_ras_login.py new file mode 100644 index 0000000000..83bbc1311c --- /dev/null +++ b/src/encoded/tests/test_ras_login.py @@ -0,0 +1,8 @@ +import pytest + + +def test_login_redirects_to_ras(ras_testapp): + """ Tests that a POST to /login will redirect the user to the RAS login page """ + res = ras_testapp.get('/login') + assert '307' in res.status + assert 'nih.gov' in res.location diff --git a/src/encoded/tests/test_redis.py b/src/encoded/tests/test_redis.py new file mode 100644 index 0000000000..04467231e5 --- /dev/null +++ b/src/encoded/tests/test_redis.py @@ -0,0 +1,18 @@ +import pytest +from dcicutils.redis_tools import RedisSessionToken + + +pytestmark = [pytest.mark.setone, pytest.mark.working] + + +class TestRedisSession: + """ Class for testing Redis sessions + No automatic setup/teardown at this point, so redis entries will persist + """ + + def test_redis_session_basic(self, redis_testapp): + """ Tests that we can grab a handle to redis when the server URL + is specified. + """ + assert redis_testapp.app.registry.settings['redis.server'] + assert redis_testapp.app.registry['redis'] diff --git a/src/encoded/types/file.py b/src/encoded/types/file.py index a8798112b0..a6b96314ed 100644 --- a/src/encoded/types/file.py +++ b/src/encoded/types/file.py @@ -29,6 +29,7 @@ load_schema, abstract_collection, ) +from snovault.authentication import session_properties from snovault.attachment import ItemWithAttachment from snovault.crud_views import ( collection_add, @@ -53,8 +54,6 @@ parse_qs, urlparse, ) -from uuid import uuid4 -from ..authentication import session_properties from ..search import make_search_subreq from ..util import check_user_is_logged_in from .base import (