diff --git a/.github/workflows/kubernetes-deploy.yaml b/.github/workflows/kubernetes-deploy.yaml index a161bfffb..d39e53f32 100644 --- a/.github/workflows/kubernetes-deploy.yaml +++ b/.github/workflows/kubernetes-deploy.yaml @@ -49,6 +49,7 @@ jobs: --set gateway.application.ray.nodeImage=ray:test \ --set gateway.application.ray.proxyImage=proxy:test \ --set gateway.application.ray.cpu=1 \ + --set gateway.application.debug=1 \ --set gateway.application.limits.keepClusterOnComplete=false \ --set gateway.application.authMockproviderRegistry=test \ --set gateway.application.proxy.enabled=false \ diff --git a/charts/qiskit-serverless/charts/gateway/values.yaml b/charts/qiskit-serverless/charts/gateway/values.yaml index c62ab2aab..30bd0b6e8 100644 --- a/charts/qiskit-serverless/charts/gateway/values.yaml +++ b/charts/qiskit-serverless/charts/gateway/values.yaml @@ -150,8 +150,8 @@ ingress: resources: limits: - cpu: "1500m" - memory: "1Gi" + cpu: "3000m" + memory: "2Gi" requests: cpu: "500m" memory: "700Mi" diff --git a/gateway/api/utils.py b/gateway/api/utils.py index 84aee2468..d70e0b8fc 100644 --- a/gateway/api/utils.py +++ b/gateway/api/utils.py @@ -5,14 +5,18 @@ import inspect import json import logging +import os import re import time import uuid +import sys +import platform from typing import Any, Optional, Tuple, Union, Callable, Dict, List from cryptography.fernet import Fernet from ray.dashboard.modules.job.common import JobStatus from django.conf import settings +from parsley import makeGrammar from .models import Job @@ -228,3 +232,175 @@ def remove_duplicates_from_list(original_list: List[Any]) -> List[Any]: a list without duplicates maintining the order """ return list(OrderedDict.fromkeys(original_list)) + + +# Utilities for parsing python dependency information +# source: https://peps.python.org/pep-0508/#complete-grammar +RAW_DEPENDENCY_GRAMMAR = """ + wsp = ' ' | '\t' + version_cmp = wsp* <'<=' | '<' | '!=' | '==' | '>=' | '>' | '~=' | '==='> + version = wsp* <( letterOrDigit | '-' | '_' | '.' | '*' | '+' | '!' )+> + version_one = version_cmp:op version:v wsp* -> (op, v) + version_many = version_one:v1 (wsp* ',' version_one)*:v2 -> [v1] + v2 + versionspec = ('(' version_many:v ')' ->v) | version_many + urlspec = '@' wsp* + marker_op = version_cmp | (wsp* 'in') | (wsp* 'not' wsp+ 'in') + python_str_c = (wsp | letter | digit | '(' | ')' | '.' | '{' | '}' | + '-' | '_' | '*' | '#' | ':' | ';' | ',' | '/' | '?' | + '[' | ']' | '!' | '~' | '`' | '@' | '$' | '%' | '^' | + '&' | '=' | '+' | '|' | '<' | '>' ) + dquote = '"' + squote = '\\'' + python_str = (squote <(python_str_c | dquote)*>:s squote | + dquote <(python_str_c | squote)*>:s dquote) -> s + env_var = ('python_version' | 'python_full_version' | + 'os_name' | 'sys_platform' | 'platform_release' | + 'platform_system' | 'platform_version' | + 'platform_machine' | 'platform_python_implementation' | + 'implementation_name' | 'implementation_version' | + 'extra' # ONLY when defined by a containing layer + ):varname -> lookup(varname) + marker_var = wsp* (env_var | python_str) + marker_expr = marker_var:l marker_op:o marker_var:r -> (o, l, r) + | wsp* '(' marker:m wsp* ')' -> m + marker_and = marker_expr:l wsp* 'and' marker_expr:r -> ('and', l, r) + | marker_expr:m -> m + marker_or = marker_and:l wsp* 'or' marker_and:r -> ('or', l, r) + | marker_and:m -> m + marker = marker_or + quoted_marker = ';' wsp* marker + identifier_end = letterOrDigit | (('-' | '_' | '.' )* letterOrDigit) + identifier = < letterOrDigit identifier_end* > + name = identifier + extras_list = identifier:i (wsp* ',' wsp* identifier)*:ids -> [i] + ids + extras = '[' wsp* extras_list?:e wsp* ']' -> e + name_req = (name:n wsp* extras?:e wsp* versionspec?:v wsp* quoted_marker?:m + -> (n, e or [], v or [], m)) + url_req = (name:n wsp* extras?:e wsp* urlspec:v (wsp+ | end) quoted_marker?:m + -> (n, e or [], v, m)) + specification = wsp* ( url_req | name_req ):s wsp* -> s + # The result is a tuple - name, list-of-extras, + # list-of-version-constraints-or-a-url, marker-ast or None + + + URI_reference = + URI = scheme ':' hier_part ('?' query )? ( '#' fragment)? + hier_part = ('//' authority path_abempty) | path_absolute | path_rootless | path_empty + absolute_URI = scheme ':' hier_part ( '?' query )? + relative_ref = relative_part ( '?' query )? ( '#' fragment )? + relative_part = '//' authority path_abempty | path_absolute | path_noscheme | path_empty + scheme = letter ( letter | digit | '+' | '-' | '.')* + authority = ( userinfo '@' )? host ( ':' port )? + userinfo = ( unreserved | pct_encoded | sub_delims | ':')* + host = IP_literal | IPv4address | reg_name + port = digit* + IP_literal = '[' ( IPv6address | IPvFuture) ']' + IPvFuture = 'v' hexdig+ '.' ( unreserved | sub_delims | ':')+ + IPv6address = ( + ( h16 ':'){6} ls32 + | '::' ( h16 ':'){5} ls32 + | ( h16 )? '::' ( h16 ':'){4} ls32 + | ( ( h16 ':')? h16 )? '::' ( h16 ':'){3} ls32 + | ( ( h16 ':'){0,2} h16 )? '::' ( h16 ':'){2} ls32 + | ( ( h16 ':'){0,3} h16 )? '::' h16 ':' ls32 + | ( ( h16 ':'){0,4} h16 )? '::' ls32 + | ( ( h16 ':'){0,5} h16 )? '::' h16 + | ( ( h16 ':'){0,6} h16 )? '::' ) + h16 = hexdig{1,4} + ls32 = ( h16 ':' h16) | IPv4address + IPv4address = dec_octet '.' dec_octet '.' dec_octet '.' dec_octet + nz = ~'0' digit + dec_octet = ( + digit # 0-9 + | nz digit # 10-99 + | '1' digit{2} # 100-199 + | '2' ('0' | '1' | '2' | '3' | '4') digit # 200-249 + | '25' ('0' | '1' | '2' | '3' | '4' | '5') )# %250-255 + reg_name = ( unreserved | pct_encoded | sub_delims)* + path = ( + path_abempty # begins with '/' or is empty + | path_absolute # begins with '/' but not '//' + | path_noscheme # begins with a non-colon segment + | path_rootless # begins with a segment + | path_empty ) # zero characters + path_abempty = ( '/' segment)* + path_absolute = '/' ( segment_nz ( '/' segment)* )? + path_noscheme = segment_nz_nc ( '/' segment)* + path_rootless = segment_nz ( '/' segment)* + path_empty = pchar{0} + segment = pchar* + segment_nz = pchar+ + segment_nz_nc = ( unreserved | pct_encoded | sub_delims | '@')+ + # non-zero-length segment without any colon ':' + pchar = unreserved | pct_encoded | sub_delims | ':' | '@' + query = ( pchar | '/' | '?')* + fragment = ( pchar | '/' | '?')* + pct_encoded = '%' hexdig + unreserved = letter | digit | '-' | '.' | '_' | '~' + reserved = gen_delims | sub_delims + gen_delims = ':' | '/' | '?' | '#' | '(' | ')?' | '@' + sub_delims = '!' | '$' | '&' | '\\'' | '(' | ')' | '*' | '+' | ',' | ';' | '=' + hexdig = digit | 'a' | 'A' | 'b' | 'B' | 'c' | 'C' | 'd' | 'D' | 'e' | 'E' | 'f' | 'F' +""" + + +def create_dependency_grammar(grammar=RAW_DEPENDENCY_GRAMMAR): + """Create dependency grammar.""" + + if hasattr(sys, "implementation"): + sys_version = sys.implementation.version + version = f"{sys_version.major}.{sys_version.minor}.{sys_version.micro}" + kind = sys.implementation.version.releaselevel + if kind != "final": + version += kind[0] + str(sys.implementation.version.serial) + implementation_version = version + implementation_name = sys.implementation.name + else: + implementation_version = "0" + implementation_name = "" + bindings = { + "implementation_name": implementation_name, + "implementation_version": implementation_version, + "os_name": os.name, + "platform_machine": platform.machine(), + "platform_python_implementation": platform.python_implementation(), + "platform_release": platform.release(), + "platform_system": platform.system(), + "platform_version": platform.version(), + "python_full_version": platform.python_version(), + "python_version": ".".join(platform.python_version_tuple()[:2]), + "sys_platform": sys.platform, + } + + dependency_grammar = makeGrammar(grammar, {"lookup": bindings.__getitem__}) + return dependency_grammar + + +def parse_dependency(dep, grammar): + """Parse dependency.""" + parsed = grammar(dep).specification() + dep_name = parsed[0] + dep_ver = parsed[2] + + return dep_name, dep_ver + + +def create_dependency_allowlist(): + """ + Create dictionary with allowed dependencies and versions. + + Sample format: + allowlist = { "wheel": ["0.44.0", "0.43.2"] } + where the values for each key are allowed versions of dependency. + """ + try: + with open(settings.GATEWAY_ALLOWLIST_CONFIG, encoding="utf-8", mode="r") as f: + allowlist = json.load(f) + except IOError as e: + logger.error("Unable to open allowlist config file: %s", e) + raise ValueError("Unable to open allowlist config file") from e + except ValueError as e: + logger.error("Unable to decode dependency allowlist: %s", e) + raise ValueError("Unable to decode dependency allowlist") from e + + return allowlist diff --git a/gateway/api/v1/serializers.py b/gateway/api/v1/serializers.py index c738134b7..2b132c117 100644 --- a/gateway/api/v1/serializers.py +++ b/gateway/api/v1/serializers.py @@ -5,9 +5,13 @@ import json import logging from rest_framework.serializers import ValidationError -from django.conf import settings from api import serializers from api.models import Provider +from api.utils import ( + create_dependency_allowlist, + create_dependency_grammar, + parse_dependency, +) logger = logging.getLogger("gateway.serializers") @@ -50,38 +54,16 @@ def validate(self, attrs): # pylint: disable=too-many-branches ) # validate dependencies - # allowlist stored in json config file (eventually via configmap) - # sample: - # allowlist = { "wheel": ["0.44.0", "0.43.2"] } - # where the values for each key are allowed versions of dependency + dependency_grammar = create_dependency_grammar() deps = json.loads(attrs.get("dependencies", None)) - try: - with open( - settings.GATEWAY_ALLOWLIST_CONFIG, encoding="utf-8", mode="r" - ) as f: - allowlist = json.load(f) - except IOError as e: - logger.error("Unable to open allowlist config file: %s", e) - raise ValueError("Unable to open allowlist config file") from e - except ValueError as e: - logger.error("Unable to decode dependency allowlist: %s", e) - raise ValueError("Unable to decode dependency allowlist") from e - - # If no allowlist specified, all dependencies allowed - if len(allowlist.keys()) > 0: + allowlist = create_dependency_allowlist() + if len(allowlist.keys()) > 0: # If no allowlist, all dependencies allowed for d in deps: - dep, ver = d.split("==") - + dep, _ = parse_dependency(d, dependency_grammar) # Determine if a dependency is allowed if dep not in allowlist: raise ValidationError(f"Dependency {dep} is not allowed") - # Determine if a specific version of a dependency is allowed - if allowlist[dep] and ver not in allowlist[dep]: - raise ValidationError( - f"Version {ver} of dependency {dep} is not allowed" - ) - title = attrs.get("title") provider = attrs.get("provider", None) if provider and "/" in title: diff --git a/gateway/requirements.txt b/gateway/requirements.txt index bde19cc27..11a314923 100644 --- a/gateway/requirements.txt +++ b/gateway/requirements.txt @@ -21,3 +21,4 @@ sqlparse>=0.5.0, <1 qiskit-ibm-runtime>=0.29.0 tzdata>=2024.1 django-cors-headers>=4.4.0, <5 +parsley>=1.3, <2 diff --git a/gateway/tests/api/test_v1_serializers.py b/gateway/tests/api/test_v1_serializers.py index 0bba2fbf9..76fafaeaa 100644 --- a/gateway/tests/api/test_v1_serializers.py +++ b/gateway/tests/api/test_v1_serializers.py @@ -323,33 +323,3 @@ def test_upload_program_serializer_blocked_dependency(self): serializer = UploadProgramSerializer(data=data) self.assertFalse(serializer.is_valid()) - - def test_upload_program_serializer_dependency_bad_version(self): - """Tests dependency allowlist.""" - path_to_resource_artifact = os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "..", - "resources", - "artifact.tar", - ) - file_data = File(open(path_to_resource_artifact, "rb")) - upload_file = SimpleUploadedFile( - "artifact.tar", file_data.read(), content_type="multipart/form-data" - ) - - user = models.User.objects.get(username="test_user") - - title = "Hello world" - entrypoint = "pattern.py" - arguments = "{}" - dependencies = '["wheel==0.4.1"]' - - data = {} - data["title"] = title - data["entrypoint"] = entrypoint - data["arguments"] = arguments - data["dependencies"] = dependencies - data["artifact"] = upload_file - - serializer = UploadProgramSerializer(data=data) - self.assertFalse(serializer.is_valid())