Add storage namespace

dmoj/checkers/bridged.py

@@ -10,13 +10,13 @@
 from dmoj.utils.unicode import utf8text
 
 
-def get_executor(problem_id, files, flags, lang, compiler_time_limit):
+def get_executor(problem_id, storage_namespace, files, flags, lang, compiler_time_limit):
     if isinstance(files, str):
         filenames = [files]
     elif isinstance(files.unwrap(), list):
         filenames = list(files.unwrap())
 
-    filenames = [os.path.join(get_problem_root(problem_id), f) for f in filenames]
+    filenames = [os.path.join(get_problem_root(problem_id, storage_namespace), f) for f in filenames]
     executor = compile_with_auxiliary_files(filenames, flags, lang, compiler_time_limit)
 
     return executor
@@ -40,6 +40,7 @@ def check(
     input_name=None,
     output_name=None,
     treat_checker_points_as_percentage=False,
+    storage_namespace=None,
     **kwargs,
 ) -> CheckerResult:
 
@@ -52,7 +53,7 @@ def check(
         flags.append('-DTHEMIS')
     elif type == 'cms':
         flags.append('-DCMS')
-    executor = get_executor(problem_id, files, flags, lang, compiler_time_limit)
+    executor = get_executor(problem_id, storage_namespace, files, flags, lang, compiler_time_limit)
 
     if type not in contrib_modules:
         raise InternalError('%s is not a valid contrib module' % type)

dmoj/commands/rejudge.py

@@ -14,7 +14,7 @@ def execute(self, line: str) -> None:
         problem_id, lang, src, tl, ml = self.get_submission_data(args.submission_id)
 
         self.judge.begin_grading(
-            Submission(self.judge.submission_id_counter, problem_id, lang, src, tl, ml, False, {}),
+            Submission(self.judge.submission_id_counter, problem_id, None, lang, src, tl, ml, False, {}),
             blocking=True,
             report=print,
         )

dmoj/commands/resubmit.py

@@ -47,7 +47,7 @@ def execute(self, line: str) -> None:
         self.judge.graded_submissions.append((problem_id, lang, src, tl, ml))
         try:
             self.judge.begin_grading(
-                Submission(self.judge.submission_id_counter, problem_id, lang, src, tl, ml, False, {}),
+                Submission(self.judge.submission_id_counter, problem_id, None, lang, src, tl, ml, False, {}),
                 blocking=True,
                 report=print,
             )

dmoj/commands/submit.py

@@ -70,7 +70,15 @@ def execute(self, line: str) -> None:
         try:
             self.judge.begin_grading(
                 Submission(
-                    self.judge.submission_id_counter, problem_id, language_id, src, time_limit, memory_limit, False, {}
+                    self.judge.submission_id_counter,
+                    problem_id,
+                    None,
+                    language_id,
+                    src,
+                    time_limit,
+                    memory_limit,
+                    False,
+                    {},
                 ),
                 blocking=True,
                 report=print,

dmoj/graders/bridged.py

@@ -129,7 +129,7 @@ def _generate_interactor_binary(self) -> BaseExecutor:
             filenames = [files]
         elif isinstance(files.unwrap(), list):
             filenames = list(files.unwrap())
-        problem_root = get_problem_root(self.problem.id)
+        problem_root = get_problem_root(self.problem.id, self.problem.storage_namespace)
         assert problem_root is not None
         filenames = [os.path.join(problem_root, f) for f in filenames]
         flags = self.handler_data.get('flags', [])

dmoj/graders/communication.py

@@ -228,7 +228,7 @@ def _generate_manager_binary(self) -> BaseExecutor:
             filenames = [files]
         elif isinstance(files.unwrap(), list):
             filenames = list(files.unwrap())
-        problem_root = get_problem_root(self.problem.id)
+        problem_root = get_problem_root(self.problem.id, self.problem.storage_namespace)
         assert problem_root is not None
         filenames = [os.path.join(problem_root, f) for f in filenames]
         flags = self.handler_data.manager.get('flags', [])

dmoj/graders/custom.py

@@ -7,7 +7,9 @@
 class CustomGrader:
     def __init__(self, judge, problem, language, source):
         self.judge = judge
-        self.mod = load_module_from_file(os.path.join(get_problem_root(problem.id), problem.config['custom_judge']))
+        self.mod = load_module_from_file(
+            os.path.join(get_problem_root(problem.id, problem.storage_namespace), problem.config['custom_judge'])
+        )
         self._grader = self.mod.Grader(judge, problem, language, source)
 
     def __getattr__(self, item):

dmoj/judge.py

@@ -61,6 +61,7 @@ class IPC(Enum):
     [
         ('id', int),
         ('problem_id', str),
+        ('storage_namespace', Optional[str]),
         ('language', str),
         ('source', str),
         ('time_limit', float),
@@ -448,7 +449,11 @@ def _report_unhandled_exception() -> None:
 
     def _grade_cases(self) -> Generator[Tuple[IPC, tuple], None, None]:
         problem = Problem(
-            self.submission.problem_id, self.submission.time_limit, self.submission.memory_limit, self.submission.meta
+            self.submission.problem_id,
+            self.submission.time_limit,
+            self.submission.memory_limit,
+            self.submission.meta,
+            storage_namespace=self.submission.storage_namespace,
         )
 
         try:

dmoj/judgeenv.py

@@ -3,6 +3,7 @@
 import logging
 import os
 import ssl
+from collections import defaultdict
 from fnmatch import fnmatch
 from operator import itemgetter
 from typing import Dict, Iterable, List, Optional, Set, Tuple
@@ -15,6 +16,7 @@
 from dmoj.utils.glob_ext import find_glob_root
 from dmoj.utils.unicode import utf8text
 
+storage_namespaces: Dict[Optional[str], List[str]] = {}
 problem_globs: List[str] = []
 problem_watches: List[str] = []
 env: ConfigNode = ConfigNode(
@@ -74,13 +76,18 @@
 only_executors: Set[str] = set()
 exclude_executors: Set[str] = set()
 
-_problem_root_cache: Dict[str, str] = {}
-_problem_roots_cache: Optional[List[str]] = None
-_supported_problems_cache = None
+
+class StorageNamespaceCache:
+    problem_root_cache: Dict[str, str] = {}
+    problem_roots_cache: Optional[List[str]] = None
+    supported_problems_cache: Optional[List[Tuple[str, float]]] = None
+
+
+_storage_namespace_cache: Dict[Optional[str], StorageNamespaceCache] = defaultdict(StorageNamespaceCache)
 
 
 def load_env(cli: bool = False, testsuite: bool = False) -> None:  # pragma: no cover
-    global problem_globs, only_executors, exclude_executors, log_file, server_host, server_port, no_ansi, no_ansi_emu, skip_self_test, env, startup_warnings, no_watchdog, problem_regex, case_regex, api_listen, secure, no_cert_check, cert_store, problem_watches, cli_history_file, cli_command, log_level
+    global storage_namespaces, problem_globs, only_executors, exclude_executors, log_file, server_host, server_port, no_ansi, no_ansi_emu, skip_self_test, env, startup_warnings, no_watchdog, problem_regex, case_regex, api_listen, secure, no_cert_check, cert_store, problem_watches, cli_history_file, cli_command, log_level
 
     if cli:
         description = 'Starts a shell for interfacing with a local judge instance.'
@@ -207,15 +214,23 @@ def load_env(cli: bool = False, testsuite: bool = False) -> None:  # pragma: no
         env['key'] = os.environ['DMOJ_JUDGE_KEY']
 
     if not testsuite:
-        problem_globs = env.problem_storage_globs
-        if problem_globs is None:
-            raise SystemExit(f'`problem_storage_globs` not specified in "{model_file}"; no problems available to grade')
+        storage_namespaces[None] = env.problem_storage_globs or []
+        storage_namespaces.update(env.storage_namespaces or {})
 
+        all_problem_globs = []
+        for globs in storage_namespaces.values():
+            all_problem_globs.extend(globs)
+
+        if not all_problem_globs:
+            raise SystemExit('no problems available to grade')
+
+        problem_globs = storage_namespaces[None]
         problem_watches = problem_globs
     else:
         if not os.path.isdir(args.tests_dir):
             raise SystemExit('Invalid tests directory')
         problem_globs = [os.path.join(args.tests_dir, '*')]
+        storage_namespaces[None] = problem_globs
 
         import re
 
@@ -230,44 +245,48 @@ def load_env(cli: bool = False, testsuite: bool = False) -> None:  # pragma: no
             except re.error:
                 raise SystemExit('Invalid case regex')
 
+    for namespace in storage_namespaces:
+        _storage_namespace_cache[namespace] = StorageNamespaceCache()
+
     skip_first_scan = False if cli else args.skip_first_scan
     if not skip_first_scan:
         # Populate cache and send warnings
         get_supported_problems_and_mtimes()
     else:
-        global _problem_roots_cache
-        global _supported_problems_cache
-        _problem_roots_cache = [str(root) for root in map(find_glob_root, problem_globs)]
-        _supported_problems_cache = []
+        for namespace, globs in storage_namespaces.items():
+            cache = _storage_namespace_cache[namespace]
+            cache.problem_roots_cache = [str(root) for root in map(find_glob_root, globs)]
+            cache.supported_problems_cache = []
 
 
 def get_problem_watches():
     return problem_watches
 
 
-def get_problem_root(problem_id) -> Optional[str]:
-    cached_root = _problem_root_cache.get(problem_id)
+def get_problem_root(problem_id, namespace=None) -> Optional[str]:
+    cache = _storage_namespace_cache[namespace]
+    cached_root = cache.problem_root_cache.get(problem_id)
     if cached_root is None or not os.path.isfile(os.path.join(cached_root, 'init.yml')):
-        for root_dir in get_problem_roots():
+        for root_dir in get_problem_roots(namespace):
             problem_root_dir = os.path.join(root_dir, problem_id)
             problem_config = os.path.join(problem_root_dir, 'init.yml')
             if os.path.isfile(problem_config):
                 if problem_globs and not any(
                     fnmatch(problem_config, os.path.join(problem_glob, 'init.yml')) for problem_glob in problem_globs
                 ):
                     continue
-                _problem_root_cache[problem_id] = problem_root_dir
+                cache.problem_root_cache[problem_id] = problem_root_dir
                 break
         else:
             return None
 
-    return _problem_root_cache[problem_id]
+    return cache.problem_root_cache[problem_id]
 
 
-def get_problem_roots() -> List[str]:
-    global _problem_roots_cache
-    assert _problem_roots_cache is not None
-    return _problem_roots_cache
+def get_problem_roots(namespace=None) -> List[str]:
+    cache = _storage_namespace_cache[namespace]
+    assert cache.problem_roots_cache is not None
+    return cache.problem_roots_cache
 
 
 def get_supported_problems_and_mtimes(warnings: bool = True, force_update: bool = False) -> List[Tuple[str, float]]:
@@ -277,11 +296,10 @@ def get_supported_problems_and_mtimes(warnings: bool = True, force_update: bool
         A list of all problems in tuple format: (problem id, mtime)
     """
 
-    global _problem_roots_cache
-    global _supported_problems_cache
+    cache = _storage_namespace_cache[None]
 
-    if _supported_problems_cache is not None and not force_update:
-        return _supported_problems_cache
+    if cache.supported_problems_cache is not None and not force_update:
+        return cache.supported_problems_cache
 
     problems = []
     root_dirs = []
@@ -309,8 +327,8 @@ def get_supported_problems_and_mtimes(warnings: bool = True, force_update: bool
                     problem_dirs[problem] = problem_dir
                     problems.append((problem, os.path.getmtime(problem_dir)))
 
-    _problem_roots_cache = root_dirs
-    _supported_problems_cache = problems
+    cache.problem_roots_cache = root_dirs
+    cache.supported_problems_cache = problems
 
     return problems
 

dmoj/packet.py

@@ -260,6 +260,7 @@ def _receive_packet(self, packet: dict):
                 Submission(
                     id=packet['submission-id'],
                     problem_id=packet['problem-id'],
+                    storage_namespace=packet.get('storage-namespace', None),
                     language=packet['language'],
                     source=packet['source'],
                     time_limit=float(packet['time-limit']),

dmoj/problem.py

@@ -50,6 +50,7 @@ class BaseTestCase:
 
 class Problem:
     id: str
+    storage_namespace: Optional[str]
     time_limit: float
     memory_limit: int
     meta: ConfigNode
@@ -58,8 +59,11 @@ class Problem:
     problem_data: 'ProblemDataManager'
     config: 'ProblemConfig'
 
-    def __init__(self, problem_id: str, time_limit: float, memory_limit: int, meta: dict) -> None:
+    def __init__(
+        self, problem_id: str, time_limit: float, memory_limit: int, meta: dict, storage_namespace: Optional[str] = None
+    ) -> None:
         self.id = problem_id
+        self.storage_namespace = storage_namespace
         self.time_limit = time_limit
         self.memory_limit = memory_limit
         self.meta = ConfigNode(meta)
@@ -68,7 +72,7 @@ def __init__(self, problem_id: str, time_limit: float, memory_limit: int, meta:
         self._testcase_counter = 0
 
         # Cache root dir so that we don't need to scan all roots (potentially very slow on networked mount).
-        root_dir = get_problem_root(problem_id)
+        root_dir = get_problem_root(problem_id, storage_namespace)
         assert root_dir is not None
         self.root_dir = root_dir
         self.problem_data = ProblemDataManager(self.root_dir)
@@ -386,7 +390,7 @@ def _run_generator(self, gen: Union[str, ConfigNode], args: Optional[Iterable[st
         compiler_time_limit = env.generator_compiler_time_limit
         lang = None  # Default to C/C++
 
-        base = get_problem_root(self.problem.id)
+        base = get_problem_root(self.problem.id, self.problem.storage_namespace)
         assert base is not None
         filenames: Union[str, list]
         if isinstance(gen, str):
@@ -489,11 +493,14 @@ def checker(self) -> partial:
         if not hasattr(checker, 'check') or not callable(checker.check):
             raise InvalidInitException('malformed checker: no check method found')
 
+        params['storage_namespace'] = self.problem.storage_namespace
+
         # Themis checker need input name and output name
         if self.config['in']:
             params['input_name'] = self.config['in']
         if self.config['out']:
             params['output_name'] = self.config['out']
+
         return partial(checker.check, **params)
 
     def free_data(self) -> None:

dmoj/testsuite.py

@@ -290,7 +290,7 @@ def output_case(data):
                 extended_feedback_cases,
             )
             self.judge.begin_grading(
-                Submission(self.sub_id, problem, language, source, time, memory, False, meta),
+                Submission(self.sub_id, problem, None, language, source, time, memory, False, meta),
                 blocking=True,
                 report=output_case,
             )