From 83ab69d4e9da5f13850798edefb67a24bed183f6 Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Sun, 22 Sep 2024 08:21:59 +0200 Subject: [PATCH] WIP add export-contest script --- misc-tools/Makefile | 3 +- misc-tools/dj_utils.py | 35 ++++++------ misc-tools/export-contest.in | 102 +++++++++++++++++++++++++++++++++++ 3 files changed, 124 insertions(+), 16 deletions(-) create mode 100755 misc-tools/export-contest.in diff --git a/misc-tools/Makefile b/misc-tools/Makefile index 883a324538..f039d6d84b 100644 --- a/misc-tools/Makefile +++ b/misc-tools/Makefile @@ -9,7 +9,8 @@ include $(TOPDIR)/Makefile.global TARGETS = OBJECTS = -SUBST_DOMSERVER = fix_permissions configure-domjudge import-contest +SUBST_DOMSERVER = fix_permissions configure-domjudge import-contest \ + export-contest SUBST_JUDGEHOST = dj_make_chroot dj_run_chroot dj_make_chroot_docker \ dj_judgehost_cleanup diff --git a/misc-tools/dj_utils.py b/misc-tools/dj_utils.py index c37a3508f4..35a10ae0bd 100644 --- a/misc-tools/dj_utils.py +++ b/misc-tools/dj_utils.py @@ -44,17 +44,10 @@ def parse_api_response(name: str, response: requests.Response): if response.status_code == 204: return None - # We got a successful HTTP response. It worked. Return the full response - try: - result = json.loads(response.text) - except json.decoder.JSONDecodeError as e: - print(response.text) - raise RuntimeError(f'Failed to JSON decode the response for API request {name}') - - return result + return response.text -def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}): +def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}, decode: bool = True): '''Perform an API call to the given endpoint and return its data. Based on whether `domjudge_webapp_folder_or_api_url` is a folder or URL this @@ -64,16 +57,18 @@ def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}): name (str): the endpoint to call method (str): the method to use, GET or PUT are supported jsonData (dict): the JSON data to PUT. Only used when method is PUT + decode (bool): whether to decode the returned JSON data, default true Returns: The endpoint contents. Raises: - RuntimeError when the response is not JSON or the HTTP status code is non 2xx. + RuntimeError when the HTTP status code is non-2xx or the response + cannot be JSON decoded. ''' if os.path.isdir(domjudge_webapp_folder_or_api_url): - return api_via_cli(name, method, {}, {}, jsonData) + result = api_via_cli(name, method, {}, {}, jsonData) else: global ca_check url = f'{domjudge_webapp_folder_or_api_url}/{name}' @@ -97,7 +92,17 @@ def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}): return do_api_request(name) except requests.exceptions.RequestException as e: raise RuntimeError(e) - return parse_api_response(name, response) + result = parse_api_response(name, response) + + if decode: + try: + result = json.loads(result) + except json.decoder.JSONDecodeError as e: + print(result) + raise RuntimeError(f'Failed to JSON decode the response for API request {name}') + + return result + def upload_file(name: str, apifilename: str, file: str, data: dict = {}): '''Upload the given file to the API at the given path with the given name. @@ -118,7 +123,7 @@ def upload_file(name: str, apifilename: str, file: str, data: dict = {}): ''' if os.path.isdir(domjudge_webapp_folder_or_api_url): - return api_via_cli(name, 'POST', data, {apifilename: file}) + response = api_via_cli(name, 'POST', data, {apifilename: file}) else: global ca_check files = [(apifilename, open(file, 'rb'))] @@ -152,7 +157,7 @@ def api_via_cli(name: str, method: str = 'GET', data: dict = {}, files: dict = { jsonData (dict): the JSON data to use. Only used when method is POST or PUT Returns: - The parsed endpoint contents. + The endpoint contents. Raises: RuntimeError when the command exit code is not 0. @@ -183,4 +188,4 @@ def api_via_cli(name: str, method: str = 'GET', data: dict = {}, files: dict = { print(response) raise RuntimeError(f'API request {name} failed') - return json.loads(response) + return response diff --git a/misc-tools/export-contest.in b/misc-tools/export-contest.in new file mode 100755 index 0000000000..2351c137a4 --- /dev/null +++ b/misc-tools/export-contest.in @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 + +''' +export-contest -- Convenience script to export a contest (including metadata, +teams and problems) from the command line. Defaults to using the CLI interface; +Specify a DOMjudge API URL as to use that. + +Reads credentials from ~/.netrc when using the API. + +Part of the DOMjudge Programming Contest Jury System and licensed +under the GNU GPL. See README and COPYING for details. +''' + +import json +import sys +# from concurrent.futures import ThreadPoolExecutor +# from multiprocessing import Pool +from pathlib import Path + +sys.path.append('@domserver_libdir@') +import dj_utils + +cid = None +webappdir = '@domserver_webappdir@' + + +def usage(): + print(f'Usage: {sys.argv[0]} []') + exit(1) + + +def api_to_file(endpoint: str, filename: str): + print(f"Fetching '{endpoint}' to '{filename}'") + data = dj_utils.do_api_request(endpoint, decode=False) + with open(filename, 'w') as f: + f.write(data) + + return data + + +def download_submission(submission): + d = f'submissions/{submission["id"]}' + Path(d).mkdir(parents=True, exist_ok=True) + for f in submission['files']: + if f['mime'] == 'application/zip': + print(f"Downloading '{f['href']}'") + data = dj_utils.do_api_request(f['href'], decode=False) + with open(f'{d}/files.zip', 'w') as f: + f.write(data) + break + + +if len(sys.argv) == 1: + dj_utils.domjudge_webapp_folder_or_api_url = webappdir +elif len(sys.argv) == 2: + dj_utils.domjudge_webapp_folder_or_api_url = sys.argv[1] +else: + usage() + + +user_data = dj_utils.do_api_request('user') +if 'admin' not in user_data['roles']: + print('Your user does not have the \'admin\' role, can not export.') + exit(1) + + +contest_id = 'wf48_systest2' + +for endpoint in [ + 'accounts', + 'awards', + 'balloons', + 'clarifications', + 'groups', + 'judgements', + 'languages', + 'organizations', + 'problems', + 'runs', + 'scoreboard', + 'submissions', + 'teams', + ]: + data = api_to_file(f'contests/{contest_id}/{endpoint}', f'{endpoint}.json') + if endpoint == 'submissions': + submissions = json.loads(data) + +api_to_file(f'contests/{contest_id}/event-feed?stream=false', 'event-feed.ndjson') + + +for submission in submissions: + download_submission(submission) + +# with Pool(processes=10) as pool: +# result = pool.map_async(download_submission, submissions) +# +# print(result) +# result.wait() + +# with ThreadPoolExecutor(20) as executor: +# for submission in submissions: +# executor.submit(download_submission, submission)