From b8b48dd0fcfda7f72d4ab3d406e30ec9780e6434 Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Sat, 20 Jul 2024 00:05:15 +0200 Subject: [PATCH 1/2] Place the --output-mode in front of job scritp (#28) Fixes #27 `--output-mode` option was wrongly put in the end of the submit command which cause the problem that stdout of submit command is not json parsable. The test was not able to catch it because the command was defined in the test but not directly get from the scheduler. Test is update to using the commond generated from scheduler and therefore being tested directly. --- aiida_hyperqueue/scheduler.py | 6 +++--- tests/test_scheduler.py | 9 +++++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/aiida_hyperqueue/scheduler.py b/aiida_hyperqueue/scheduler.py index 5b31b2f..7ac08b4 100644 --- a/aiida_hyperqueue/scheduler.py +++ b/aiida_hyperqueue/scheduler.py @@ -148,7 +148,7 @@ def _get_submit_command(self, submit_script: str) -> str: submit_script: the path of the submit script relative to the working directory. """ - submit_command = f"hq submit {submit_script} --output-mode=json" + submit_command = f"hq submit --output-mode=json {submit_script}" self.logger.info(f"Submitting with: {submit_command}") @@ -178,10 +178,10 @@ def _parse_submit_output(self, retval: int, stdout: str, stderr: str) -> str: f"in _parse_submit_output{transport_string}: there was some text in stderr: {stderr}" ) - hq_job_dict = json.loads(stdout) try: + hq_job_dict = json.loads(stdout) return str(hq_job_dict["id"]) - except KeyError: + except Exception: # If no valid line is found, log and raise an error self.logger.error( f"in _parse_submit_output{transport_string}: unable to find the job id: {stdout}" diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index e9e3e25..b5c47c2 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -69,7 +69,9 @@ def test_submit_command(): """Test submit command""" scheduler = HyperQueueScheduler() - assert "hq submit job.sh" in scheduler._get_submit_command("job.sh") + assert "hq submit --output-mode=json job.sh" == scheduler._get_submit_command( + "job.sh" + ) def test_parse_submit_command_output(hq_env: HqEnv, valid_submit_script): @@ -78,8 +80,11 @@ def test_parse_submit_command_output(hq_env: HqEnv, valid_submit_script): hq_env.start_worker(cpus="1") Path("_aiidasubmit.sh").write_text(valid_submit_script) + scheduler = HyperQueueScheduler() + args = scheduler._get_submit_command("_aiidasubmit.sh") + args = args.split(" ")[1:] process = hq_env.command( - ["submit", "--output-mode=json", "_aiidasubmit.sh"], + args, wait=False, ignore_stderr=True, ) From 40ac1539de9634545b11271885737d51d65b462e Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Mon, 22 Jul 2024 11:25:42 +0200 Subject: [PATCH 2/2] Change `id` field passed to JobInfo from int to str (#30) fixes #29 The id field of JobInfo is expecting a str. In #24 when parsing JSON output of `hq job list` the json loads will use the int for the id parsed directly. Wrong type causes the subtle issue that when job is waiting it not get into QUEUED state, but immediatly finished and get nothing to parse from output. --- aiida_hyperqueue/scheduler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aiida_hyperqueue/scheduler.py b/aiida_hyperqueue/scheduler.py index 7ac08b4..5523b7f 100644 --- a/aiida_hyperqueue/scheduler.py +++ b/aiida_hyperqueue/scheduler.py @@ -228,7 +228,9 @@ def _parse_joblist_output(self, retval: int, stdout: str, stderr: str) -> list: job_info_list = [] for hq_job_dict in hq_job_info_list: job_info = JobInfo() - job_info.job_id = hq_job_dict["id"] + job_info.job_id = str( + hq_job_dict["id"] + ) # must be str, if it is a int job will not waiting job_info.title = hq_job_dict["name"] stats: t.List[str] = [ stat for stat, v in hq_job_dict["task_stats"].items() if v > 0