From a867834f2703e8c46211c98318f7d24ccd183ded Mon Sep 17 00:00:00 2001 From: Oliver Wissett Date: Fri, 4 Oct 2024 19:26:03 +0100 Subject: [PATCH 1/3] Adds non-blocking mode to hydra_submitit_launcher (#2479) --- .../hydra_submitit_launcher/config.py | 5 +++++ .../hydra_submitit_launcher/submitit_launcher.py | 16 ++++++++++++++-- .../tests/test_submitit_launcher.py | 13 +++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/plugins/hydra_submitit_launcher/hydra_plugins/hydra_submitit_launcher/config.py b/plugins/hydra_submitit_launcher/hydra_plugins/hydra_submitit_launcher/config.py index 6da5658a862..0345eb86086 100644 --- a/plugins/hydra_submitit_launcher/hydra_plugins/hydra_submitit_launcher/config.py +++ b/plugins/hydra_submitit_launcher/hydra_plugins/hydra_submitit_launcher/config.py @@ -28,6 +28,11 @@ class BaseQueueConf: # redirect stderr to stdout stderr_to_stdout: bool = False + # If True, the launcher will not wait for the job to finish (useful for very long runs) + # This value is not passed to submitit, it is used by the launcher itself. When enabled, + # a set of sentinel values are returned to the sweeper to indicate that the job is running. + no_block: bool = False + @dataclass class SlurmQueueConf(BaseQueueConf): diff --git a/plugins/hydra_submitit_launcher/hydra_plugins/hydra_submitit_launcher/submitit_launcher.py b/plugins/hydra_submitit_launcher/hydra_plugins/hydra_submitit_launcher/submitit_launcher.py index 1efc8e4ce86..de42635d50f 100644 --- a/plugins/hydra_submitit_launcher/hydra_plugins/hydra_submitit_launcher/submitit_launcher.py +++ b/plugins/hydra_submitit_launcher/hydra_plugins/hydra_submitit_launcher/submitit_launcher.py @@ -5,7 +5,13 @@ from typing import Any, Dict, List, Optional, Sequence from hydra.core.singleton import Singleton -from hydra.core.utils import JobReturn, filter_overrides, run_job, setup_globals +from hydra.core.utils import ( + JobReturn, + filter_overrides, + run_job, + setup_globals, + JobStatus, +) from hydra.plugins.launcher import Launcher from hydra.types import HydraContext, TaskFunction from omegaconf import DictConfig, OmegaConf, open_dict @@ -19,6 +25,7 @@ class BaseSubmititLauncher(Launcher): _EXECUTOR = "abstract" def __init__(self, **params: Any) -> None: + self.no_block = params.pop("no_block", False) self.params = {} for k, v in params.items(): if OmegaConf.is_config(v): @@ -142,7 +149,12 @@ def launch( ) jobs = executor.map_array(self, *zip(*job_params)) - return [j.results()[0] for j in jobs] + + if self.no_block: + sentinal = JobReturn(status=JobStatus.COMPLETED, _return_value=None) + return [sentinal] * num_jobs + else: + return [j.results()[0] for j in jobs] class LocalLauncher(BaseSubmititLauncher): diff --git a/plugins/hydra_submitit_launcher/tests/test_submitit_launcher.py b/plugins/hydra_submitit_launcher/tests/test_submitit_launcher.py index 65f7a0e6a4c..7d9bd943bc5 100644 --- a/plugins/hydra_submitit_launcher/tests/test_submitit_launcher.py +++ b/plugins/hydra_submitit_launcher/tests/test_submitit_launcher.py @@ -61,3 +61,16 @@ def test_example(tmpdir: Path) -> None: ], allow_warnings=True, ) + + +def test_example_no_block(tmpdir: Path) -> None: + run_python_script( + [ + "example/my_app.py", + "-m", + f"hydra.sweep.dir={tmpdir}", + "hydra/launcher=submitit_local", + "hydra.launcher.no_block=True", + ], + allow_warnings=True, + ) From ac6cb0481c015253989aa7ca78869ad1778bf27f Mon Sep 17 00:00:00 2001 From: Oliver Wissett Date: Fri, 4 Oct 2024 19:28:52 +0100 Subject: [PATCH 2/3] isort --- .../hydra_plugins/hydra_submitit_launcher/submitit_launcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/hydra_submitit_launcher/hydra_plugins/hydra_submitit_launcher/submitit_launcher.py b/plugins/hydra_submitit_launcher/hydra_plugins/hydra_submitit_launcher/submitit_launcher.py index de42635d50f..6d6d8ad53ef 100644 --- a/plugins/hydra_submitit_launcher/hydra_plugins/hydra_submitit_launcher/submitit_launcher.py +++ b/plugins/hydra_submitit_launcher/hydra_plugins/hydra_submitit_launcher/submitit_launcher.py @@ -7,10 +7,10 @@ from hydra.core.singleton import Singleton from hydra.core.utils import ( JobReturn, + JobStatus, filter_overrides, run_job, setup_globals, - JobStatus, ) from hydra.plugins.launcher import Launcher from hydra.types import HydraContext, TaskFunction From 85d2fa7ed506428edcac10ff67a3317d691b0680 Mon Sep 17 00:00:00 2001 From: Oliver Wissett Date: Tue, 22 Oct 2024 12:07:05 +0100 Subject: [PATCH 3/3] Update plugins/hydra_submitit_launcher/hydra_plugins/hydra_submitit_launcher/submitit_launcher.py Co-authored-by: vilhub --- .../hydra_submitit_launcher/submitit_launcher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/hydra_submitit_launcher/hydra_plugins/hydra_submitit_launcher/submitit_launcher.py b/plugins/hydra_submitit_launcher/hydra_plugins/hydra_submitit_launcher/submitit_launcher.py index 6d6d8ad53ef..e4d152f68cb 100644 --- a/plugins/hydra_submitit_launcher/hydra_plugins/hydra_submitit_launcher/submitit_launcher.py +++ b/plugins/hydra_submitit_launcher/hydra_plugins/hydra_submitit_launcher/submitit_launcher.py @@ -151,8 +151,8 @@ def launch( jobs = executor.map_array(self, *zip(*job_params)) if self.no_block: - sentinal = JobReturn(status=JobStatus.COMPLETED, _return_value=None) - return [sentinal] * num_jobs + sentinel = JobReturn(status=JobStatus.COMPLETED, _return_value=None) + return [sentinel] * num_jobs else: return [j.results()[0] for j in jobs]