From 0326bb858634502b311ae9a03598f3d34c3655aa Mon Sep 17 00:00:00 2001 From: Luke Humphrey Date: Tue, 12 Mar 2024 09:39:41 +0000 Subject: [PATCH 1/5] =?UTF-8?q?=E2=9C=A8=20Set=20ray=20tune=20storage=20pa?= =?UTF-8?q?th=20to=20be=20the=20Optimiser=20data=20directory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/sledo/optimiser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sledo/optimiser.py b/src/sledo/optimiser.py index 72e6e4a..45ea017 100644 --- a/src/sledo/optimiser.py +++ b/src/sledo/optimiser.py @@ -104,6 +104,7 @@ def __init__( num_samples=max_total_trials, ), run_config=train.RunConfig( + storage_path=self.data_dir, name=self.name, ), param_space=search_space, From 4ba08cbd9931872b5d30eb113806a5f60bdc607f Mon Sep 17 00:00:00 2001 From: Luke Humphrey Date: Tue, 12 Mar 2024 09:54:15 +0000 Subject: [PATCH 2/5] =?UTF-8?q?=F0=9F=8E=A8=20Add=20Optimiser.trial=20meth?= =?UTF-8?q?od,=20replacing=20lambda=20function=20in=20=5F=5Finit=5F=5F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/sledo/optimiser.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/sledo/optimiser.py b/src/sledo/optimiser.py index 45ea017..9db36cb 100644 --- a/src/sledo/optimiser.py +++ b/src/sledo/optimiser.py @@ -88,15 +88,9 @@ def __init__( if not self.data_dir.exists(): self.data_dir.mkdir() - # Add a step to the evaluation function reporting the results of each - # design evaluation to the ray tune optimiser. - self.evaluation_function = lambda results_dict: train.report( - self.design_evaluator.evaluate_design(results_dict) - ) - # Instantiate ray tune Tuner object. self.tuner = tune.Tuner( - self.evaluation_function, + self.trial, tune_config=tune.TuneConfig( mode=mode, metric=self.metrics[0], @@ -110,6 +104,22 @@ def __init__( param_space=search_space, ) + def trial(self, parameters: dict): + """Trial evaluation function to be passed to the tuner object. + + Calls the evaluate_design method of the passed DesignEvaluator subclass + with the parameters for a given trial and reports the result to + the optimiser via train.report(). + + Note: users should not need to call this method directly. + + Parameters + ---------- + parameters : dict + Dictionary of parameters describing the design to be evaluated. + """ + train.report(self.design_evaluator.evaluate_design(parameters)) + def run_optimisation(self) -> ResultGrid: """Run the optimisation loop and return the results. From 2a29e01b24ed7e730cf92bc3152b47f02e5682d8 Mon Sep 17 00:00:00 2001 From: Luke Humphrey Date: Tue, 12 Mar 2024 10:20:06 +0000 Subject: [PATCH 3/5] =?UTF-8?q?=F0=9F=9A=9A=20Use=20trial=20directories=20?= =?UTF-8?q?for=20design=20evaluation=20working=20directory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/example_1_simple_monoblock.py | 4 ++-- examples/example_2_monoblock.py | 4 ++-- examples/example_3_catbird_monoblock.py | 4 ++-- src/sledo/design_evaluator.py | 14 ++------------ 4 files changed, 8 insertions(+), 18 deletions(-) diff --git a/examples/example_1_simple_monoblock.py b/examples/example_1_simple_monoblock.py index d58882c..c32d4d7 100644 --- a/examples/example_1_simple_monoblock.py +++ b/examples/example_1_simple_monoblock.py @@ -25,7 +25,7 @@ # In general, the user will set their own paths and pass them where required. EXAMPLES_DIR = SLEDO_ROOT / "examples" INPUT_FILE = EXAMPLES_DIR / "input_files" / "simple_monoblock_thermomech.i" -WORKING_DIR = EXAMPLES_DIR / "results" / "example_1" +WORKING_DIR = EXAMPLES_DIR / "results" PICKLE_FILEPATH = WORKING_DIR / "example_1_optimiser.pickle" if __name__ == "__main__": @@ -58,7 +58,7 @@ design_evaluator, search_space, max_total_trials=20, - name="simple-monoblock-optimiser", + name="example_1", data_dir=WORKING_DIR, ) diff --git a/examples/example_2_monoblock.py b/examples/example_2_monoblock.py index e14dba9..8481097 100644 --- a/examples/example_2_monoblock.py +++ b/examples/example_2_monoblock.py @@ -25,7 +25,7 @@ # In general, the user will set their own paths and pass them where required. EXAMPLES_DIR = SLEDO_ROOT / "examples" INPUT_FILE = EXAMPLES_DIR / "input_files" / "monoblock_thermomech.i" -WORKING_DIR = EXAMPLES_DIR / "results" / "example_2" +WORKING_DIR = EXAMPLES_DIR / "results" PICKLE_FILEPATH = WORKING_DIR / "example_2_optimiser.pickle" if __name__ == "__main__": @@ -60,7 +60,7 @@ design_evaluator, search_space, max_total_trials=20, - name="monoblock-optimiser", + name="example_2", data_dir=WORKING_DIR, ) diff --git a/examples/example_3_catbird_monoblock.py b/examples/example_3_catbird_monoblock.py index 36d9a2d..6d0e7f1 100644 --- a/examples/example_3_catbird_monoblock.py +++ b/examples/example_3_catbird_monoblock.py @@ -38,7 +38,7 @@ # Set the paths required for this example. # In general, the user will set their own paths and pass them where required. EXAMPLES_DIR = SLEDO_ROOT / "examples" -WORKING_DIR = EXAMPLES_DIR / "results" / "example_3" +WORKING_DIR = EXAMPLES_DIR / "results" FACTORY_CONFIG_PATH = WORKING_DIR / "factory_config.json" INPUT_FILE_PATH = WORKING_DIR / "trial.i" PICKLE_FILEPATH = WORKING_DIR / "example_3_optimiser.pickle" @@ -97,7 +97,7 @@ design_evaluator, search_space, max_total_trials=20, - name="catbird-monoblock-optimiser", + name="example_3", data_dir=WORKING_DIR, ) diff --git a/src/sledo/design_evaluator.py b/src/sledo/design_evaluator.py index b170438..92cc6d0 100644 --- a/src/sledo/design_evaluator.py +++ b/src/sledo/design_evaluator.py @@ -107,7 +107,6 @@ def __init__( self, metrics: list[str], base_input_file: Path | str, - working_dir: Path | str = Path.cwd(), config_path: Path | str = MOOSE_CONFIG_FILE, run_options: dict = { "n_tasks": 1, @@ -127,9 +126,6 @@ def __init__( base_input_file : Path | str Path to the base MOOSE input file (.i) to use as the basis for generating modified files. This file will not be modified. - working_dir : Path | str, optional - Path to the working directory to use for storing modified MOOSE - input files (.i) and running MOOSE, by default Path.cwd(). config_path : Path | str, optional Path to the config file containing the required paths to run MOOSE, by default 'moose_config.json' in the sledo root folder. @@ -139,7 +135,6 @@ def __init__( """ self._metrics = metrics self.base_input_file = Path(base_input_file) - self.working_dir = Path(working_dir) self.config_path = Path(config_path) self.run_options = run_options @@ -170,7 +165,7 @@ def evaluate_design(self, parameters: dict, timestep: int = -1) -> dict: # Generate input file. trial_filepath = generate_modified_input_file( self.base_input_file, - self.working_dir / "trial.i", + Path.cwd() / "trial.i", parameters, ) # Run simulation. @@ -199,7 +194,6 @@ def __init__( self, metrics: list[str], model: MooseModel, - working_dir: Path | str = Path.cwd(), config_path: Path | str = MOOSE_CONFIG_FILE, run_options: dict = { "n_tasks": 1, @@ -219,9 +213,6 @@ def __init__( model : MooseModel A catbird MooseModel capable of updating parameters and writing a MOOSE input file. - working_dir : Path | str, optional - Path to the working directory to use for storing modified MOOSE - input files (.i) and running MOOSE, by default Path.cwd(). config_path : Path | str, optional Path to the config file containing the required paths to run MOOSE, by default 'moose_config.json' in the sledo root folder. @@ -231,7 +222,6 @@ def __init__( """ self._metrics = metrics self._model = model - self.working_dir = Path(working_dir) self.config_path = Path(config_path) self.run_options = run_options @@ -333,7 +323,7 @@ def evaluate_design(self, parameters: dict, timestep: int = -1) -> dict: """ # Generate input file. trial_filepath = self.generate_input_file( - self.working_dir / "trial.i", + Path.cwd() / "trial.i", parameters, ) # Run simulation. From 473119ebbcabd1f9cdeacf73646c639708e60525 Mon Sep 17 00:00:00 2001 From: Luke Humphrey Date: Tue, 12 Mar 2024 10:23:14 +0000 Subject: [PATCH 4/5] =?UTF-8?q?=F0=9F=94=A5=20Remove=20unused=20working=20?= =?UTF-8?q?dir=20arg=20from=20examples?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/example_1_simple_monoblock.py | 1 - examples/example_2_monoblock.py | 1 - examples/example_3_catbird_monoblock.py | 1 - 3 files changed, 3 deletions(-) diff --git a/examples/example_1_simple_monoblock.py b/examples/example_1_simple_monoblock.py index c32d4d7..f7a0e65 100644 --- a/examples/example_1_simple_monoblock.py +++ b/examples/example_1_simple_monoblock.py @@ -39,7 +39,6 @@ design_evaluator = MooseHerderDesignEvaluator( metrics, INPUT_FILE, # The base input file to be modified per design iteration. - working_dir=WORKING_DIR, # Directory to store generated files. config_path=MOOSE_CONFIG_FILE, # Contains required MOOSE paths. ) diff --git a/examples/example_2_monoblock.py b/examples/example_2_monoblock.py index 8481097..c3cb439 100644 --- a/examples/example_2_monoblock.py +++ b/examples/example_2_monoblock.py @@ -39,7 +39,6 @@ design_evaluator = MooseHerderDesignEvaluator( metrics, INPUT_FILE, # The base input file to be modified per design iteration. - working_dir=WORKING_DIR, # Directory to store generated files. config_path=MOOSE_CONFIG_FILE, # Contains required MOOSE paths. ) diff --git a/examples/example_3_catbird_monoblock.py b/examples/example_3_catbird_monoblock.py index 6d0e7f1..6348b2a 100644 --- a/examples/example_3_catbird_monoblock.py +++ b/examples/example_3_catbird_monoblock.py @@ -76,7 +76,6 @@ design_evaluator = CatBirdMooseHerderDesignEvaluator( metrics, model, - working_dir=WORKING_DIR, # Directory to store generated files. config_path=MOOSE_CONFIG_FILE, # Contains required MOOSE paths. ) From 345d075084e74917bcc07ab46b635e206a840b80 Mon Sep 17 00:00:00 2001 From: Luke Humphrey Date: Tue, 12 Mar 2024 10:42:56 +0000 Subject: [PATCH 5/5] =?UTF-8?q?=F0=9F=91=BD=EF=B8=8F=20Add=20workaround=20?= =?UTF-8?q?to=20bug=20where=20trial=20data=20is=20saved=20twice.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: when storage_path passed to RunConfig, data is saved to storage path and to the default ray_results dir. Workaround: set TUNE_RESULT_DIR environment variable to match the passed storage_dir. --- src/sledo/optimiser.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/sledo/optimiser.py b/src/sledo/optimiser.py index 9db36cb..dea83d5 100644 --- a/src/sledo/optimiser.py +++ b/src/sledo/optimiser.py @@ -4,6 +4,7 @@ (c) Copyright UKAEA 2023-2024. """ +import os from pathlib import Path import dill @@ -88,6 +89,10 @@ def __init__( if not self.data_dir.exists(): self.data_dir.mkdir() + # Workaround to a bug which causes ray to save to both the passed + # storage directory and the default (~/ray-results). + os.environ['TUNE_RESULT_DIR'] = str(self.data_dir) + # Instantiate ray tune Tuner object. self.tuner = tune.Tuner( self.trial, @@ -100,6 +105,7 @@ def __init__( run_config=train.RunConfig( storage_path=self.data_dir, name=self.name, + log_to_file=True, ), param_space=search_space, )