Merge pull request #5 from AntaresSimulatorTeam/fix/simpler-cleaner

Refactor : make code easier to understand

scripts/antares_test_utils.py

@@ -1,7 +1,6 @@
 from pathlib import Path
 import glob
 import shutil
-import subprocess
 
 from study import Study
 
@@ -14,81 +13,35 @@ def list_directories(directory):
             dir_list.append(x)
     return dir_list
 
-def list_studies(directory):
+def find_studies_in_batch_dir(batch_name):
+    batch_directory = Path(batch_name).resolve()
     studies = []
-    dir_path = Path(directory)
-    if (dir_path.is_dir()):
-        study = Study(dir_path)
+    if (batch_directory.is_dir()):
+        study = Study(batch_directory)
         if study.check_files_existence():
-            studies.append(directory)
+            studies.append(batch_directory)
         else:
-            for x in dir_path.iterdir():
-                studies.extend(list_studies(x))
+            for x in batch_directory.iterdir():
+                studies.extend(find_studies_in_batch_dir(x))
 
     return studies
 
-def find_output_result_dir(output_dir):
-    list_output_dir = list_directories(output_dir)
-    assert len(list_output_dir) == 1
-
-    list_dir = list_directories(list_output_dir[0])
-
-    dir_list = []
-    for x in list_dir:
-        dir_path = Path(x)
-        if dir_path.is_dir() and (dir_path.name in ["adequacy", "economy", "adequacy-draft"]):
-            dir_list.append(x)
-    assert len(dir_list) == 1
-    return dir_list[0]
-
 def get_headers(df) -> set :
     return set(df.columns)
 
-def remove_outputs(study_path):
+def remove_possibly_remaining_outputs(study_path):
     output_path = study_path / 'output'
-    files = glob.glob(str(output_path))
-    for f in files:
-        shutil.rmtree(f)
-
-def launch_solver(solver_path, study_path, use_ortools = False, ortools_solver = "sirius", named_mps_problems = False, ts_generator_path = ""):
-    # Clean study output
-    remove_outputs(study_path)
-
-    solver_path_full = str(Path(solver_path).resolve())
-
-    command = [solver_path_full, "-i", str(study_path)]
-    if use_ortools:
-        command.append('--use-ortools')
-        command.append('--ortools-solver='+ortools_solver)
-    if named_mps_problems:
-        command.append('--named-mps-problems')
-    if ts_generator_path != "":
-        cluster_to_gen_file = open(study_path / "clustersToGen.txt", 'r')
-        cluster_to_gen = cluster_to_gen_file.readline().rstrip() # remove new line char
-        cluster_to_gen_file.close()
-        command = [ts_generator_path, cluster_to_gen, str(study_path)]
-
-    process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=None)
-    stdout, stderr = process.communicate()
-    exit_code = process.wait()
-
-    return (exit_code == 0)
-
-def generate_reference_values(solver_path, path, use_ortools, ortools_solver, named_mps_problems, ts_generator_path):
-
-    enable_study_output(path, True)
-
-    result = launch_solver(solver_path,path, use_ortools, ortools_solver, named_mps_problems, ts_generator_path)
-
-    output_path = path / 'output'
-    list_dir = list_directories(output_path)
-    assert len(list_dir) == 1
+    shutil.rmtree(output_path, ignore_errors=True)
 
+def move_output_to_reference(study_path):
+    output_path = study_path / 'output'
+    list_dir = list_directories(output_path) # list of 'output' sub-directories
+    if len(list_dir) != 1 : # We should have only 1 results directory
+        raise AssertionError("Too many results directories in output")
     result_dir = list_dir[0]
-
-    reference_path = path / 'output' / 'reference'
+    reference_path = study_path / 'output' / 'reference'
     shutil.move(result_dir, reference_path)
-    return result
+
 
 def enable_study_output(study_path, enable):
     st = Study(study_path)

scripts/generate_reference.py

@@ -1,67 +1,39 @@
 import argparse
-from pathlib import Path
 
 import antares_test_utils as antares_utils
+from run_command_building import *
 import sys
 
+# Parsing args
 parser = argparse.ArgumentParser()
-parser.add_argument("root", help="Directory containing studies",
+parser.add_argument("batch_name", help="Batch directory (containing studies)",
                     type=str)
-
-parser.add_argument("solver", help="Path to antares-solver",
+parser.add_argument("path_where_to_find_exe", help="Path to executable",
                     type=str)
-
+#  Storing args
 args = parser.parse_args()
-root = Path(args.root).resolve()
-solver_path = Path(args.solver).resolve()
-
-def find_binary(path, binary_name):
-    if sys.platform.startswith("win"):
-        suffix=".exe"
-    else:
-        suffix=""
-
-    binary_path = Path(path)
-    if binary_path.is_file():
-        return path.resolve()
-    if binary_path.is_dir():
-        results = []
-        for x in binary_path.iterdir():
-            if x.is_file() and (f"antares-{binary_name}{suffix}" == x.name):
-                results.append(x)
-        assert(len(results) == 1)
-        return results[0].resolve()
-    raise RuntimeError("Missing {binary_name}")
+batch_name = args.batch_name
+path_where_to_find_exe = Path(args.path_where_to_find_exe)
 
+if not path_where_to_find_exe.is_dir() and not path_where_to_find_exe.is_file():
+    raise RuntimeError("Path where to find an executable does not exist")
 
-solver_path = find_binary(args.solver, "solver")
-print(f"Found solver {solver_path}")
 
-ts_generator_path = find_binary(args.solver, "ts-generator")
-print(f"Found ts-generator {ts_generator_path}")
-
-studies = antares_utils.list_studies(root)
-
-def solver_config(study_name):
-    if study_name == "valid-milp":
-        return ("coin", True)
-    else:
-        return ("sirius", False)
+# Looking for studies in batch directory
+study_path_collection = antares_utils.find_studies_in_batch_dir(batch_name)
 
 ret = []
-for study in studies:
-    print(study.name + '...', end='')
+for study_path in study_path_collection:
+    print("Study : %s ... " % study_path.name)
 
-    # Do we need named MPS problems ?
-    named_mps_problems = (study.parent.name == 'valid-named-mps')
-    # Are we testing the time series generator ?
-    if study.parent.name != 'ts-generator':
-        ts_generator_path = ""
-    # What optimization solver to use ?
-    (opt_solver, use_ortools) = solver_config(study.parent.name)
+    antares_utils.remove_possibly_remaining_outputs(study_path)
+    antares_utils.enable_study_output(study_path, True)
 
-    result = antares_utils.generate_reference_values(solver_path, study, use_ortools, opt_solver, named_mps_problems, ts_generator_path)
+    command_to_run = make_command_to_run(path_where_to_find_exe, batch_name, study_path)
+    result = run_command(command_to_run)
     ret.append(result)
     print('OK' if result else 'KO')
 
+    antares_utils.move_output_to_reference(study_path)
+
 sys.exit(not all(ret))

scripts/run_command_building.py

@@ -0,0 +1,50 @@
+from pathlib import Path
+import subprocess
+import sys
+
+def find_exe_path(path_to_search_exe_in, exe_identifier):
+    searched_exe = f"antares-{exe_identifier}"
+    if sys.platform.startswith("win"):
+        searched_exe += ".exe"
+
+    if path_to_search_exe_in.is_file() and path_to_search_exe_in.name == searched_exe:
+        return path_to_search_exe_in.resolve()
+    for path_item in path_to_search_exe_in.iterdir():
+        if path_item.is_file() and (path_item.name == searched_exe):
+            return path_item.resolve()
+    raise RuntimeError("Missing {searched_exe}")
+
+def make_command_to_run(path_where_to_find_exe, batch_name, study_path):
+    command_to_run = []
+    exe_path = Path()
+    exe_identifier = "solver" # Default value
+
+    if batch_name == "ts-generator":
+        exe_identifier = "ts-generator"
+        exe_path = find_exe_path(path_where_to_find_exe, exe_identifier)
+        print(f"Found executabled : {exe_path}")
+
+        cluster_to_gen_file = open(study_path / "clustersToGen.txt", 'r')
+        cluster_to_gen = cluster_to_gen_file.readline().rstrip()
+        cluster_to_gen_file.close()
+        command_to_run = [exe_path, cluster_to_gen, str(study_path)]
+
+    else:
+        exe_path = find_exe_path(path_where_to_find_exe, exe_identifier)
+        print(f"Found executabled : {exe_path}")
+
+        command_to_run = [exe_path, "-i", str(study_path)]
+        if batch_name == "valid-milp":
+            command_to_run.append('--use-ortools')
+            command_to_run.append('--ortools-solver=coin')
+
+        if batch_name == "valid-named-mps":
+            command_to_run.append('--named-mps-problems')
+
+    return command_to_run
+
+def run_command(command):
+    process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=None)
+    process.communicate()
+    exit_code = process.wait()
+    return (exit_code == 0)