diff --git a/zppy/e3sm_diags.py b/zppy/e3sm_diags.py index d874d8e3..6d26280a 100644 --- a/zppy/e3sm_diags.py +++ b/zppy/e3sm_diags.py @@ -33,10 +33,12 @@ def e3sm_diags(config, scriptDir, existing_bundles, job_ids_file): # noqa: C901 return existing_bundles # --- Generate and submit e3sm_diags scripts --- - dependencies: List[str] = [] + carried_over_dependencies: List[str] = [] for c in tasks: + dependencies: List[str] = carried_over_dependencies + c["scriptDir"] = scriptDir if "ts_num_years" in c.keys(): @@ -237,6 +239,8 @@ def e3sm_diags(config, scriptDir, existing_bundles, job_ids_file): # noqa: C901 end_yr, c["ts_num_years"], ) + + c["dependencies"] = dependencies with open(settingsFile, "w") as sf: p = pprint.PrettyPrinter(indent=2, stream=sf) p.pprint(c) @@ -267,7 +271,7 @@ def e3sm_diags(config, scriptDir, existing_bundles, job_ids_file): # noqa: C901 # Note that this line should still be executed even if jobid == -1 # The later tc_analysis-using e3sm_diags tasks still depend on this task (and thus will also fail). # Add to the dependency list - dependencies.append(statusFile) + carried_over_dependencies.append(statusFile) else: print("...adding to bundle '%s'" % (c["bundle"])) diff --git a/zppy/global_time_series.py b/zppy/global_time_series.py index 0ca682fd..8cb97005 100644 --- a/zppy/global_time_series.py +++ b/zppy/global_time_series.py @@ -183,6 +183,7 @@ def global_time_series(config, scriptDir, existing_bundles, job_ids_file): # no ) ) + c["dependencies"] = dependencies with open(settingsFile, "w") as sf: p = pprint.PrettyPrinter(indent=2, stream=sf) p.pprint(c) diff --git a/zppy/ilamb.py b/zppy/ilamb.py index f1928f8b..9f0d0661 100644 --- a/zppy/ilamb.py +++ b/zppy/ilamb.py @@ -32,10 +32,10 @@ def ilamb(config, scriptDir, existing_bundles, job_ids_file): return existing_bundles # --- Generate and submit ilamb scripts --- - dependencies: List[str] = [] - for c in tasks: + dependencies: List[str] = [] + if "ts_num_years" in c.keys(): c["ts_num_years"] = int(c["ts_num_years"]) @@ -94,6 +94,7 @@ def ilamb(config, scriptDir, existing_bundles, job_ids_file): f.write(template.render(**c)) makeExecutable(scriptFile) + c["dependencies"] = dependencies with open(settingsFile, "w") as sf: p = pprint.PrettyPrinter(indent=2, stream=sf) p.pprint(c) diff --git a/zppy/mpas_analysis.py b/zppy/mpas_analysis.py index 40a1c664..5c5eb686 100644 --- a/zppy/mpas_analysis.py +++ b/zppy/mpas_analysis.py @@ -1,5 +1,6 @@ import os import pprint +from typing import List import jinja2 @@ -35,10 +36,14 @@ def mpas_analysis(config, scriptDir, existing_bundles, job_ids_file): # job should run at once. To gracefully handle this, we make each # MAPS-Analysis task dependant on all previous ones. This may not # be 100% fool-proof, but should be a reasonable start - dependencies = [] + + # Dependencies carried over from previous task. + carried_over_dependencies: List[str] = [] for c in tasks: + dependencies: List[str] = carried_over_dependencies + if config["mpas_analysis"]["shortTermArchive"]: c["subdir_ocean"] = "/archive/ocn/hist" c["subdir_ice"] = "/archive/ice/hist" @@ -105,6 +110,7 @@ def mpas_analysis(config, scriptDir, existing_bundles, job_ids_file): f.write(template.render(**c)) makeExecutable(scriptFile) + c["dependencies"] = dependencies with open(settingsFile, "w") as sf: p = pprint.PrettyPrinter(indent=2, stream=sf) p.pprint(c) @@ -133,7 +139,7 @@ def mpas_analysis(config, scriptDir, existing_bundles, job_ids_file): # Note that this line should still be executed even if jobid == -1 # The later MPAS-Analysis tasks still depend on this task (and thus will also fail). # Add to the dependency list - dependencies.append(statusFile) + carried_over_dependencies.append(statusFile) else: print("...adding to bundle '%s'" % (c["bundle"])) diff --git a/zppy/tc_analysis.py b/zppy/tc_analysis.py index 52d7124c..10bf51a2 100644 --- a/zppy/tc_analysis.py +++ b/zppy/tc_analysis.py @@ -27,10 +27,14 @@ def tc_analysis(config, scriptDir, existing_bundles, job_ids_file): # There is a `GenerateConnectivityFile: error while loading shared libraries: libnetcdf.so.11: cannot open shared object file: No such file or directory` error # when multiple year_sets are run simultaneously. Therefore, we will wait for the completion of one year_set before moving on to the next. - dependencies: List[str] = [] + + # Dependencies carried over from previous task. + carried_over_dependencies: List[str] = [] for c in tasks: + dependencies: List[str] = carried_over_dependencies + # Loop over year sets year_sets = getYears(c["years"]) for s in year_sets: @@ -61,6 +65,7 @@ def tc_analysis(config, scriptDir, existing_bundles, job_ids_file): f.write(template.render(**c)) makeExecutable(scriptFile) + c["dependencies"] = dependencies with open(settingsFile, "w") as sf: p = pprint.PrettyPrinter(indent=2, stream=sf) p.pprint(c) @@ -88,7 +93,7 @@ def tc_analysis(config, scriptDir, existing_bundles, job_ids_file): # Note that this line should still be executed even if jobid == -1 # The later tc_analysis tasks still depend on this task (and thus will also fail). # Add to the dependency list - dependencies.append(statusFile) + carried_over_dependencies.append(statusFile) else: print("...adding to bundle '%s'" % (c["bundle"]))