Improve carryover dependency handling

zppy/e3sm_diags.py

@@ -33,10 +33,12 @@ def e3sm_diags(config, scriptDir, existing_bundles, job_ids_file):  # noqa: C901
         return existing_bundles
 
     # --- Generate and submit e3sm_diags scripts ---
-    dependencies: List[str] = []
+    carried_over_dependencies: List[str] = []
 
     for c in tasks:
 
+        dependencies: List[str] = carried_over_dependencies
+
         c["scriptDir"] = scriptDir
 
         if "ts_num_years" in c.keys():
@@ -237,6 +239,8 @@ def e3sm_diags(config, scriptDir, existing_bundles, job_ids_file):  # noqa: C901
                             end_yr,
                             c["ts_num_years"],
                         )
+
+            c["dependencies"] = dependencies
             with open(settingsFile, "w") as sf:
                 p = pprint.PrettyPrinter(indent=2, stream=sf)
                 p.pprint(c)
@@ -267,7 +271,7 @@ def e3sm_diags(config, scriptDir, existing_bundles, job_ids_file):  # noqa: C901
                         # Note that this line should still be executed even if jobid == -1
                         # The later tc_analysis-using e3sm_diags tasks still depend on this task (and thus will also fail).
                         # Add to the dependency list
-                        dependencies.append(statusFile)
+                        carried_over_dependencies.append(statusFile)
                 else:
                     print("...adding to bundle '%s'" % (c["bundle"]))
 

zppy/global_time_series.py

@@ -183,6 +183,7 @@ def global_time_series(config, scriptDir, existing_bundles, job_ids_file):  # no
                         )
                     )
 
+            c["dependencies"] = dependencies
             with open(settingsFile, "w") as sf:
                 p = pprint.PrettyPrinter(indent=2, stream=sf)
                 p.pprint(c)

zppy/ilamb.py

@@ -32,10 +32,10 @@ def ilamb(config, scriptDir, existing_bundles, job_ids_file):
         return existing_bundles
 
     # --- Generate and submit ilamb scripts ---
-    dependencies: List[str] = []
-
     for c in tasks:
 
+        dependencies: List[str] = []
+
         if "ts_num_years" in c.keys():
             c["ts_num_years"] = int(c["ts_num_years"])
 
@@ -94,6 +94,7 @@ def ilamb(config, scriptDir, existing_bundles, job_ids_file):
                 f.write(template.render(**c))
             makeExecutable(scriptFile)
 
+            c["dependencies"] = dependencies
             with open(settingsFile, "w") as sf:
                 p = pprint.PrettyPrinter(indent=2, stream=sf)
                 p.pprint(c)

zppy/mpas_analysis.py

@@ -1,5 +1,6 @@
 import os
 import pprint
+from typing import List
 
 import jinja2
 
@@ -35,10 +36,14 @@ def mpas_analysis(config, scriptDir, existing_bundles, job_ids_file):
     # job should run at once. To gracefully handle this, we make each
     # MAPS-Analysis task dependant on all previous ones. This may not
     # be 100% fool-proof, but should be a reasonable start
-    dependencies = []
+
+    # Dependencies carried over from previous task.
+    carried_over_dependencies: List[str] = []
 
     for c in tasks:
 
+        dependencies: List[str] = carried_over_dependencies
+
         if config["mpas_analysis"]["shortTermArchive"]:
             c["subdir_ocean"] = "/archive/ocn/hist"
             c["subdir_ice"] = "/archive/ice/hist"
@@ -105,6 +110,7 @@ def mpas_analysis(config, scriptDir, existing_bundles, job_ids_file):
                 f.write(template.render(**c))
             makeExecutable(scriptFile)
 
+            c["dependencies"] = dependencies
             with open(settingsFile, "w") as sf:
                 p = pprint.PrettyPrinter(indent=2, stream=sf)
                 p.pprint(c)
@@ -133,7 +139,7 @@ def mpas_analysis(config, scriptDir, existing_bundles, job_ids_file):
                     # Note that this line should still be executed even if jobid == -1
                     # The later MPAS-Analysis tasks still depend on this task (and thus will also fail).
                     # Add to the dependency list
-                    dependencies.append(statusFile)
+                    carried_over_dependencies.append(statusFile)
                 else:
                     print("...adding to bundle '%s'" % (c["bundle"]))
 

zppy/tc_analysis.py

@@ -27,10 +27,14 @@ def tc_analysis(config, scriptDir, existing_bundles, job_ids_file):
 
     # There is a `GenerateConnectivityFile: error while loading shared libraries: libnetcdf.so.11: cannot open shared object file: No such file or directory` error
     # when multiple year_sets are run simultaneously. Therefore, we will wait for the completion of one year_set before moving on to the next.
-    dependencies: List[str] = []
+
+    # Dependencies carried over from previous task.
+    carried_over_dependencies: List[str] = []
 
     for c in tasks:
 
+        dependencies: List[str] = carried_over_dependencies
+
         # Loop over year sets
         year_sets = getYears(c["years"])
         for s in year_sets:
@@ -61,6 +65,7 @@ def tc_analysis(config, scriptDir, existing_bundles, job_ids_file):
                 f.write(template.render(**c))
             makeExecutable(scriptFile)
 
+            c["dependencies"] = dependencies
             with open(settingsFile, "w") as sf:
                 p = pprint.PrettyPrinter(indent=2, stream=sf)
                 p.pprint(c)
@@ -88,7 +93,7 @@ def tc_analysis(config, scriptDir, existing_bundles, job_ids_file):
                     # Note that this line should still be executed even if jobid == -1
                     # The later tc_analysis tasks still depend on this task (and thus will also fail).
                     # Add to the dependency list
-                    dependencies.append(statusFile)
+                    carried_over_dependencies.append(statusFile)
                 else:
                     print("...adding to bundle '%s'" % (c["bundle"]))