#141 Create separate function to combine yamls

- create separate functions to call for combining compile and pp yamls - Also: fix import for combine_yamls functions, change `SystemExit` to `ValueError`
NOAA-GFDL · Sep 10, 2024 · 1256ec5 · 1256ec5
1 parent 644e64d
commit 1256ec5
Show file tree

Hide file tree

Showing 7 changed files with 73 additions and 86 deletions.
diff --git a/fre/make/createCheckout.py b/fre/make/createCheckout.py
@@ -7,11 +7,7 @@
 import click
 from pathlib import Path
 from .gfdlfremake import varsfre, platformfre, yamlfre, checkout, targetfre
-
-# Relative import
-f = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-sys.path.append(f)
-import yamltools.combine_yamls as cy
+import fre.yamltools.combine_yamls as cy
 
 @click.command()
 def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose):
@@ -45,17 +41,14 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v
     combined = Path(f"combined-{name}.yaml")
     combined_path=os.path.join(cd,combined)
 
+    # Combine model, compile, and platform yamls
     # If fre yammltools combine-yamls tools was used, the combined yaml should exist
     if Path(combined_path).exists():
         full_combined = combined_path
         print("\nNOTE: Yamls previously merged.")
     else:
-        ## Combine yaml files to parse
         comb = cy.init_compile_yaml(yml,platform,target)
-        comb_model = comb.combine_model()
-        comb_compile = comb.combine_compile()
-        comb_platform = comb.combine_platforms()
-        full_combined = comb.clean_yaml()
+        full_combined = cy.get_combined_compileyaml(comb)
 
     ## Get the variables in the model yaml
     freVars = varsfre.frevars(full_combined) 
@@ -76,7 +69,7 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v
          if modelYaml.platforms.hasPlatform(platformName):
               pass
          else:
-              raise SystemExit (platformName + " does not exist in platforms.yaml") #modelYaml.combined.get("compile").get("platformYaml"))
+              raise ValueError (platformName + " does not exist in platforms.yaml") #modelYaml.combined.get("compile").get("platformYaml"))
          (compiler,modules,modulesInit,fc,cc,modelRoot,iscontainer,mkTemplate,containerBuild,ContainerRun,RUNenv)=modelYaml.platforms.getPlatformFromName(platformName)
 
     ## Create the source directory for the platform

diff --git a/fre/make/createCompile.py b/fre/make/createCompile.py
@@ -7,11 +7,7 @@
 from multiprocessing.dummy import Pool
 import click
 from .gfdlfremake import varsfre, platformfre, yamlfre, targetfre, buildBaremetal
-
-# Relative import
-f = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-sys.path.append(f)
-import yamltools.combine_yamls as cy
+import fre.yamltools.combine_yamls as cy
 
 @click.command()
 def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose):
@@ -40,16 +36,14 @@ def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose):
     combined = Path(f"combined-{name}.yaml")
     combined_path=os.path.join(cd,combined)
 
-    if Path(combined_path).exists:
+    # Combine model, compile, and platform yamls
+    # If fre yammltools combine-yamls tools was used, the combined yaml should exist
+    if Path(combined_path).exists():
         full_combined = combined_path
         print("\nNOTE: Yamls previously merged.")
     else:
-        ## Combine yaml files to parse
         comb = cy.init_compile_yaml(yml,platform,target)
-        comb_model = comb.combine_model()
-        comb_compile = comb.combine_compile()
-        comb_platform = comb.combine_platforms()
-        full_combined = comb.clean_yaml()
+        full_combined = cy.get_combined_compileyaml(comb)
 
     ## Get the variables in the model yaml
     freVars = varsfre.frevars(full_combined)
@@ -70,7 +64,7 @@ def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose):
          if modelYaml.platforms.hasPlatform(platformName):
               pass
          else:
-              raise SystemExit (platformName + " does not exist in " + modelYaml.combined.get("compile").get("platformYaml"))
+              raise ValueError (platformName + " does not exist in " + modelYaml.combined.get("compile").get("platformYaml"))
 
          (compiler,modules,modulesInit,fc,cc,modelRoot,iscontainer,mkTemplate,containerBuild,ContainerRun,RUNenv)=modelYaml.platforms.getPlatformFromName(platformName)
 

diff --git a/fre/make/createDocker.py b/fre/make/createDocker.py
@@ -5,11 +5,7 @@
 from pathlib import Path
 import click
 from .gfdlfremake import varsfre, targetfre, makefilefre, platformfre, yamlfre, buildDocker
-
-# Relative import
-f = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-sys.path.append(f)
-import yamltools.combine_yamls as cy
+import fre.yamltools.combine_yamls as cy
 
 @click.command()
 def dockerfile_create(yamlfile,platform,target,execute):
@@ -28,16 +24,14 @@ def dockerfile_create(yamlfile,platform,target,execute):
     combined = Path(f"combined-{name}.yaml")
     combined_path=os.path.join(cd,combined)
 
-    if Path(combined_path).exists:
+    # Combine model, compile, and platform yamls
+    # If fre yammltools combine-yamls tools was used, the combined yaml should exist
+    if Path(combined_path).exists():
         full_combined = combined_path
         print("\nNOTE: Yamls previously merged.")
     else:
-        ## Combine yaml files to parse
         comb = cy.init_compile_yaml(yml,platform,target)
-        comb_model = comb.combine_model()
-        comb_compile = comb.combine_compile()
-        comb_platform = comb.combine_platforms()
-        full_combined = comb.clean_yaml()
+        full_combined = cy.get_combined_compileyaml(comb)
 
     ## Get the variables in the model yaml
     freVars = varsfre.frevars(full_combined)
@@ -54,7 +48,7 @@ def dockerfile_create(yamlfile,platform,target,execute):
             if modelYaml.platforms.hasPlatform(platformName):
                 pass
             else:
-                raise SystemExit (platformName + " does not exist in " + modelYaml.combined.get("compile").get("platformYaml"))
+                raise ValueError (platformName + " does not exist in " + modelYaml.combined.get("compile").get("platformYaml"))
 
             (compiler,modules,modulesInit,fc,cc,modelRoot,iscontainer,mkTemplate,containerBuild,containerRun,RUNenv)=modelYaml.platforms.getPlatformFromName(platformName)
 

diff --git a/fre/make/createMakefile.py b/fre/make/createMakefile.py
@@ -5,11 +5,7 @@
 from pathlib import Path
 import click
 from .gfdlfremake import makefilefre, varsfre, targetfre, yamlfre
-
-# Relative import
-f = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-sys.path.append(f)
-import yamltools.combine_yamls as cy
+import fre.yamltools.combine_yamls as cy 
 
 @click.command()
 def makefile_create(yamlfile,platform,target):
@@ -27,16 +23,14 @@ def makefile_create(yamlfile,platform,target):
     combined = Path(f"combined-{name}.yaml")
     combined_path=os.path.join(cd,combined)
 
-    if Path(combined_path).exists:
+    # Combine model, compile, and platform yamls
+    # If fre yammltools combine-yamls tools was used, the combined yaml should exist
+    if Path(combined_path).exists():
         full_combined = combined_path
         print("\nNOTE: Yamls previously merged.")
     else:
-        ## Combine yaml files to parse
         comb = cy.init_compile_yaml(yml,platform,target)
-        comb_model = comb.combine_model()
-        comb_compile = comb.combine_compile()
-        comb_platform = comb.combine_platforms()
-        full_combined = comb.clean_yaml()
+        full_combined = cy.get_combined_compileyaml(comb)
 
     ## Get the variables in the model yaml
     freVars = varsfre.frevars(full_combined)
@@ -53,7 +47,7 @@ def makefile_create(yamlfile,platform,target):
             if modelYaml.platforms.hasPlatform(platformName):
                 pass
             else:
-                raise SystemExit (platformName + " does not exist in " + modelYaml.combined.get("compile").get("platformYaml"))
+                raise ValueError (platformName + " does not exist in " + modelYaml.combined.get("compile").get("platformYaml"))
 
             (compiler,modules,modulesInit,fc,cc,modelRoot,iscontainer,mkTemplate,containerBuild,ContainerRun,RUNenv)=modelYaml.platforms.getPlatformFromName(platformName)
 

diff --git a/fre/make/runFremake.py b/fre/make/runFremake.py
@@ -8,11 +8,9 @@
 import os
 import logging
 from multiprocessing.dummy import Pool
-
 import click
-
 from .gfdlfremake import targetfre, varsfre, yamlfre, checkout, makefilefre, buildDocker, buildBaremetal
-
+import fre.yamltools.combine_yamls as cy
 
 @click.command()
 def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose):
@@ -41,17 +39,19 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verb
     plist = platform
     tlist = target
 
-    # If fre yamltools combine-yamls tools was used, the combined yaml should exist
+    ## If combined yaml does not exist, combine model, compile, and platform yamls
+    cd = Path.cwd()
+    combined = Path(f"combined-{name}.yaml")
+    combined_path=os.path.join(cd,combined)
+
+    # Combine model, compile, and platform yamls
+    # If fre yammltools combine-yamls tools was used, the combined yaml should exist
     if Path(combined_path).exists():
         full_combined = combined_path
         print("\nNOTE: Yamls previously merged.")
     else:
-        ## Combine yaml files to parse
         comb = cy.init_compile_yaml(yml,platform,target)
-        comb_model = comb.combine_model()
-        comb_compile = comb.combine_compile()
-        comb_platform = comb.combine_platforms()
-        full_combined = comb.clean_yaml()
+        full_combined = cy.get_combined_compileyaml(comb)
 
     ## Get the variables in the model yaml
     freVars = varsfre.frevars(full_combined)
@@ -72,7 +72,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verb
          if modelYaml.platforms.hasPlatform(platformName):
               pass
          else:
-              raise SystemExit (platformName + " does not exist in " + modelYaml.combined.get("compile").get("platformYaml"))
+              raise ValueError (platformName + " does not exist in " + modelYaml.combined.get("compile").get("platformYaml"))
 
          (compiler,modules,modulesInit,fc,cc,modelRoot,iscontainer,mkTemplate,containerBuild,ContainerRun,RUNenv)=modelYaml.platforms.getPlatformFromName(platformName)
 
@@ -98,7 +98,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verb
          if modelYaml.platforms.hasPlatform(platformName):
               pass
          else:
-              raise SystemExit (platformName + " does not exist in " + modelYaml.platformsfile)
+              raise ValueError (platformName + " does not exist in " + modelYaml.platformsfile)
          (compiler,modules,modulesInit,fc,cc,modelRoot,iscontainer,mkTemplate,containerBuild,containerRun,RUNenv)=modelYaml.platforms.getPlatformFromName(platformName)
 
          ## Make the source directory based on the modelRoot and platform

diff --git a/fre/pp/configure_script_yaml.py b/fre/pp/configure_script_yaml.py
@@ -167,18 +167,14 @@ def _yamlInfo(yamlfile,experiment,platform,target):
     combined = Path(f"combined-{e}.yaml")
     combined_path=os.path.join(cd,combined)
 
-    # If fre yamltools combine-yamls tools was used, the combined yaml should exist
+    # Combine model, experiment, and analysis yamls
+    # If fre yammltools combine-yamls tools was used, the combined yaml should exist
     if Path(combined_path).exists():
-        ## Make sure that the previously created combined yaml is valid
         full_combined = combined_path
-        print("\nNOTE: Yamls merged from combine-yamls tool")
+        print("\nNOTE: Yamls previously merged.")
     else:
-        ## Combine yaml files to parse
         comb = cy.init_pp_yaml(yml,e,p,t)
-        comb_model = comb.combine_model()
-        comb_exp = comb.combine_experiment()
-        comb_analysis = comb.combine_analysis()
-        full_combined = comb.clean_yaml()
+        full_combined = cy.get_combined_compileyaml(comb)
 
     # Validate yaml
     validate_yaml(full_combined)

diff --git a/fre/yamltools/combine_yamls.py b/fre/yamltools/combine_yamls.py
@@ -280,6 +280,40 @@ def clean_yaml(self):
       print(f"Combined yaml located here: {os.path.dirname(self.combined)}/{self.combined}")
       return self.combined
 
+def get_combined_compileyaml(comb):
+    """
+    Combine the model, compile, and platform yamls
+    Arguments:
+        - comb : combined yaml object 
+    """
+    # Merge model into combined file
+    comb_model = comb.combine_model()
+    # Merge compile.yaml into combined file
+    comb_compile = comb.combine_compile()
+    # Merge platforms.yaml into combined file
+    comb_platform = comb.combine_platforms()
+    # Clean the yaml
+    full_combined = comb.clean_yaml()
+
+    return full_combined
+
+def get_combined_ppyaml(comb):
+    """
+    Combine the model, experiment, and analysis yamls
+    Arguments:
+        - comb : comine yaml object
+    """
+    # Merge model into combined file
+    comb_model = comb.combine_model()
+    # Merge pp experiment yamls into combined file
+    comb_exp = comb.combine_experiment()
+    # Merge pp analysis yamls, if defined, into combined file
+    comb_analysis = comb.combine_analysis()
+    # Clean the yaml
+    full_combined = comb.clean_yaml()
+
+    return full_combined
+
 ###########################################################################################
 def _consolidate_yamls(yamlfile,experiment,platform,target,use):
     # Regsiter tag handler
@@ -289,28 +323,10 @@ def _consolidate_yamls(yamlfile,experiment,platform,target,use):
     mainyaml_dir = os.path.dirname(yamlfile)
 
     if use == "compile":
-        # Define yaml object
-        comb = init_compile_yaml(yamlfile,platform,target)
-        # Merge model into combined file
-        comb.combine_model()
-        # Merge compile.yaml into combined file
-        comb.combine_compile()
-        # Merge platforms.yaml into combined file
-        comb.combine_platforms()
-        # Clean the yaml
-        comb.clean_yaml()
+        get_combined_compileyaml(yamlfile,platform,target)
 
     if use =="pp":
-        # Define yaml object
-        comb = init_pp_yaml(yamlfile,experiment,platform,target)
-        # Merge model into combined file
-        comb.combine_model()
-        # Merge pp experiment yamls into combined file
-        comb.combine_experiment()
-        # Merge pp analysis yamls, if defined, into combined file
-        comb.combine_analysis()
-        # Clean the yaml
-        comb.clean_yaml()
+        get_combined_ppyaml(yamlfile,experiment,platform,target)
 
 @click.command()
 def consolidate_yamls(yamlfile,experiment,platform,target,use):