Compile yaml integration and parse correctly

fre/make/createCheckout.py

@@ -1,17 +1,19 @@
 #!/usr/bin/python3
 
 import os
+import subprocess
 import logging
 import sys
 import click
+from pathlib import Path
 from .gfdlfremake import varsfre, platformfre, yamlfre, checkout, targetfre
+import fre.yamltools.combine_yamls as cy
 
 @click.command()
 def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose):
     # Define variables
     yml = yamlfile
-    ps = platform
-    ts = target
+    name = yamlfile.split(".")[0]
     run = execute
     jobs = str(jobs)
     pcheck = no_parallel_checkout
@@ -34,11 +36,16 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v
     plist = platform
     tlist = target
 
+    # Combine model, compile, and platform yamls
+    # Default behavior - combine yamls / rewrite combined yaml
+    comb = cy.init_compile_yaml(yml,platform,target)
+    full_combined = cy.get_combined_compileyaml(comb)
+
     ## Get the variables in the model yaml
-    freVars = varsfre.frevars(yml)
+    freVars = varsfre.frevars(full_combined) 
 
-    ## Open the yaml file and parse as fremakeYaml
-    modelYaml = yamlfre.freyaml(yml,freVars)
+    ## Open the yaml file, validate the yaml, and parse as fremakeYaml
+    modelYaml = yamlfre.freyaml(full_combined,freVars)
     fremakeYaml = modelYaml.getCompileYaml()
 
     ## Error checking the targets
@@ -53,7 +60,7 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v
          if modelYaml.platforms.hasPlatform(platformName):
               pass
          else:
-              raise SystemExit (platformName + " does not exist in " + modelYaml.platformsfile)
+              raise ValueError (platformName + " does not exist in platforms.yaml") 
          (compiler,modules,modulesInit,fc,cc,modelRoot,iscontainer,mkTemplate,containerBuild,ContainerRun,RUNenv)=modelYaml.platforms.getPlatformFromName(platformName)
 
     ## Create the source directory for the platform
@@ -67,22 +74,34 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v
                    freCheckout = checkout.checkout("checkout.sh",srcDir)
                    freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(),jobs,pc)
                    freCheckout.finish(pc)
-                   click.echo("\nCheckout script created at " + srcDir + "/checkout.sh" + "\n")
+                   click.echo("\nCheckout script created in "+ srcDir + "/checkout.sh \n")
 
-              # Run the checkout script
-              if run:
-                   freCheckout.run()
+                   # Run the checkout script
+                   if run == True:
+                        freCheckout.run()
+                   else:
+                        sys.exit()
               else:
-                   sys.exit()
+                   print("\nCheckout script PREVIOUSLY created in "+ srcDir + "/checkout.sh \n")
+                   if run == True:
+                        os.chmod(srcDir+"/checkout.sh", 0o744)
+                        try:
+                             subprocess.run(args=[srcDir+"/checkout.sh"], check=True)
+                        except:
+                             print("\nThere was an error with the checkout script "+srcDir+"/checkout.sh.",
+                                   "\nTry removing test folder: " + modelRoot +"\n")
+                             raise
+                   else:
+                        sys.exit()
+
          else:
-              ## Run the checkout script
               image="ecpe4s/noaa-intel-prototype:2023.09.25"
               bldDir = modelRoot + "/" + fremakeYaml["experiment"] + "/exec"
               tmpDir = "tmp/"+platformName
               freCheckout = checkout.checkoutForContainer("checkout.sh", srcDir, tmpDir)
               freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(),jobs,pc)
               freCheckout.finish(pc)
-              click.echo("\nCheckout script created at " + srcDir + "/checkout.sh" + "\n")
+              click.echo("\nCheckout script created at " + tmpDir + "/checkout.sh" + "\n")
 
 
 if __name__ == "__main__":

fre/make/createCompile.py

@@ -3,17 +3,17 @@
 import os
 import sys
 import logging
+from pathlib import Path
 from multiprocessing.dummy import Pool
-
 import click
 from .gfdlfremake import varsfre, platformfre, yamlfre, targetfre, buildBaremetal
+import fre.yamltools.combine_yamls as cy
 
 @click.command()
 def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose):
     # Define variables
     yml = yamlfile
-    ps = platform
-    ts = target
+    name = yamlfile.split(".")[0]
     nparallel = parallel
     jobs = str(jobs)
     run = execute
@@ -31,11 +31,18 @@ def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose):
     plist = platform
     tlist = target
 
+    # Combined compile yaml file
+    combined = Path(f"combined-{name}.yaml")
+
+    ## If combined yaml exists, note message of its existence
+    ## If combined yaml does not exist, combine model, compile, and platform yamls
+    full_combined = cy.combined_compile_existcheck(combined,yml,platform,target)
+
     ## Get the variables in the model yaml
-    freVars = varsfre.frevars(yml)
+    freVars = varsfre.frevars(full_combined)
 
     ## Open the yaml file and parse as fremakeYaml
-    modelYaml = yamlfre.freyaml(yml,freVars)
+    modelYaml = yamlfre.freyaml(full_combined,freVars)
     fremakeYaml = modelYaml.getCompileYaml()
 
     ## Error checking the targets
@@ -50,7 +57,8 @@ def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose):
          if modelYaml.platforms.hasPlatform(platformName):
               pass
          else:
-              raise SystemExit (platformName + " does not exist in " + modelYaml.platformsfile)
+              raise ValueError (platformName + " does not exist in " + modelYaml.combined.get("compile").get("platformYaml"))
+
          (compiler,modules,modulesInit,fc,cc,modelRoot,iscontainer,mkTemplate,containerBuild,ContainerRun,RUNenv)=modelYaml.platforms.getPlatformFromName(platformName)
     ## Make the bldDir based on the modelRoot, the platform, and the target
          srcDir = modelRoot + "/" + fremakeYaml["experiment"] + "/src"

fre/make/createDocker.py

@@ -2,24 +2,35 @@
 
 import os
 import sys
+from pathlib import Path
 import click
 from .gfdlfremake import varsfre, targetfre, makefilefre, platformfre, yamlfre, buildDocker
+import fre.yamltools.combine_yamls as cy
 
 @click.command()
-def dockerfile_create(yamlfile, platform, target, execute):
+def dockerfile_create(yamlfile,platform,target,execute):
     srcDir="src"
     checkoutScriptName = "checkout.sh"
     baremetalRun = False # This is needed if there are no bare metal runs
     ## Split and store the platforms and targets in a list
     plist = platform
     tlist = target
     yml = yamlfile
+    name = yamlfile.split(".")[0]
     run = execute
 
+    # Combined compile yaml file
+    combined = Path(f"combined-{name}.yaml")
+
+    ## If combined yaml exists, note message of its existence
+    ## If combined yaml does not exist, combine model, compile, and platform yamls
+    full_combined = cy.combined_compile_existcheck(combined,yml,platform,target)
+
     ## Get the variables in the model yaml
-    freVars = varsfre.frevars(yml)
+    freVars = varsfre.frevars(full_combined)
+
     ## Open the yaml file and parse as fremakeYaml
-    modelYaml = yamlfre.freyaml(yml,freVars)
+    modelYaml = yamlfre.freyaml(full_combined,freVars)
     fremakeYaml = modelYaml.getCompileYaml()
 
     fremakeBuildList = []
@@ -30,7 +41,7 @@ def dockerfile_create(yamlfile, platform, target, execute):
             if modelYaml.platforms.hasPlatform(platformName):
                 pass
             else:
-                raise SystemExit (platformName + " does not exist in " + modelYaml.platformsfile)
+                raise ValueError (platformName + " does not exist in " + modelYaml.combined.get("compile").get("platformYaml"))
 
             (compiler,modules,modulesInit,fc,cc,modelRoot,iscontainer,mkTemplate,containerBuild,containerRun,RUNenv)=modelYaml.platforms.getPlatformFromName(platformName)
 
@@ -42,33 +53,21 @@ def dockerfile_create(yamlfile, platform, target, execute):
                 bldDir = modelRoot + "/" + fremakeYaml["experiment"] + "/exec"
                 tmpDir = "tmp/"+platformName
 
-                freMakefile = makefilefre.makefileContainer(exp = fremakeYaml["experiment"],
-                                                      libs = fremakeYaml["container_addlibs"],
-                                                      srcDir = srcDir,
-                                                      bldDir = bldDir,
-                                                      mkTemplatePath = mkTemplate,
-                                                      tmpDir = tmpDir)
-
-                # Loop through components and send the component name and requires for the Makefile
-                for c in fremakeYaml['src']:
-                     freMakefile.addComponent(c['component'],c['requires'],c['makeOverrides'])
-                freMakefile.writeMakefile()
-
                 dockerBuild = buildDocker.container(base = image,
                                               exp = fremakeYaml["experiment"],
                                               libs = fremakeYaml["container_addlibs"],
                                               RUNenv = RUNenv,
                                               target = targetObject)
                 dockerBuild.writeDockerfileCheckout("checkout.sh", tmpDir+"/checkout.sh")
-                dockerBuild.writeDockerfileMakefile(freMakefile.getTmpDir() + "/Makefile", freMakefile.getTmpDir()+"/linkline.sh")
+                dockerBuild.writeDockerfileMakefile(tmpDir+"/Makefile", tmpDir+"/linkline.sh")
+
                 for c in fremakeYaml['src']:
                     dockerBuild.writeDockerfileMkmf(c)
 
                 dockerBuild.writeRunscript(RUNenv,containerRun,tmpDir+"/execrunscript.sh")
-
                 currDir = os.getcwd()
-                click.echo("\ntmpDir created at " + currDir + "/tmp")
-                click.echo("Dockerfile created at " + currDir + "\n")
+                click.echo("\ntmpDir created in " + currDir + "/tmp")
+                click.echo("Dockerfile created in " + currDir +"\n")
 
             if run:
                 dockerBuild.build(containerBuild, containerRun)