diff --git a/fre/make/createCheckout.py b/fre/make/createCheckout.py index ad522ee3..d264ff15 100644 --- a/fre/make/createCheckout.py +++ b/fre/make/createCheckout.py @@ -7,11 +7,7 @@ import click from pathlib import Path from .gfdlfremake import varsfre, platformfre, yamlfre, checkout, targetfre - -# Relative import -f = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.append(f) -import yamltools.combine_yamls as cy +import fre.yamltools.combine_yamls as cy @click.command() def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose): @@ -45,17 +41,14 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v combined = Path(f"combined-{name}.yaml") combined_path=os.path.join(cd,combined) + # Combine model, compile, and platform yamls # If fre yammltools combine-yamls tools was used, the combined yaml should exist if Path(combined_path).exists(): full_combined = combined_path print("\nNOTE: Yamls previously merged.") else: - ## Combine yaml files to parse comb = cy.init_compile_yaml(yml,platform,target) - comb_model = comb.combine_model() - comb_compile = comb.combine_compile() - comb_platform = comb.combine_platforms() - full_combined = comb.clean_yaml() + full_combined = cy.get_combined_compileyaml(comb) ## Get the variables in the model yaml freVars = varsfre.frevars(full_combined) @@ -76,7 +69,7 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v if modelYaml.platforms.hasPlatform(platformName): pass else: - raise SystemExit (platformName + " does not exist in platforms.yaml") #modelYaml.combined.get("compile").get("platformYaml")) + raise ValueError (platformName + " does not exist in platforms.yaml") #modelYaml.combined.get("compile").get("platformYaml")) (compiler,modules,modulesInit,fc,cc,modelRoot,iscontainer,mkTemplate,containerBuild,ContainerRun,RUNenv)=modelYaml.platforms.getPlatformFromName(platformName) ## Create the source directory for the platform diff --git a/fre/make/createCompile.py b/fre/make/createCompile.py index 8dc3905e..07745bbc 100644 --- a/fre/make/createCompile.py +++ b/fre/make/createCompile.py @@ -7,11 +7,7 @@ from multiprocessing.dummy import Pool import click from .gfdlfremake import varsfre, platformfre, yamlfre, targetfre, buildBaremetal - -# Relative import -f = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.append(f) -import yamltools.combine_yamls as cy +import fre.yamltools.combine_yamls as cy @click.command() def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose): @@ -40,16 +36,14 @@ def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose): combined = Path(f"combined-{name}.yaml") combined_path=os.path.join(cd,combined) - if Path(combined_path).exists: + # Combine model, compile, and platform yamls + # If fre yammltools combine-yamls tools was used, the combined yaml should exist + if Path(combined_path).exists(): full_combined = combined_path print("\nNOTE: Yamls previously merged.") else: - ## Combine yaml files to parse comb = cy.init_compile_yaml(yml,platform,target) - comb_model = comb.combine_model() - comb_compile = comb.combine_compile() - comb_platform = comb.combine_platforms() - full_combined = comb.clean_yaml() + full_combined = cy.get_combined_compileyaml(comb) ## Get the variables in the model yaml freVars = varsfre.frevars(full_combined) @@ -70,7 +64,7 @@ def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose): if modelYaml.platforms.hasPlatform(platformName): pass else: - raise SystemExit (platformName + " does not exist in " + modelYaml.combined.get("compile").get("platformYaml")) + raise ValueError (platformName + " does not exist in " + modelYaml.combined.get("compile").get("platformYaml")) (compiler,modules,modulesInit,fc,cc,modelRoot,iscontainer,mkTemplate,containerBuild,ContainerRun,RUNenv)=modelYaml.platforms.getPlatformFromName(platformName) diff --git a/fre/make/createDocker.py b/fre/make/createDocker.py index a775dc9b..7170ac97 100644 --- a/fre/make/createDocker.py +++ b/fre/make/createDocker.py @@ -5,11 +5,7 @@ from pathlib import Path import click from .gfdlfremake import varsfre, targetfre, makefilefre, platformfre, yamlfre, buildDocker - -# Relative import -f = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.append(f) -import yamltools.combine_yamls as cy +import fre.yamltools.combine_yamls as cy @click.command() def dockerfile_create(yamlfile,platform,target,execute): @@ -28,16 +24,14 @@ def dockerfile_create(yamlfile,platform,target,execute): combined = Path(f"combined-{name}.yaml") combined_path=os.path.join(cd,combined) - if Path(combined_path).exists: + # Combine model, compile, and platform yamls + # If fre yammltools combine-yamls tools was used, the combined yaml should exist + if Path(combined_path).exists(): full_combined = combined_path print("\nNOTE: Yamls previously merged.") else: - ## Combine yaml files to parse comb = cy.init_compile_yaml(yml,platform,target) - comb_model = comb.combine_model() - comb_compile = comb.combine_compile() - comb_platform = comb.combine_platforms() - full_combined = comb.clean_yaml() + full_combined = cy.get_combined_compileyaml(comb) ## Get the variables in the model yaml freVars = varsfre.frevars(full_combined) @@ -54,7 +48,7 @@ def dockerfile_create(yamlfile,platform,target,execute): if modelYaml.platforms.hasPlatform(platformName): pass else: - raise SystemExit (platformName + " does not exist in " + modelYaml.combined.get("compile").get("platformYaml")) + raise ValueError (platformName + " does not exist in " + modelYaml.combined.get("compile").get("platformYaml")) (compiler,modules,modulesInit,fc,cc,modelRoot,iscontainer,mkTemplate,containerBuild,containerRun,RUNenv)=modelYaml.platforms.getPlatformFromName(platformName) diff --git a/fre/make/createMakefile.py b/fre/make/createMakefile.py index ce95bc6d..9758c5a5 100644 --- a/fre/make/createMakefile.py +++ b/fre/make/createMakefile.py @@ -5,11 +5,7 @@ from pathlib import Path import click from .gfdlfremake import makefilefre, varsfre, targetfre, yamlfre - -# Relative import -f = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.append(f) -import yamltools.combine_yamls as cy +import fre.yamltools.combine_yamls as cy @click.command() def makefile_create(yamlfile,platform,target): @@ -27,16 +23,14 @@ def makefile_create(yamlfile,platform,target): combined = Path(f"combined-{name}.yaml") combined_path=os.path.join(cd,combined) - if Path(combined_path).exists: + # Combine model, compile, and platform yamls + # If fre yammltools combine-yamls tools was used, the combined yaml should exist + if Path(combined_path).exists(): full_combined = combined_path print("\nNOTE: Yamls previously merged.") else: - ## Combine yaml files to parse comb = cy.init_compile_yaml(yml,platform,target) - comb_model = comb.combine_model() - comb_compile = comb.combine_compile() - comb_platform = comb.combine_platforms() - full_combined = comb.clean_yaml() + full_combined = cy.get_combined_compileyaml(comb) ## Get the variables in the model yaml freVars = varsfre.frevars(full_combined) @@ -53,7 +47,7 @@ def makefile_create(yamlfile,platform,target): if modelYaml.platforms.hasPlatform(platformName): pass else: - raise SystemExit (platformName + " does not exist in " + modelYaml.combined.get("compile").get("platformYaml")) + raise ValueError (platformName + " does not exist in " + modelYaml.combined.get("compile").get("platformYaml")) (compiler,modules,modulesInit,fc,cc,modelRoot,iscontainer,mkTemplate,containerBuild,ContainerRun,RUNenv)=modelYaml.platforms.getPlatformFromName(platformName) diff --git a/fre/make/runFremake.py b/fre/make/runFremake.py index fe024037..821336c5 100644 --- a/fre/make/runFremake.py +++ b/fre/make/runFremake.py @@ -8,11 +8,9 @@ import os import logging from multiprocessing.dummy import Pool - import click - from .gfdlfremake import targetfre, varsfre, yamlfre, checkout, makefilefre, buildDocker, buildBaremetal - +import fre.yamltools.combine_yamls as cy @click.command() def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose): @@ -41,17 +39,19 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verb plist = platform tlist = target - # If fre yamltools combine-yamls tools was used, the combined yaml should exist + ## If combined yaml does not exist, combine model, compile, and platform yamls + cd = Path.cwd() + combined = Path(f"combined-{name}.yaml") + combined_path=os.path.join(cd,combined) + + # Combine model, compile, and platform yamls + # If fre yammltools combine-yamls tools was used, the combined yaml should exist if Path(combined_path).exists(): full_combined = combined_path print("\nNOTE: Yamls previously merged.") else: - ## Combine yaml files to parse comb = cy.init_compile_yaml(yml,platform,target) - comb_model = comb.combine_model() - comb_compile = comb.combine_compile() - comb_platform = comb.combine_platforms() - full_combined = comb.clean_yaml() + full_combined = cy.get_combined_compileyaml(comb) ## Get the variables in the model yaml freVars = varsfre.frevars(full_combined) @@ -72,7 +72,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verb if modelYaml.platforms.hasPlatform(platformName): pass else: - raise SystemExit (platformName + " does not exist in " + modelYaml.combined.get("compile").get("platformYaml")) + raise ValueError (platformName + " does not exist in " + modelYaml.combined.get("compile").get("platformYaml")) (compiler,modules,modulesInit,fc,cc,modelRoot,iscontainer,mkTemplate,containerBuild,ContainerRun,RUNenv)=modelYaml.platforms.getPlatformFromName(platformName) @@ -98,7 +98,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verb if modelYaml.platforms.hasPlatform(platformName): pass else: - raise SystemExit (platformName + " does not exist in " + modelYaml.platformsfile) + raise ValueError (platformName + " does not exist in " + modelYaml.platformsfile) (compiler,modules,modulesInit,fc,cc,modelRoot,iscontainer,mkTemplate,containerBuild,containerRun,RUNenv)=modelYaml.platforms.getPlatformFromName(platformName) ## Make the source directory based on the modelRoot and platform diff --git a/fre/pp/configure_script_yaml.py b/fre/pp/configure_script_yaml.py index f8874bd0..2d7d9ac6 100644 --- a/fre/pp/configure_script_yaml.py +++ b/fre/pp/configure_script_yaml.py @@ -167,18 +167,14 @@ def _yamlInfo(yamlfile,experiment,platform,target): combined = Path(f"combined-{e}.yaml") combined_path=os.path.join(cd,combined) - # If fre yamltools combine-yamls tools was used, the combined yaml should exist + # Combine model, experiment, and analysis yamls + # If fre yammltools combine-yamls tools was used, the combined yaml should exist if Path(combined_path).exists(): - ## Make sure that the previously created combined yaml is valid full_combined = combined_path - print("\nNOTE: Yamls merged from combine-yamls tool") + print("\nNOTE: Yamls previously merged.") else: - ## Combine yaml files to parse comb = cy.init_pp_yaml(yml,e,p,t) - comb_model = comb.combine_model() - comb_exp = comb.combine_experiment() - comb_analysis = comb.combine_analysis() - full_combined = comb.clean_yaml() + full_combined = cy.get_combined_compileyaml(comb) # Validate yaml validate_yaml(full_combined) diff --git a/fre/yamltools/combine_yamls.py b/fre/yamltools/combine_yamls.py index 1520f3bb..577f9968 100755 --- a/fre/yamltools/combine_yamls.py +++ b/fre/yamltools/combine_yamls.py @@ -280,6 +280,40 @@ def clean_yaml(self): print(f"Combined yaml located here: {os.path.dirname(self.combined)}/{self.combined}") return self.combined +def get_combined_compileyaml(comb): + """ + Combine the model, compile, and platform yamls + Arguments: + - comb : combined yaml object + """ + # Merge model into combined file + comb_model = comb.combine_model() + # Merge compile.yaml into combined file + comb_compile = comb.combine_compile() + # Merge platforms.yaml into combined file + comb_platform = comb.combine_platforms() + # Clean the yaml + full_combined = comb.clean_yaml() + + return full_combined + +def get_combined_ppyaml(comb): + """ + Combine the model, experiment, and analysis yamls + Arguments: + - comb : comine yaml object + """ + # Merge model into combined file + comb_model = comb.combine_model() + # Merge pp experiment yamls into combined file + comb_exp = comb.combine_experiment() + # Merge pp analysis yamls, if defined, into combined file + comb_analysis = comb.combine_analysis() + # Clean the yaml + full_combined = comb.clean_yaml() + + return full_combined + ########################################################################################### def _consolidate_yamls(yamlfile,experiment,platform,target,use): # Regsiter tag handler @@ -289,28 +323,10 @@ def _consolidate_yamls(yamlfile,experiment,platform,target,use): mainyaml_dir = os.path.dirname(yamlfile) if use == "compile": - # Define yaml object - comb = init_compile_yaml(yamlfile,platform,target) - # Merge model into combined file - comb.combine_model() - # Merge compile.yaml into combined file - comb.combine_compile() - # Merge platforms.yaml into combined file - comb.combine_platforms() - # Clean the yaml - comb.clean_yaml() + get_combined_compileyaml(yamlfile,platform,target) if use =="pp": - # Define yaml object - comb = init_pp_yaml(yamlfile,experiment,platform,target) - # Merge model into combined file - comb.combine_model() - # Merge pp experiment yamls into combined file - comb.combine_experiment() - # Merge pp analysis yamls, if defined, into combined file - comb.combine_analysis() - # Clean the yaml - comb.clean_yaml() + get_combined_ppyaml(yamlfile,experiment,platform,target) @click.command() def consolidate_yamls(yamlfile,experiment,platform,target,use):