From 5b0eb4522b47b30ddcb6625c6c210774fba1b400 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 5 Dec 2024 17:05:20 -0500 Subject: [PATCH 01/18] #224 Add `fre make create-compile` pytests --- fre/make/tests/test_create_compile.py | 97 +++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 fre/make/tests/test_create_compile.py diff --git a/fre/make/tests/test_create_compile.py b/fre/make/tests/test_create_compile.py new file mode 100644 index 00000000..d1792a06 --- /dev/null +++ b/fre/make/tests/test_create_compile.py @@ -0,0 +1,97 @@ +""" +Test fre make create-compile +""" +import os +import shutil +from pathlib import Path +from fre.make import create_compile_script + +# SET-UP +TEST_DIR = Path("fre/make/tests") +NM_EXAMPLE = Path("null_example") +YAMLFILE = "null_model.yaml" +PLATFORM = ["ncrc5.intel23"] +TARGET = ["debug"] +EXPERIMENT = "null_model_full" + +# Create output location +OUT = f"{TEST_DIR}/compile_out" +if Path(OUT).exists(): + # remove + shutil.rmtree(OUT) + # create output directory + Path(OUT).mkdir(parents=True,exist_ok=True) +else: + Path(OUT).mkdir(parents=True,exist_ok=True) + +def test_modelyaml_exists(): + """ + Check the model yaml exists + """ + assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}").exists() + +def test_compileyaml_exists(): + """ + Check the compile yaml exists + """ + assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/compile.yaml").exists() + +def test_platformyaml_exists(): + """ + Check the platform yaml exists + """ + assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/platforms.yaml").exists() + +def test_compile_creation(): + """ + Check for the creation of the compile script + """ + # Save default HOME location + def_home = str(os.environ["HOME"]) + # Change HOME for compile script output + os.environ["HOME"]=OUT + + plat = PLATFORM[0] + targ = TARGET[0] + yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" + + # Create the compile script + create_compile_script.compile_create(yamlfile_path, PLATFORM, TARGET, 4, 1, False, False) + + # Set HOME back to default + os.environ["HOME"] = def_home + + # Check for creation of compile script + # Check for correct default HOME location set + assert [Path(f"{OUT}/fremake_Canopy/test/null_model_full/{plat}-{targ}/exec/compile.sh").exists(), + os.environ["HOME"] == def_home] + +def test_compile_execution(): + """ + Check for the successful execution of the compile script + """ + # Save default HOME location + def_home = str(os.environ["HOME"]) + # Change HOME for compile script output + os.environ["HOME"]=OUT + + plat = PLATFORM[0] + targ = TARGET[0] + yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" + + # Execute the compile script + create_compile_script.compile_create(yamlfile_path, PLATFORM, TARGET, 4, 1, True, False) + + # Set HOME back to default + os.environ["HOME"] = def_home + + # Check for creation of compile script + # Check for FMS directory + # Check for log.compile file + # Check for correct default HOME location set + assert [Path(f"{OUT}/fremake_Canopy/test/null_model_full/{plat}-{targ}/exec/compile.sh").exists(), + Path(f"{OUT}/fremake_Canopy/test/null_model_full/{plat}-{targ}/exec/FMS").is_dir(), + Path(f"{OUT}/fremake_Canopy/test/null_model_full/{plat}-{targ}/exec/log.compile"), + os.environ["HOME"] == def_home] + +#TO-DO: check for failures, ETC.... From 2ca2234efc86950caf11f3e930bf02362a602aef Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 11 Dec 2024 13:12:20 -0500 Subject: [PATCH 02/18] #224 Fix test so it uses null_example instead of AM5 --- fre/tests/test_fre_make_cli.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fre/tests/test_fre_make_cli.py b/fre/tests/test_fre_make_cli.py index 279760b3..9ba88b5f 100644 --- a/fre/tests/test_fre_make_cli.py +++ b/fre/tests/test_fre_make_cli.py @@ -26,7 +26,7 @@ def test_cli_fre_make_create_checkout_baremetal(): ''' fre make create-checkout -y am5.yaml -p ncrc5.intel23 -t debug''' # Set paths and click options test_dir = Path("fre/tests") - yamlfile = Path("fre/make/tests/AM5_example/") + yamlfile = Path("fre/make/tests/null_example") platform = "ncrc5.intel23" target = "debug" @@ -38,12 +38,12 @@ def test_cli_fre_make_create_checkout_baremetal(): os.environ["HOME"]=str(Path(out_path)) # run create-checkout - result = runner.invoke(fre.fre, args=["make", "create-checkout", "-y", f"{yamlfile}/am5.yaml", "-p", platform, "-t", target]) + result = runner.invoke(fre.fre, args=["make", "create-checkout", "-y", f"{yamlfile}/null_model.yaml", "-p", platform, "-t", target]) # Check for successful command, creation of checkout script, and that script is executable (os.access - checks is file has specific access mode, os.X_OK - checks executable permission) assert all ([result.exit_code == 0, - Path(f"{out_path}/fremake_canopy/test/am5/src/checkout.sh").exists(), - os.access(Path(f"{out_path}/fremake_canopy/test/am5/src/checkout.sh"), os.X_OK)]) + Path(f"{out_path}/fremake_canopy/test/null_model_full/src/checkout.sh").exists(), + os.access(Path(f"{out_path}/fremake_canopy/test/null_model_full/src/checkout.sh"), os.X_OK)]) def test_cli_fre_make_create_checkout_container(): ''' fre make create-checkout -y am5.yaml -p hpcme.2023 -t debug''' From e9d6beeeed289321c7405e0352e64e6126fce651 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 11 Dec 2024 13:13:37 -0500 Subject: [PATCH 03/18] #224 Remove AM5 from `fre/make/tests` --- fre/make/tests/AM5_example/am5.yaml | 104 ------------------ fre/make/tests/AM5_example/compile.yaml | 67 ----------- .../AM5_example/yaml_include/platforms.yaml | 26 ----- .../AM5_example/yaml_include/pp.c96_amip.yaml | 88 --------------- 4 files changed, 285 deletions(-) delete mode 100644 fre/make/tests/AM5_example/am5.yaml delete mode 100644 fre/make/tests/AM5_example/compile.yaml delete mode 100644 fre/make/tests/AM5_example/yaml_include/platforms.yaml delete mode 100644 fre/make/tests/AM5_example/yaml_include/pp.c96_amip.yaml diff --git a/fre/make/tests/AM5_example/am5.yaml b/fre/make/tests/AM5_example/am5.yaml deleted file mode 100644 index 359755ec..00000000 --- a/fre/make/tests/AM5_example/am5.yaml +++ /dev/null @@ -1,104 +0,0 @@ -# reusable variables -fre_properties: - - &AM5_VERSION "am5f7b12r1" - - &FRE_STEM !join [am5/, *AM5_VERSION] - - # amip - - &EXP_AMIP_START "1979" - - &EXP_AMIP_END "2020" - - &ANA_AMIP_START "1980" - - &ANA_AMIP_END "2020" - - - &PP_AMIP_CHUNK96 "1yr" - - &PP_AMIP_CHUNK384 "1yr" - - &PP_XYINTERP96 "180,288" - - &PP_XYINTERP384 "720,1152" - - # climo - - &EXP_CLIMO_START96 "0001" - - &EXP_CLIMO_END96 "0011" - - &ANA_CLIMO_START96 "0002" - - &ANA_CLIMO_END96 "0011" - - - &EXP_CLIMO_START384 "0001" - - &EXP_CLIMO_END384 "0006" - - &ANA_CLIMO_START384 "0002" - - &ANA_CLIMO_END384 "0006" - - # coupled - - &PP_CPLD_CHUNK_A "5yr" - - &PP_CPLD_CHUNK_B "20yr" - - # grids - - &GRID_SPEC96 "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" - - # compile information - - &release "f1a1r1" - - &INTEL "intel-classic" - - &FMSincludes "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" - - &momIncludes "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" - -build: - # compile information - compileYaml: "compile.yaml" - platformYaml: "yaml_include/platforms.yaml" - -shared: - # directories shared across tools - directories: &shared_directories - history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] - pp_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] - analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name] - ptmp_dir: "/xtmp/$USER/ptmp" - fre_analysis_home: "/home/fms/local/opt/fre-analysis/test" - - # shared pp settings - postprocess: - settings: &shared_settings - history_segment: "P1Y" - site: "ppan" - switches: &shared_switches - do_statics: True - do_timeavgs: True - clean_work: True - do_refinediag: False - do_atmos_plevel_masking: True - do_preanalysis: False - do_analysis: True - -experiments: - - name: "c96L65_am5f7b12r1_amip" - pp: - - "yaml_include/pp.c96_amip.yaml" - - name: "c96L65_am5f7b12r1_pdclim1850F" - pp: - - "yaml_include/pp.c96_clim.yaml" - - name: "c96L65_am5f7b12r1_pdclim2010F" - pp: - - "yaml_include/pp.c96_clim.yaml" - - name: "c96L65_am5f7b12r1_pdclim2010AERF" - pp: - - "yaml_include/pp.c96_clim.yaml" - - name: "c384L65_am5f7b12r1_amip" - pp: - - "yaml_include/pp.c384_amip.yaml" - - name: "c384L65_am5f7b12r1_pdclim2010F" - pp: - - "yaml_include/pp.c384_clim.yaml" - - name: "c384L65_am5f7b12r1_pdclim1850F" - pp: - - "yaml_include/pp.c384_clim.yaml" - - name: "c384L65_am5f7b12r1_pdclim2010AERF" - pp: - - "yaml_include/pp.c384_clim.yaml" - - name: "c384L65_am5f7b12r1_OM4_p25_piControl_noBLING_DynVeg" - pp: - - "yaml_include/pp.c384_amip.yaml" - - "yaml_include/pp.om4.yaml" - - name: "c96L65_am5f7b12r1_OM4_p25_piControl_noBLING_DynVeg" - pp: - - "yaml_include/pp.c96_amip.yaml" - - "yaml_include/pp.om4.yaml" - - name: "c96L65_am5f7b12r1_amip_cosp" - pp: - - "yaml_include/pp.c96_amip.yaml" diff --git a/fre/make/tests/AM5_example/compile.yaml b/fre/make/tests/AM5_example/compile.yaml deleted file mode 100644 index 5f9a361b..00000000 --- a/fre/make/tests/AM5_example/compile.yaml +++ /dev/null @@ -1,67 +0,0 @@ -compile: - experiment: "am5" - container_addlibs: - baremetal_linkerflags: - src: - - component: "FMS" - repo: "https://github.com/NOAA-GFDL/FMS.git" - cppdefs: "-DINTERNAL_FILE_NML -Duse_libMPI -Duse_netCDF" - branch: "2022.01" - cppdefs: "-DHAVE_GETTID -Duse_libMPI -Duse_netCDF" - otherFlags: *FMSincludes - - component: "am5_phys" - requires: ["FMS"] - repo: "https://gitlab.gfdl.noaa.gov/FMS/am5_phys.git" - branch: "2022.01" - otherFlags: *FMSincludes - - component: "GFDL_atmos_cubed_sphere" - requires: ["FMS", "am5_phys"] - repo: "https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere.git" - cppdefs: "-DSPMD -DCLIMATE_NUDGE -DINTERNAL_FILE_NML" - branch: "2022.01" - paths: ["GFDL_atmos_cubed_sphere/driver/GFDL", - "GFDL_atmos_cubed_sphere/model", - "GFDL_atmos_cubed_sphere/driver/SHiELD/cloud_diagnosis.F90", - "GFDL_atmos_cubed_sphere/driver/SHiELD/gfdl_cloud_microphys.F90", - "GFDL_atmos_cubed_sphere/tools", - "GFDL_atmos_cubed_sphere/GFDL_tools"] - otherFlags: *FMSincludes - - component: "atmos_drivers" - requires: ["FMS", "am5_phys", "GFDL_atmos_cubed_sphere"] - repo: "https://github.com/NOAA-GFDL/atmos_drivers.git" - cppdefs: "-DSPMD -DCLIMATE_NUDGE" - branch: "2022.01" - paths: ["atmos_drivers/coupled"] - otherFlags: *FMSincludes - - component: "ice_sis" - requires: ["FMS", "ice_param", "mom6"] - repo: "https://gitlab.gfdl.noaa.gov/FMS/ice_sis.git" - branch: "2021.02" - otherFlags: !join [*FMSincludes, " ", *momIncludes] - - component: "ice_param" - repo: "https://github.com/NOAA-GFDL/ice_param.git" - cppdefs: "-Duse_yaml -Duse_libMPI -Duse_netCDF" - branch: "2021.02" - requires: ["FMS", "mom6"] - otherFlags: !join [*FMSincludes," ", *momIncludes] - - component: "land_lad2" - requires: ["FMS"] - repo: "https://gitlab.gfdl.noaa.gov/FMS/land_lad2.git" - branch: "2022.01" - branch: "land_lad2_2021.02" - doF90Cpp: True - cppdefs: "-DINTERNAL_FILE_NML" - otherFlags: *FMSincludes - - component: "mom6" - requires: ["FMS"] - paths: ["mom6/MOM6-examples/src/MOM6/config_src/dynamic", "mom6/MOM6-examples/src/MOM6/config_src/coupled_driver", "mom6/MOM6-examples/src/MOM6/src/*/", "mom6/MOM6-examples/src/MOM6/src/*/*/", "mom6/ocean_BGC/generic_tracers", "mom6/ocean_BGC/mocsy/src"] - branch: ["2021.02","dev/gfdl/2018.04.06"] - repo: ["https://github.com/NOAA-GFDL/ocean_BGC.git","https://github.com/NOAA-GFDL/MOM6-examples.git"] - makeOverrides: 'OPENMP=""' - otherFlags: !join [*FMSincludes, " ", *momIncludes] - - component: "FMScoupler" - paths: ["FMScoupler/full", "FMScoupler/shared"] - repo: "https://github.com/NOAA-GFDL/FMScoupler.git" - branch: "2022.01" - requires: ["FMS", "atmos_drivers", "am5_phys", "land_lad2", "ice_sis", "ice_param", "mom6"] - otherFlags: !join [*FMSincludes, " ", *momIncludes] diff --git a/fre/make/tests/AM5_example/yaml_include/platforms.yaml b/fre/make/tests/AM5_example/yaml_include/platforms.yaml deleted file mode 100644 index 60d1aad2..00000000 --- a/fre/make/tests/AM5_example/yaml_include/platforms.yaml +++ /dev/null @@ -1,26 +0,0 @@ -platforms: - - name: ncrc5.intel - compiler: intel - modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"] - modules: [ !join [*INTEL, "/2022.2.1"],"fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3] - fc: ftn - cc: cc - mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/", *INTEL, ".mk"] - modelRoot: ${HOME}/fremake_canopy/test - - name: ncrc5.intel23 - compiler: intel - modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"] - modules: [!join [*INTEL, "/2023.1.0"],"fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3] - fc: ftn - cc: cc - mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/", *INTEL, ".mk"] - modelRoot: ${HOME}/fremake_canopy/test - - name: hpcme.2023 - compiler: intel - RUNenv: [". /spack/share/spack/setup-env.sh", "spack load libyaml", "spack load netcdf-fortran@4.5.4", "spack load hdf5@1.14.0"] - modelRoot: /apps - fc: mpiifort - cc: mpiicc - container: True - containerBuild: "podman" - containerRun: "apptainer" diff --git a/fre/make/tests/AM5_example/yaml_include/pp.c96_amip.yaml b/fre/make/tests/AM5_example/yaml_include/pp.c96_amip.yaml deleted file mode 100644 index 117c66c6..00000000 --- a/fre/make/tests/AM5_example/yaml_include/pp.c96_amip.yaml +++ /dev/null @@ -1,88 +0,0 @@ -# local reusable variable overrides -fre_properties: - - &custom_interp "180,360" - -# directory overrides -#c96_amip_directories: -directories: - <<: *shared_directories - ptmp_dir: "/ptmp/$USER" - pp_grid_spec: *GRID_SPEC96 - -#c96_amip_postprocess: -postprocess: - # pp setting overrides - settings: - <<: *shared_settings - pp_start: *ANA_AMIP_START - pp_stop: *ANA_AMIP_END - pp_chunk_a: *PP_AMIP_CHUNK96 - pp_components: "atmos atmos_scalar" - switches: - <<: *shared_switches - do_statics: False - - # main pp instructions - components: - - type: "atmos_cmip" - sources: "atmos_month_cmip atmos_8xdaily_cmip atmos_daily_cmip" - sourceGrid: "cubedsphere" - xyInterp: *custom_interp - interpMethod: "conserve_order2" - inputRealm: 'atmos' - - type: "atmos" - sources: "atmos_month" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order2" - inputRealm: 'atmos' - - type: "atmos_level_cmip" - sources: "atmos_level_cmip" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order2" - inputRealm: 'atmos' - - type: "atmos_level" - sources: "atmos_month" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order2" - inputRealm: 'atmos' - - type: "atmos_month_aer" - sources: "atmos_month_aer" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order1" - inputRealm: 'atmos' - - type: "atmos_diurnal" - sources: "atmos_diurnal" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order2" - inputRealm: 'atmos' - - type: "atmos_scalar" - sources: "atmos_scalar" - - type: "aerosol_cmip" - xyInterp: *PP_XYINTERP96 - sources: "aerosol_month_cmip" - sourceGrid: "cubedsphere" - interpMethod: "conserve_order1" - inputRealm: 'atmos' - - type: "land" - sources: "land_month" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order1" - inputRealm: 'land' - - type: "land_cmip" - sources: "land_month_cmip" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order1" - inputRealm: 'land' - - type: "tracer_level" - sources: "atmos_tracer" - sourceGrid: "cubedsphere" - xyInterp: *PP_XYINTERP96 - interpMethod: "conserve_order1" - inputRealm: 'atmos' From 419c37a63ca933b5ccad98feef623ce7cb5ffb0f Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 16 Dec 2024 10:35:40 -0500 Subject: [PATCH 04/18] #224 Get rid of AM5 --- fre/tests/test_fre_make_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/tests/test_fre_make_cli.py b/fre/tests/test_fre_make_cli.py index 9ba88b5f..8a5dc577 100644 --- a/fre/tests/test_fre_make_cli.py +++ b/fre/tests/test_fre_make_cli.py @@ -49,7 +49,7 @@ def test_cli_fre_make_create_checkout_container(): ''' fre make create-checkout -y am5.yaml -p hpcme.2023 -t debug''' # Set paths and click options test_dir = Path("fre/tests") - yamlfile = Path("fre/make/tests/AM5_example/") + yamlfile = Path("fre/make/tests/null_example/") platform = "hpcme.2023" target = "debug" From 30959181f886574fb9f7b2c9c77b06f504a7cf12 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 16 Dec 2024 10:37:42 -0500 Subject: [PATCH 05/18] #224 Change `sys.exit` to `click.echo` statement --- fre/make/create_compile_script.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fre/make/create_compile_script.py b/fre/make/create_compile_script.py index 36068329..cf8dc057 100644 --- a/fre/make/create_compile_script.py +++ b/fre/make/create_compile_script.py @@ -79,13 +79,13 @@ def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose): fremakeBuild.writeBuildComponents(c) fremakeBuild.writeScript() fremakeBuildList.append(fremakeBuild) - click.echo("\nCompile script created at " + bldDir + "/compile.sh" + "\n") +# click.echo("\nCompile script created at " + bldDir + "/compile.sh" + "\n") if run: if baremetalRun: pool = Pool(processes=nparallel) # Create a multiprocessing Pool pool.map(buildBaremetal.fremake_parallel,fremakeBuildList) # process data_inputs iterable with pool else: - sys.exit() + click.echo("\nCompile script created at " + bldDir + "/compile.sh" + "\n") @click.command() def _compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose): From 66800365d966b0bb54afee514ccf23a68954fa10 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 16 Dec 2024 10:48:24 -0500 Subject: [PATCH 06/18] #225 Remove more "am5" usages --- fre/tests/test_fre_make_cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fre/tests/test_fre_make_cli.py b/fre/tests/test_fre_make_cli.py index 8a5dc577..d96c41d7 100644 --- a/fre/tests/test_fre_make_cli.py +++ b/fre/tests/test_fre_make_cli.py @@ -23,7 +23,7 @@ def test_cli_fre_make_opt_dne(): assert result.exit_code == 2 def test_cli_fre_make_create_checkout_baremetal(): - ''' fre make create-checkout -y am5.yaml -p ncrc5.intel23 -t debug''' + ''' fre make create-checkout -y null_model.yaml -p ncrc5.intel23 -t debug''' # Set paths and click options test_dir = Path("fre/tests") yamlfile = Path("fre/make/tests/null_example") @@ -46,7 +46,7 @@ def test_cli_fre_make_create_checkout_baremetal(): os.access(Path(f"{out_path}/fremake_canopy/test/null_model_full/src/checkout.sh"), os.X_OK)]) def test_cli_fre_make_create_checkout_container(): - ''' fre make create-checkout -y am5.yaml -p hpcme.2023 -t debug''' + ''' fre make create-checkout -y null_model.yaml -p hpcme.2023 -t debug''' # Set paths and click options test_dir = Path("fre/tests") yamlfile = Path("fre/make/tests/null_example/") @@ -54,7 +54,7 @@ def test_cli_fre_make_create_checkout_container(): target = "debug" # run create-checkout - result = runner.invoke(fre.fre, args=["make", "create-checkout", "-y", f"{yamlfile}/am5.yaml", "-p", platform, "-t", target]) + result = runner.invoke(fre.fre, args=["make", "create-checkout", "-y", f"{yamlfile}/null_model.yaml", "-p", platform, "-t", target]) # Check for successful command, creation of checkout script, and that script is executable (os.access - checks is file has specific access mode, os.X_OK - checks executable permission) assert all ([result.exit_code == 0, From 42d449b2d73f27cd2afd4888b29fe16119334a7a Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 18 Dec 2024 15:10:51 -0500 Subject: [PATCH 07/18] #224 Add output files from fre make tests --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 71828433..e07d03d1 100644 --- a/.gitignore +++ b/.gitignore @@ -61,6 +61,10 @@ fre/tests/test_files/outdirfre/ fre/pp/tests/configure_yaml_out/ fre/yamltools/tests/combine_yamls_out/ fre/app/generate_time_averages/tests/time_avg_test_files/ +fre/make/tests/null_example/combined-null_model.yaml +fre/make/tests/makefile_out +fre/make/tests/compile_out +fre/tests/fremake_out # Translations *.mo From b7047e7a787c62af3bc4cc7b45554e1a7c807c79 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 18 Dec 2024 15:11:13 -0500 Subject: [PATCH 08/18] #224 Remove re-setting HOME; use ci.gnu platform for tests --- fre/make/tests/test_create_compile.py | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/fre/make/tests/test_create_compile.py b/fre/make/tests/test_create_compile.py index d1792a06..e9f2a42e 100644 --- a/fre/make/tests/test_create_compile.py +++ b/fre/make/tests/test_create_compile.py @@ -10,7 +10,7 @@ TEST_DIR = Path("fre/make/tests") NM_EXAMPLE = Path("null_example") YAMLFILE = "null_model.yaml" -PLATFORM = ["ncrc5.intel23"] +PLATFORM = ["ci.gnu"] TARGET = ["debug"] EXPERIMENT = "null_model_full" @@ -24,6 +24,9 @@ else: Path(OUT).mkdir(parents=True,exist_ok=True) +# Set environment variable for use in ci.gnu platform +os.environ["TEST_BUILD_DIR"] = OUT + def test_modelyaml_exists(): """ Check the model yaml exists @@ -46,11 +49,6 @@ def test_compile_creation(): """ Check for the creation of the compile script """ - # Save default HOME location - def_home = str(os.environ["HOME"]) - # Change HOME for compile script output - os.environ["HOME"]=OUT - plat = PLATFORM[0] targ = TARGET[0] yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" @@ -58,23 +56,14 @@ def test_compile_creation(): # Create the compile script create_compile_script.compile_create(yamlfile_path, PLATFORM, TARGET, 4, 1, False, False) - # Set HOME back to default - os.environ["HOME"] = def_home - # Check for creation of compile script # Check for correct default HOME location set - assert [Path(f"{OUT}/fremake_Canopy/test/null_model_full/{plat}-{targ}/exec/compile.sh").exists(), - os.environ["HOME"] == def_home] + assert [Path(f"{OUT}/fremake_Canopy/test/null_model_full/{plat}-{targ}/exec/compile.sh").exists()] def test_compile_execution(): """ Check for the successful execution of the compile script """ - # Save default HOME location - def_home = str(os.environ["HOME"]) - # Change HOME for compile script output - os.environ["HOME"]=OUT - plat = PLATFORM[0] targ = TARGET[0] yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" @@ -82,16 +71,12 @@ def test_compile_execution(): # Execute the compile script create_compile_script.compile_create(yamlfile_path, PLATFORM, TARGET, 4, 1, True, False) - # Set HOME back to default - os.environ["HOME"] = def_home - # Check for creation of compile script # Check for FMS directory # Check for log.compile file # Check for correct default HOME location set assert [Path(f"{OUT}/fremake_Canopy/test/null_model_full/{plat}-{targ}/exec/compile.sh").exists(), Path(f"{OUT}/fremake_Canopy/test/null_model_full/{plat}-{targ}/exec/FMS").is_dir(), - Path(f"{OUT}/fremake_Canopy/test/null_model_full/{plat}-{targ}/exec/log.compile"), - os.environ["HOME"] == def_home] + Path(f"{OUT}/fremake_Canopy/test/null_model_full/{plat}-{targ}/exec/log.compile")] #TO-DO: check for failures, ETC.... From 60b7ce6483fb4a27455848953cce42eb021623a9 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 18 Dec 2024 15:18:51 -0500 Subject: [PATCH 09/18] #224 Clean script/move echo statements --- fre/make/create_compile_script.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fre/make/create_compile_script.py b/fre/make/create_compile_script.py index 74e0c329..cd593021 100644 --- a/fre/make/create_compile_script.py +++ b/fre/make/create_compile_script.py @@ -15,7 +15,6 @@ def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose): name = yamlfile.split(".")[0] nparallel = parallel jobs = str(jobs) - run = execute if verbose: logging.basicCOnfig(level=logging.INFO) @@ -79,8 +78,9 @@ def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose): fremakeBuild.writeBuildComponents(c) fremakeBuild.writeScript() fremakeBuildList.append(fremakeBuild) -# click.echo("\nCompile script created at " + bldDir + "/compile.sh" + "\n") - if run: + + if execute: + click.echo("\nCompile script created at " + bldDir + "/compile.sh" + "\n") if baremetalRun: pool = Pool(processes=nparallel) # Create a multiprocessing Pool pool.map(buildBaremetal.fremake_parallel,fremakeBuildList) # process data_inputs iterable with pool From 61e889c0be1e7e2f58fd14285289d14ce6e7bc0f Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 18 Dec 2024 16:07:40 -0500 Subject: [PATCH 10/18] #224 Add more compile failure tests --- fre/make/tests/test_create_compile.py | 54 +++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/fre/make/tests/test_create_compile.py b/fre/make/tests/test_create_compile.py index e9f2a42e..cd57c918 100644 --- a/fre/make/tests/test_create_compile.py +++ b/fre/make/tests/test_create_compile.py @@ -3,10 +3,11 @@ """ import os import shutil +import pytest from pathlib import Path from fre.make import create_compile_script -# SET-UP +## SET-UP TEST_DIR = Path("fre/make/tests") NM_EXAMPLE = Path("null_example") YAMLFILE = "null_model.yaml" @@ -14,6 +15,13 @@ TARGET = ["debug"] EXPERIMENT = "null_model_full" +# Multi-plat-targ +MULTI_TARGET = ["prod","repro"] + +# Bad plat/targ +BAD_PLATFORM=["no_plat"] +BAD_TARGET=["no_targ"] + # Create output location OUT = f"{TEST_DIR}/compile_out" if Path(OUT).exists(): @@ -57,8 +65,7 @@ def test_compile_creation(): create_compile_script.compile_create(yamlfile_path, PLATFORM, TARGET, 4, 1, False, False) # Check for creation of compile script - # Check for correct default HOME location set - assert [Path(f"{OUT}/fremake_Canopy/test/null_model_full/{plat}-{targ}/exec/compile.sh").exists()] + assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{plat}-{targ}/exec/compile.sh").exists() def test_compile_execution(): """ @@ -74,9 +81,40 @@ def test_compile_execution(): # Check for creation of compile script # Check for FMS directory # Check for log.compile file - # Check for correct default HOME location set - assert [Path(f"{OUT}/fremake_Canopy/test/null_model_full/{plat}-{targ}/exec/compile.sh").exists(), - Path(f"{OUT}/fremake_Canopy/test/null_model_full/{plat}-{targ}/exec/FMS").is_dir(), - Path(f"{OUT}/fremake_Canopy/test/null_model_full/{plat}-{targ}/exec/log.compile")] + assert [Path(f"{OUT}/fremake_canopy/test/null_model_full/{plat}-{targ}/exec/compile.sh").exists(), + Path(f"{OUT}/fremake_canopy/test/null_model_full/{plat}-{targ}/exec/FMS").is_dir(), + Path(f"{OUT}/fremake_canopy/test/null_model_full/{plat}-{targ}/exec/log.compile")] + +@pytest.mark.xfail() +def test_bad_platform(): + """ + Check for the failure of compile script creation + due to a bad platform passed. + """ + yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" + + # Create the compile script + create_compile_script.compile_create(yamlfile_path, BAD_PLATFORM, TARGET, 4, 1, False, False) + +@pytest.mark.xfail() +def test_bad_target(): + """ + Check for the failure of compile script creation + due to a bad target passed. + """ + yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" + + # Create the compile script + create_compile_script.compile_create(yamlfile_path, PLATFORM, BAD_TARGET, 4, 1, False, False) + +def test_multi_target(): + """ + Check for the creation of the compile script for each target passed + """ + yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" + + # Create the compile script + create_compile_script.compile_create(yamlfile_path, PLATFORM, MULTI_TARGET, 4, 1, False, False) -#TO-DO: check for failures, ETC.... + for t in MULTI_TARGET: + assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{PLATFORM[0]}-{t}/exec/compile.sh").exists() From 5c99b29ac9f73663d7487fb7ed584a049338909f Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 18 Dec 2024 17:13:50 -0500 Subject: [PATCH 11/18] #224 Add tests in compile tool test --- fre/make/tests/test_create_compile.py | 30 +++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/fre/make/tests/test_create_compile.py b/fre/make/tests/test_create_compile.py index cd57c918..31a235d0 100644 --- a/fre/make/tests/test_create_compile.py +++ b/fre/make/tests/test_create_compile.py @@ -85,7 +85,7 @@ def test_compile_execution(): Path(f"{OUT}/fremake_canopy/test/null_model_full/{plat}-{targ}/exec/FMS").is_dir(), Path(f"{OUT}/fremake_canopy/test/null_model_full/{plat}-{targ}/exec/log.compile")] -@pytest.mark.xfail() +@pytest.mark.xfail(raises=AttributeError) def test_bad_platform(): """ Check for the failure of compile script creation @@ -96,7 +96,20 @@ def test_bad_platform(): # Create the compile script create_compile_script.compile_create(yamlfile_path, BAD_PLATFORM, TARGET, 4, 1, False, False) -@pytest.mark.xfail() +def test_bad_platform_compilelog(): + """ + Check that compile log still created from the failure + of compile script creation due to a bad platform passed. + """ + yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" + + try: + # Create the compile script + create_compile_script.compile_create(yamlfile_path, BAD_PLATFORM, TARGET, 4, 1, False, False) + except: + assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{BAD_PLATFORM}-{TARGET}/exec/log.compile") + +@pytest.mark.xfail(raises=AttributeError) def test_bad_target(): """ Check for the failure of compile script creation @@ -107,6 +120,19 @@ def test_bad_target(): # Create the compile script create_compile_script.compile_create(yamlfile_path, PLATFORM, BAD_TARGET, 4, 1, False, False) +def test_bad_target_compilelog(): + """ + Check that compile log still created from the failure + of compile script creation due to a bad target passed. + """ + yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" + + try: + # Create the compile script + create_compile_script.compile_create(yamlfile_path, PLATFORM, BAD_TARGET, 4, 1, False, False) + except: + assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{BAD_PLATFORM}-{TARGET}/exec/log.compile") + def test_multi_target(): """ Check for the creation of the compile script for each target passed From 62d1c176c6c1783a2f8a33cee02fdb4eeaf18e7e Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 23 Dec 2024 11:16:55 -0500 Subject: [PATCH 12/18] #224 Remove unecessary print statements --- fre/make/gfdlfremake/platformfre.py | 1 - fre/make/gfdlfremake/yamlfre.py | 1 - 2 files changed, 2 deletions(-) diff --git a/fre/make/gfdlfremake/platformfre.py b/fre/make/gfdlfremake/platformfre.py index 69820d46..a5a3c4cf 100644 --- a/fre/make/gfdlfremake/platformfre.py +++ b/fre/make/gfdlfremake/platformfre.py @@ -57,7 +57,6 @@ def __init__(self,platforminfo): raise Exception("You must specify the program used to build the container (containerBuild) on the "+p["name"]+" platform in the file "+fname+"\n") if p["containerBuild"] != "podman" and p["containerBuild"] != "docker": raise ValueError("Container builds only supported with docker or podman, but you listed "+p["containerBuild"]+"\n") - print (p["containerBuild"]) ## Check for container environment set up for RUN commands try: p["containerBase"] diff --git a/fre/make/gfdlfremake/yamlfre.py b/fre/make/gfdlfremake/yamlfre.py index 0683ad4e..0fa91678 100644 --- a/fre/make/gfdlfremake/yamlfre.py +++ b/fre/make/gfdlfremake/yamlfre.py @@ -171,7 +171,6 @@ def __init__(self,combinedyaml,v): #get platform info self.platformsdict = self.freyaml.get("platforms") - print(self.platformsdict) self.platforms = platformfre.platforms(self.platformsdict) self.platformsyaml = self.platforms.getPlatformsYaml() From 6945dbb33f4c7738e0f563fa7b7b3562a8645228 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 23 Dec 2024 11:24:11 -0500 Subject: [PATCH 13/18] #224 Fix fail case --- fre/make/tests/test_create_compile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/make/tests/test_create_compile.py b/fre/make/tests/test_create_compile.py index 31a235d0..68975abe 100644 --- a/fre/make/tests/test_create_compile.py +++ b/fre/make/tests/test_create_compile.py @@ -109,7 +109,7 @@ def test_bad_platform_compilelog(): except: assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{BAD_PLATFORM}-{TARGET}/exec/log.compile") -@pytest.mark.xfail(raises=AttributeError) +@pytest.mark.xfail(raises=ValueError) def test_bad_target(): """ Check for the failure of compile script creation From 50c0e9e14d2d470ed8d3bba86396cc888cc0b721 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 23 Dec 2024 14:14:03 -0500 Subject: [PATCH 14/18] #224 Fix assertion for multi-target output paths --- fre/make/tests/test_create_compile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fre/make/tests/test_create_compile.py b/fre/make/tests/test_create_compile.py index 68975abe..cdfa45a7 100644 --- a/fre/make/tests/test_create_compile.py +++ b/fre/make/tests/test_create_compile.py @@ -142,5 +142,5 @@ def test_multi_target(): # Create the compile script create_compile_script.compile_create(yamlfile_path, PLATFORM, MULTI_TARGET, 4, 1, False, False) - for t in MULTI_TARGET: - assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{PLATFORM[0]}-{t}/exec/compile.sh").exists() + assert [Path(f"{OUT}/fremake_canopy/test/null_model_full/{PLATFORM[0]}-{MULTI-TARGET[0]}/exec/compile.sh").exists(), + Path(f"{OUT}/fremake_canopy/test/null_model_full/{PLATFORM[0]}-{MULTI-TARGET[1]}/exec/compile.sh").exists()] From c94f5d9ba2fba7509d77099c6d5b432ac71425fc Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 23 Dec 2024 14:36:15 -0500 Subject: [PATCH 15/18] #224 Fix `MULTI_TARGET` variable reference --- fre/make/tests/test_create_compile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fre/make/tests/test_create_compile.py b/fre/make/tests/test_create_compile.py index cdfa45a7..929e518b 100644 --- a/fre/make/tests/test_create_compile.py +++ b/fre/make/tests/test_create_compile.py @@ -142,5 +142,5 @@ def test_multi_target(): # Create the compile script create_compile_script.compile_create(yamlfile_path, PLATFORM, MULTI_TARGET, 4, 1, False, False) - assert [Path(f"{OUT}/fremake_canopy/test/null_model_full/{PLATFORM[0]}-{MULTI-TARGET[0]}/exec/compile.sh").exists(), - Path(f"{OUT}/fremake_canopy/test/null_model_full/{PLATFORM[0]}-{MULTI-TARGET[1]}/exec/compile.sh").exists()] + assert [Path(f"{OUT}/fremake_canopy/test/null_model_full/{PLATFORM[0]}-{MULTI_TARGET[0]}/exec/compile.sh").exists(), + Path(f"{OUT}/fremake_canopy/test/null_model_full/{PLATFORM[0]}-{MULTI_TARGET[1]}/exec/compile.sh").exists()] From 71be40a21d91b5fc1778324bba94663ddb82ba00 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 26 Dec 2024 11:26:08 -0500 Subject: [PATCH 16/18] #224 See where test is running --- fre/make/tests/test_create_compile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/make/tests/test_create_compile.py b/fre/make/tests/test_create_compile.py index 929e518b..3224ea72 100644 --- a/fre/make/tests/test_create_compile.py +++ b/fre/make/tests/test_create_compile.py @@ -63,7 +63,7 @@ def test_compile_creation(): # Create the compile script create_compile_script.compile_create(yamlfile_path, PLATFORM, TARGET, 4, 1, False, False) - + print(os.getcwd()) # Check for creation of compile script assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{plat}-{targ}/exec/compile.sh").exists() From d99dddbc65ec926ff3cacf01556f91bb7d796585 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 26 Dec 2024 11:53:05 -0500 Subject: [PATCH 17/18] #224 Move `os.environ` to inside tests --- fre/make/tests/test_create_compile.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/fre/make/tests/test_create_compile.py b/fre/make/tests/test_create_compile.py index 3224ea72..ae2e590f 100644 --- a/fre/make/tests/test_create_compile.py +++ b/fre/make/tests/test_create_compile.py @@ -32,9 +32,6 @@ else: Path(OUT).mkdir(parents=True,exist_ok=True) -# Set environment variable for use in ci.gnu platform -os.environ["TEST_BUILD_DIR"] = OUT - def test_modelyaml_exists(): """ Check the model yaml exists @@ -57,6 +54,9 @@ def test_compile_creation(): """ Check for the creation of the compile script """ + # Set environment variable for use in ci.gnu platform + os.environ["TEST_BUILD_DIR"] = OUT + plat = PLATFORM[0] targ = TARGET[0] yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" @@ -71,6 +71,9 @@ def test_compile_execution(): """ Check for the successful execution of the compile script """ + # Set environment variable for use in ci.gnu platform + os.environ["TEST_BUILD_DIR"] = OUT + plat = PLATFORM[0] targ = TARGET[0] yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" @@ -91,6 +94,9 @@ def test_bad_platform(): Check for the failure of compile script creation due to a bad platform passed. """ + # Set environment variable for use in ci.gnu platform + os.environ["TEST_BUILD_DIR"] = OUT + yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" # Create the compile script @@ -101,6 +107,9 @@ def test_bad_platform_compilelog(): Check that compile log still created from the failure of compile script creation due to a bad platform passed. """ + # Set environment variable for use in ci.gnu platform + os.environ["TEST_BUILD_DIR"] = OUT + yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" try: @@ -115,6 +124,9 @@ def test_bad_target(): Check for the failure of compile script creation due to a bad target passed. """ + # Set environment variable for use in ci.gnu platform + os.environ["TEST_BUILD_DIR"] = OUT + yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" # Create the compile script @@ -125,6 +137,9 @@ def test_bad_target_compilelog(): Check that compile log still created from the failure of compile script creation due to a bad target passed. """ + # Set environment variable for use in ci.gnu platform + os.environ["TEST_BUILD_DIR"] = OUT + yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" try: @@ -137,6 +152,9 @@ def test_multi_target(): """ Check for the creation of the compile script for each target passed """ + # Set environment variable for use in ci.gnu platform + os.environ["TEST_BUILD_DIR"] = OUT + yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" # Create the compile script From 4b169f4eb98d769ef90378268382d804be32cd75 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 26 Dec 2024 13:43:38 -0500 Subject: [PATCH 18/18] #224 Remove unecessary print statement --- fre/make/tests/test_create_compile.py | 1 - 1 file changed, 1 deletion(-) diff --git a/fre/make/tests/test_create_compile.py b/fre/make/tests/test_create_compile.py index ae2e590f..ab73128d 100644 --- a/fre/make/tests/test_create_compile.py +++ b/fre/make/tests/test_create_compile.py @@ -63,7 +63,6 @@ def test_compile_creation(): # Create the compile script create_compile_script.compile_create(yamlfile_path, PLATFORM, TARGET, 4, 1, False, False) - print(os.getcwd()) # Check for creation of compile script assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{plat}-{targ}/exec/compile.sh").exists()